From 24413844ae1c7b9b24b69f452e76cf73da662c34 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 30 May 2026 14:23:40 +0200 Subject: [PATCH 1/4] Add Windows release binaries (#127) * Add Windows release binaries * Fix Windows installer downloads --- .github/workflows/ci.yml | 57 +++++++++++ .github/workflows/release-edge.yml | 46 ++++++++- .github/workflows/release.yml | 47 ++++++++- docs/dev/ci.md | 5 +- docs/user/deployment.md | 8 +- docs/user/install.md | 62 +++++++++++- scripts/install.ps1 | 151 +++++++++++++++++++++++++++++ scripts/local-rustfs-bootstrap.sh | 3 - 8 files changed, 364 insertions(+), 15 deletions(-) create mode 100644 scripts/install.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3dc2e80..918a472 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,6 +249,63 @@ jobs: if: needs.classify_changes.outputs.run_full_ci == 'true' run: cargo test --locked -p omnigraph-server --features aws + test_windows_binaries: + name: Test Windows release binaries + needs: classify_changes + runs-on: windows-latest + timeout-minutes: 75 + permissions: + contents: read + env: + CARGO_TERM_COLOR: always + steps: + - name: Skip for text-only changes + if: needs.classify_changes.outputs.run_full_ci != 'true' + run: Write-Host "Text-only change detected; skipping Windows binary build." + + - name: Checkout source + if: needs.classify_changes.outputs.run_full_ci == 'true' + uses: actions/checkout@v5.0.1 + + - name: Install system dependencies + if: needs.classify_changes.outputs.run_full_ci == 'true' + run: choco install protoc -y + + - name: Install Rust stable + if: needs.classify_changes.outputs.run_full_ci == 'true' + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - name: Cache Rust build data + if: needs.classify_changes.outputs.run_full_ci == 'true' + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + . -> target + key: windows-release-binaries + + - name: Build Windows binaries + if: needs.classify_changes.outputs.run_full_ci == 'true' + run: cargo build --release --locked -p omnigraph-cli -p omnigraph-server + + - name: Smoke test Windows binaries + if: needs.classify_changes.outputs.run_full_ci == 'true' + run: | + & ./target/release/omnigraph.exe version + & ./target/release/omnigraph-server.exe --help + + - name: Check PowerShell installer syntax + if: needs.classify_changes.outputs.run_full_ci == 'true' + run: | + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile("scripts/install.ps1", [ref]$tokens, [ref]$errors) | Out-Null + if ($errors.Count -gt 0) { + $errors | Format-List + exit 1 + } + rustfs_integration: name: RustFS S3 Integration needs: diff --git a/.github/workflows/release-edge.yml b/.github/workflows/release-edge.yml index 6147646..3996e65 100644 --- a/.github/workflows/release-edge.yml +++ b/.github/workflows/release-edge.yml @@ -43,6 +43,8 @@ jobs: asset_name: omnigraph-linux-x86_64 - runner: macos-14 asset_name: omnigraph-macos-arm64 + - runner: windows-latest + asset_name: omnigraph-windows-x86_64 env: CARGO_TERM_COLOR: always steps: @@ -59,6 +61,10 @@ jobs: if: runner.os == 'macOS' run: brew install protobuf + - name: Install Windows dependencies + if: runner.os == 'Windows' + run: choco install protoc -y + - name: Install Rust stable uses: dtolnay/rust-toolchain@stable with: @@ -73,7 +79,8 @@ jobs: - name: Build release binaries run: cargo build --release --locked -p omnigraph-cli -p omnigraph-server - - name: Package release archive + - name: Package Unix release archive + if: runner.os != 'Windows' run: | mkdir -p release install -m 0755 target/release/omnigraph release/omnigraph @@ -81,6 +88,22 @@ jobs: tar -C release -czf "${{ matrix.asset_name }}.tar.gz" omnigraph omnigraph-server shasum -a 256 "${{ matrix.asset_name }}.tar.gz" > "${{ matrix.asset_name }}.sha256" + - name: Package Windows release archive + if: runner.os == 'Windows' + run: | + New-Item -ItemType Directory -Force -Path release | Out-Null + Copy-Item target/release/omnigraph.exe release/omnigraph.exe + Copy-Item target/release/omnigraph-server.exe release/omnigraph-server.exe + Compress-Archive -Path release/omnigraph.exe, release/omnigraph-server.exe -DestinationPath "${{ matrix.asset_name }}.zip" -Force + $hash = (Get-FileHash "${{ matrix.asset_name }}.zip" -Algorithm SHA256).Hash.ToLowerInvariant() + "$hash ${{ matrix.asset_name }}.zip" | Out-File -FilePath "${{ matrix.asset_name }}.sha256" -Encoding ascii + New-Item -ItemType Directory -Force -Path verify | Out-Null + Expand-Archive -Path "${{ matrix.asset_name }}.zip" -DestinationPath verify -Force + $items = Get-ChildItem -Path verify -File + if ($items.Count -ne 2 -or !(Test-Path verify/omnigraph.exe) -or !(Test-Path verify/omnigraph-server.exe)) { + throw "Windows release archive is missing expected binaries" + } + - name: Publish edge release assets uses: softprops/action-gh-release@v2.5.0 with: @@ -91,5 +114,22 @@ jobs: body: | Rolling prerelease from `${{ github.sha }}`. files: | - ${{ matrix.asset_name }}.tar.gz - ${{ matrix.asset_name }}.sha256 + ${{ matrix.asset_name }}.* + + smoke_windows_installer: + name: Smoke Windows installer + needs: build_release + runs-on: windows-latest + permissions: + contents: read + steps: + - name: Checkout source + uses: actions/checkout@v5.0.1 + + - name: Install from edge release + run: ./scripts/install.ps1 -ReleaseChannel edge -InstallDir "$env:RUNNER_TEMP/omnigraph-bin" + + - name: Smoke installed binaries + run: | + & "$env:RUNNER_TEMP/omnigraph-bin/omnigraph.exe" version + & "$env:RUNNER_TEMP/omnigraph-bin/omnigraph-server.exe" --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7fc75f..48ab38c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,8 @@ jobs: asset_name: omnigraph-linux-x86_64 - runner: macos-14 asset_name: omnigraph-macos-arm64 + - runner: windows-latest + asset_name: omnigraph-windows-x86_64 env: CARGO_TERM_COLOR: always steps: @@ -36,6 +38,10 @@ jobs: if: runner.os == 'macOS' run: brew install protobuf + - name: Install Windows dependencies + if: runner.os == 'Windows' + run: choco install protoc -y + - name: Install Rust stable uses: dtolnay/rust-toolchain@stable with: @@ -50,7 +56,8 @@ jobs: - name: Build release binaries run: cargo build --release --locked -p omnigraph-cli -p omnigraph-server - - name: Package release archive + - name: Package Unix release archive + if: runner.os != 'Windows' run: | mkdir -p release install -m 0755 target/release/omnigraph release/omnigraph @@ -58,12 +65,27 @@ jobs: tar -C release -czf "${{ matrix.asset_name }}.tar.gz" omnigraph omnigraph-server shasum -a 256 "${{ matrix.asset_name }}.tar.gz" > "${{ matrix.asset_name }}.sha256" + - name: Package Windows release archive + if: runner.os == 'Windows' + run: | + New-Item -ItemType Directory -Force -Path release | Out-Null + Copy-Item target/release/omnigraph.exe release/omnigraph.exe + Copy-Item target/release/omnigraph-server.exe release/omnigraph-server.exe + Compress-Archive -Path release/omnigraph.exe, release/omnigraph-server.exe -DestinationPath "${{ matrix.asset_name }}.zip" -Force + $hash = (Get-FileHash "${{ matrix.asset_name }}.zip" -Algorithm SHA256).Hash.ToLowerInvariant() + "$hash ${{ matrix.asset_name }}.zip" | Out-File -FilePath "${{ matrix.asset_name }}.sha256" -Encoding ascii + New-Item -ItemType Directory -Force -Path verify | Out-Null + Expand-Archive -Path "${{ matrix.asset_name }}.zip" -DestinationPath verify -Force + $items = Get-ChildItem -Path verify -File + if ($items.Count -ne 2 -or !(Test-Path verify/omnigraph.exe) -or !(Test-Path verify/omnigraph-server.exe)) { + throw "Windows release archive is missing expected binaries" + } + - name: Publish GitHub release assets uses: softprops/action-gh-release@v2.5.0 with: files: | - ${{ matrix.asset_name }}.tar.gz - ${{ matrix.asset_name }}.sha256 + ${{ matrix.asset_name }}.* update_homebrew_tap: name: Update Homebrew tap @@ -113,3 +135,22 @@ jobs: git add Formula/omnigraph.rb git commit -m "Update Omnigraph formula to ${GITHUB_REF_NAME}" git push origin HEAD:main + + smoke_windows_installer: + name: Smoke Windows installer + needs: build_release + if: startsWith(github.ref, 'refs/tags/v') + runs-on: windows-latest + permissions: + contents: read + steps: + - name: Checkout source + uses: actions/checkout@v5.0.1 + + - name: Install from tagged release + run: ./scripts/install.ps1 -Version "$env:GITHUB_REF_NAME" -InstallDir "$env:RUNNER_TEMP/omnigraph-bin" + + - name: Smoke installed binaries + run: | + & "$env:RUNNER_TEMP/omnigraph-bin/omnigraph.exe" version + & "$env:RUNNER_TEMP/omnigraph-bin/omnigraph-server.exe" --help diff --git a/docs/dev/ci.md b/docs/dev/ci.md index 8495d5e..1124cb4 100644 --- a/docs/dev/ci.md +++ b/docs/dev/ci.md @@ -4,7 +4,8 @@ - **ci.yml**: text-only changes skip; otherwise `cargo test --workspace --locked` on ubuntu-latest with protobuf compiler. OpenAPI-drift check that auto-commits the regenerated `openapi.json` for same-repository PRs. Also runs the AGENTS.md cross-link integrity check (`scripts/check-agents-md.sh`). - **AWS feature build job**: `cargo build/test -p omnigraph-server --features aws` on ubuntu-latest. +- **Windows binary build job**: `cargo build --release --locked -p omnigraph-cli -p omnigraph-server` on windows-latest with smoke checks for `omnigraph.exe version`, `omnigraph-server.exe --help`, and PowerShell installer syntax. - **RustFS S3 integration**: spins up RustFS in Docker, runs `s3_storage`, `server_opens_s3_graph_directly_and_serves_snapshot_and_read`, and `local_cli_s3_end_to_end_init_load_read_flow`. -- **release-edge.yml**: on every push to main, retags `edge`, builds Linux x86_64 / macOS arm64 archives + sha256, publishes a rolling prerelease. -- **release.yml**: on `v*` tags, builds the Linux x86_64 / macOS arm64 release matrix and updates the Homebrew tap (`scripts/update-homebrew-formula.sh`) by pushing the regenerated formula to `ModernRelay/homebrew-tap`. +- **release-edge.yml**: on every push to main, retags `edge`, builds Linux x86_64 / macOS arm64 archives and Windows x86_64 zip + sha256, publishes a rolling prerelease, then smoke-tests the Windows PowerShell installer against `edge`. +- **release.yml**: on `v*` tags, builds the Linux x86_64 / macOS arm64 archives and Windows x86_64 zip release matrix, updates the Homebrew tap (`scripts/update-homebrew-formula.sh`) by pushing the regenerated formula to `ModernRelay/homebrew-tap`, and smoke-tests the Windows PowerShell installer against the tag. - **package.yml**: manual ECR image build; emits two image tags per commit (``, `-aws`) via CodeBuild. diff --git a/docs/user/deployment.md b/docs/user/deployment.md index fc5ee08..8613dec 100644 --- a/docs/user/deployment.md +++ b/docs/user/deployment.md @@ -20,6 +20,8 @@ Build or install: - `omnigraph` - `omnigraph-server` +On Windows, the binaries are `omnigraph.exe` and `omnigraph-server.exe`. + Run against a local graph: ```bash @@ -141,8 +143,10 @@ The server binary ships in two flavors: | **AWS** | `cargo build --release --features aws` | Adds AWS Secrets Manager backend for bearer tokens | Tagged release archives contain the default `omnigraph` and -`omnigraph-server` binaries. AWS-enabled server binaries are built from source -with `cargo build --release --features aws -p omnigraph-server` when needed. +`omnigraph-server` binaries on macOS / Linux, and `omnigraph.exe` plus +`omnigraph-server.exe` on Windows. AWS-enabled server binaries are built from +source with `cargo build --release --features aws -p omnigraph-server` when +needed. The AWS build adds ~150 transitive deps and ~30-60s of first-build compile time. Default builds don't pay that cost. diff --git a/docs/user/install.md b/docs/user/install.md index ea9fb8c..4a11372 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -2,16 +2,29 @@ ## Quick Install +macOS / Linux: + ```bash curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/install.sh | bash ``` +Windows PowerShell: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -Command "iwr -UseBasicParsing https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/install.ps1 | iex" +``` + By default the installer places: - `omnigraph` - `omnigraph-server` -in `~/.local/bin`. +in `~/.local/bin` on macOS / Linux, or: + +- `omnigraph.exe` +- `omnigraph-server.exe` + +in `%USERPROFILE%\.local\bin` on Windows. The default installer is binary-only. It downloads a published release asset, verifies the SHA256 checksum, and unpacks it. It does not build from source. @@ -39,6 +52,13 @@ Rolling edge binaries from `main`: curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/install.sh | RELEASE_CHANNEL=edge bash ``` +Windows rolling edge binaries: + +```powershell +iwr -UseBasicParsing https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/install.ps1 -OutFile install.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File .\install.ps1 -ReleaseChannel edge +``` + Install from source: ```bash @@ -53,12 +73,24 @@ Install to a different directory: curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/install.sh | INSTALL_DIR="$HOME/bin" bash ``` +Windows: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File .\install.ps1 -InstallDir "$env:USERPROFILE\bin" +``` + Install a specific tag: ```bash curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/install.sh | VERSION=v0.1.0 bash ``` +Windows: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File .\install.ps1 -Version v0.1.0 +``` + Build from a specific git ref: ```bash @@ -67,27 +99,53 @@ curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/ ## Manual Source Build +macOS / Linux: + ```bash cargo build --release --locked -p omnigraph-cli -p omnigraph-server install -m 0755 target/release/omnigraph ~/.local/bin/omnigraph install -m 0755 target/release/omnigraph-server ~/.local/bin/omnigraph-server ``` +Windows: + +```powershell +cargo build --release --locked -p omnigraph-cli -p omnigraph-server +New-Item -ItemType Directory -Force "$env:USERPROFILE\.local\bin" | Out-Null +Copy-Item target\release\omnigraph.exe "$env:USERPROFILE\.local\bin\omnigraph.exe" +Copy-Item target\release\omnigraph-server.exe "$env:USERPROFILE\.local\bin\omnigraph-server.exe" +``` + ## Release Assets Tagged releases are expected to publish: - `omnigraph-linux-x86_64.tar.gz` - `omnigraph-macos-arm64.tar.gz` +- `omnigraph-windows-x86_64.zip` -Each archive contains both binaries: +The macOS / Linux archives contain both binaries: - `omnigraph` - `omnigraph-server` +The Windows archive contains: + +- `omnigraph.exe` +- `omnigraph-server.exe` + ## Verify The Install +macOS / Linux: + ```bash omnigraph version omnigraph-server --help ``` + +Windows: + +```powershell +omnigraph.exe version +omnigraph-server.exe --help +``` diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 0000000..3bfd0f1 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,151 @@ +param( + [string]$RepoSlug = "ModernRelay/omnigraph", + [string]$InstallDir = "$env:USERPROFILE\.local\bin", + [ValidateSet("stable", "edge")] + [string]$ReleaseChannel = "stable", + [string]$Version = "" +) + +$ErrorActionPreference = "Stop" + +$assetName = "omnigraph-windows-x86_64.zip" +$assetStem = "omnigraph-windows-x86_64" +$workDir = Join-Path ([System.IO.Path]::GetTempPath()) ("omnigraph-install-" + [System.Guid]::NewGuid().ToString("N")) +$selectedChannel = "" + +function Write-Log { + param([string]$Message) + Write-Host "==> $Message" +} + +function Get-ReleaseBaseUrl { + param([string]$Channel) + + if ($Version -ne "") { + return "https://github.com/$RepoSlug/releases/download/$Version" + } + + if ($Channel -eq "stable") { + return "https://github.com/$RepoSlug/releases/latest/download" + } + + if ($Channel -eq "edge") { + return "https://github.com/$RepoSlug/releases/download/edge" + } + + throw "unsupported ReleaseChannel '$Channel' (expected stable or edge)" +} + +function Download-ReleaseFiles { + param( + [string]$BaseUrl, + [string]$ArchivePath, + [string]$ChecksumPath + ) + + try { + Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/$assetName" -OutFile $ArchivePath + Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/$assetStem.sha256" -OutFile $ChecksumPath + return $true + } catch { + return $false + } +} + +function Verify-Checksum { + param( + [string]$ArchivePath, + [string]$ChecksumPath + ) + + $checksumText = (Get-Content -Path $ChecksumPath -Raw).Trim() + $expected = ($checksumText -split "\s+")[0].ToLowerInvariant() + if ($expected -eq "") { + throw "checksum file did not contain a SHA256 digest" + } + + $actual = (Get-FileHash -Path $ArchivePath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -ne $expected) { + throw "checksum verification failed for $assetName" + } +} + +function Install-FromDirectory { + param([string]$SourceDir) + + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + Copy-Item -Path (Join-Path $SourceDir "omnigraph.exe") -Destination (Join-Path $InstallDir "omnigraph.exe") -Force + Copy-Item -Path (Join-Path $SourceDir "omnigraph-server.exe") -Destination (Join-Path $InstallDir "omnigraph-server.exe") -Force +} + +function Install-FromRelease { + New-Item -ItemType Directory -Force -Path $workDir | Out-Null + + $archivePath = Join-Path $workDir $assetName + $checksumPath = Join-Path $workDir "$assetStem.sha256" + + if ($Version -ne "") { + $script:selectedChannel = $Version + $baseUrl = Get-ReleaseBaseUrl -Channel $ReleaseChannel + Write-Log "Downloading $assetName from $Version" + if (!(Download-ReleaseFiles -BaseUrl $baseUrl -ArchivePath $archivePath -ChecksumPath $checksumPath)) { + throw "no published binary found for $assetName at release $Version" + } + } else { + $script:selectedChannel = $ReleaseChannel + $baseUrl = Get-ReleaseBaseUrl -Channel $selectedChannel + Write-Log "Downloading $assetName from $selectedChannel" + if (!(Download-ReleaseFiles -BaseUrl $baseUrl -ArchivePath $archivePath -ChecksumPath $checksumPath)) { + if ($ReleaseChannel -ne "stable") { + throw "no published binary found for $assetName on channel $ReleaseChannel" + } + + Write-Log "Stable release binaries are not published yet; falling back to edge" + $script:selectedChannel = "edge" + $baseUrl = Get-ReleaseBaseUrl -Channel $selectedChannel + if (!(Download-ReleaseFiles -BaseUrl $baseUrl -ArchivePath $archivePath -ChecksumPath $checksumPath)) { + throw "no published binary found for $assetName on stable or edge; build from source" + } + } + } + + Verify-Checksum -ArchivePath $archivePath -ChecksumPath $checksumPath + + $extractDir = Join-Path $workDir "extract" + New-Item -ItemType Directory -Force -Path $extractDir | Out-Null + Expand-Archive -Path $archivePath -DestinationPath $extractDir -Force + Install-FromDirectory -SourceDir $extractDir +} + +function Print-Summary { + $omnigraphPath = Join-Path $InstallDir "omnigraph.exe" + $serverPath = Join-Path $InstallDir "omnigraph-server.exe" + + Write-Host "" + Write-Host "Installed:" + Write-Host " $omnigraphPath" + Write-Host " $serverPath" + Write-Host "" + Write-Host "Verify:" + Write-Host " $omnigraphPath version" + Write-Host " $serverPath --help" + Write-Host "" + + if ($selectedChannel -ne "") { + Write-Host "Installed from release channel: $selectedChannel" + } + + $pathParts = $env:Path -split [System.IO.Path]::PathSeparator + if ($pathParts -notcontains $InstallDir) { + Write-Host "Add $InstallDir to PATH if needed." + } +} + +try { + Install-FromRelease + Print-Summary +} finally { + if (Test-Path $workDir) { + Remove-Item -Path $workDir -Recurse -Force + } +} diff --git a/scripts/local-rustfs-bootstrap.sh b/scripts/local-rustfs-bootstrap.sh index 6327f77..29427de 100755 --- a/scripts/local-rustfs-bootstrap.sh +++ b/scripts/local-rustfs-bootstrap.sh @@ -74,9 +74,6 @@ platform_asset_name() { Linux/x86_64) printf 'omnigraph-linux-x86_64.tar.gz\n' ;; - Darwin/x86_64) - printf 'omnigraph-macos-x86_64.tar.gz\n' - ;; Darwin/arm64) printf 'omnigraph-macos-arm64.tar.gz\n' ;; From 8eba37cc60c31b34d0f2e314ac34eff4363dcdc1 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 30 May 2026 14:29:49 +0200 Subject: [PATCH 2/4] README: link to TypeScript SDK and MCP server (#92) Adds a `Clients` section pointing at the TypeScript SDK and the Model Context Protocol server for programmatic / LLM access to omnigraph-server. Both are published on npm and developed in ModernRelay/omnigraph-ts; this README is the canonical discovery surface so it should mention them. Notes the major.minor lockstep convention so users know which client version targets which server. --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 946297c..0f6ebea 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,24 @@ omnigraph branch merge feature-x --into main ./graph.omni See [docs/user/cli.md](docs/user/cli.md) for schema apply, snapshots, ingest, commits, and policy commands. +## Clients + +For programmatic access to a running `omnigraph-server`: + +- **TypeScript SDK** — [`@modernrelay/omnigraph`](https://www.npmjs.com/package/@modernrelay/omnigraph) ([source](https://github.com/ModernRelay/omnigraph-ts/tree/main/packages/sdk)). Instance-per-client, typed errors, camelCase types, async-iterator streaming export. + + ```bash + npm install @modernrelay/omnigraph + ``` + +- **Model Context Protocol server** — [`@modernrelay/omnigraph-mcp`](https://www.npmjs.com/package/@modernrelay/omnigraph-mcp) ([source](https://github.com/ModernRelay/omnigraph-ts/tree/main/packages/mcp)). Bridges Omnigraph to LLM hosts (Claude Desktop, Claude Code, …) over stdio. Exposes tools and resources for schema, branches, queries, mutations, ingest, and bundles curated best-practices guidance from the cookbook. + + ```bash + npm install -g @modernrelay/omnigraph-mcp + ``` + +Both packages are versioned in lockstep with `omnigraph-server` on major.minor: `@modernrelay/omnigraph@X.Y.*` targets `omnigraph-server@X.Y.*`. See [`ModernRelay/omnigraph-ts`](https://github.com/ModernRelay/omnigraph-ts) for the monorepo. + ## Docs - [Install guide](docs/user/install.md) From 854ad0afcb444ccbaa57e0f464a26f3073886f8d Mon Sep 17 00:00:00 2001 From: Andrew Altshuler Date: Sat, 30 May 2026 20:17:55 +0100 Subject: [PATCH 3/4] feat(server): compose OMNIGRAPH_TARGET_URI with OMNIGRAPH_CONFIG in entrypoint (#129) The container entrypoint's URI and config branches were mutually exclusive, so a deployment driven by OMNIGRAPH_TARGET_URI could never load a policy file. Forward --config alongside the positional URI when OMNIGRAPH_CONFIG is also set (the URI still wins via resolve_target_uri), enabling Cedar policy without changing how the URI is provided. Add docker/entrypoint_test.sh (arg-composition cases) + a CI job, and document the env-var contract in docs/user/deployment.md. Co-authored-by: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 12 ++++++++ docker/entrypoint.sh | 10 +++++- docker/entrypoint_test.sh | 65 +++++++++++++++++++++++++++++++++++++++ docs/user/deployment.md | 29 +++++++++++++++++ 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100755 docker/entrypoint_test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 918a472..5b7b7b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,18 @@ jobs: - name: Verify AGENTS.md ↔ docs/ cross-links run: bash scripts/check-agents-md.sh + entrypoint_test: + name: Container Entrypoint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout source + uses: actions/checkout@v5.0.1 + + - name: Verify omnigraph-server entrypoint arg composition + run: sh docker/entrypoint_test.sh + test: name: Test Workspace needs: classify_changes diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 83b7d34..a5fb275 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,8 +9,14 @@ fi bind="${OMNIGRAPH_BIND:-0.0.0.0:8080}" +# URI comes from the env var (the positional arg wins over any config +# `graphs` block in resolve_target_uri). OMNIGRAPH_CONFIG, when also set, +# is forwarded as --config purely to supply a policy file — the two +# compose. Without OMNIGRAPH_CONFIG the behavior is unchanged. if [ -n "${OMNIGRAPH_TARGET_URI:-}" ]; then - exec "$SERVER_BIN" "${OMNIGRAPH_TARGET_URI}" --bind "${bind}" + exec "$SERVER_BIN" "${OMNIGRAPH_TARGET_URI}" \ + ${OMNIGRAPH_CONFIG:+--config "$OMNIGRAPH_CONFIG"} \ + --bind "${bind}" fi if [ -n "${OMNIGRAPH_CONFIG:-}" ]; then @@ -28,5 +34,7 @@ omnigraph-server container startup requires one of: Optional: - OMNIGRAPH_BIND (default: 0.0.0.0:8080) - OMNIGRAPH_TARGET (used with OMNIGRAPH_CONFIG) + - OMNIGRAPH_CONFIG (may also accompany OMNIGRAPH_TARGET_URI to add a + policy file; the URI still comes from OMNIGRAPH_TARGET_URI) EOF exit 64 diff --git a/docker/entrypoint_test.sh b/docker/entrypoint_test.sh new file mode 100755 index 0000000..01fbee2 --- /dev/null +++ b/docker/entrypoint_test.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Self-contained test for docker/entrypoint.sh argument composition. +# Runs the entrypoint against a stub server that echoes its args, and +# asserts the forwarded argv for each startup mode. No Docker required. +# +# sh docker/entrypoint_test.sh +# +# Exits 0 on success, 1 on the first mismatch. +set -eu + +here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +entrypoint="$here/entrypoint.sh" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/bin" +cat > "$work/bin/omnigraph-server" <<'EOF' +#!/bin/sh +echo "ARGS: $*" +EOF +chmod +x "$work/bin/omnigraph-server" + +# Run the real entrypoint with SERVER_BIN pointed at the stub. +ep="$work/entrypoint.sh" +sed "s#SERVER_BIN=\"/usr/local/bin/omnigraph-server\"#SERVER_BIN=\"$work/bin/omnigraph-server\"#" \ + "$entrypoint" > "$ep" + +fail=0 +check() { + desc=$1; want=$2; got=$3 + if [ "$got" != "$want" ]; then + echo "FAIL: $desc" + echo " want: $want" + echo " got: $got" + fail=1 + else + echo "ok: $desc" + fi +} + +got=$(OMNIGRAPH_TARGET_URI="s3://b/g" OMNIGRAPH_BIND="0.0.0.0:8080" sh "$ep") +check "TARGET_URI only (legacy)" \ + "ARGS: s3://b/g --bind 0.0.0.0:8080" "$got" + +got=$(OMNIGRAPH_TARGET_URI="s3://b/g" OMNIGRAPH_CONFIG="/etc/omnigraph/omnigraph.yaml" OMNIGRAPH_BIND="0.0.0.0:8080" sh "$ep") +check "TARGET_URI + CONFIG composes (policy)" \ + "ARGS: s3://b/g --config /etc/omnigraph/omnigraph.yaml --bind 0.0.0.0:8080" "$got" + +got=$(OMNIGRAPH_CONFIG="/etc/omnigraph/omnigraph.yaml" OMNIGRAPH_BIND="0.0.0.0:8080" sh "$ep") +check "CONFIG only" \ + "ARGS: --config /etc/omnigraph/omnigraph.yaml --bind 0.0.0.0:8080" "$got" + +got=$(OMNIGRAPH_CONFIG="/etc/omnigraph/omnigraph.yaml" OMNIGRAPH_TARGET="active" OMNIGRAPH_BIND="0.0.0.0:8080" sh "$ep") +check "CONFIG + TARGET" \ + "ARGS: --config /etc/omnigraph/omnigraph.yaml --target active --bind 0.0.0.0:8080" "$got" + +got=$(sh "$ep" some-uri --bind 1.2.3.4:9 --extra) +check "explicit args passthrough" \ + "ARGS: some-uri --bind 1.2.3.4:9 --extra" "$got" + +if [ "$fail" -ne 0 ]; then + echo "entrypoint_test: FAILED" + exit 1 +fi +echo "entrypoint_test: all cases passed" diff --git a/docs/user/deployment.md b/docs/user/deployment.md index 8613dec..9a4466c 100644 --- a/docs/user/deployment.md +++ b/docs/user/deployment.md @@ -109,6 +109,35 @@ docker run --rm -p 8080:8080 \ --bind 0.0.0.0:8080 ``` +### Container entrypoint env vars + +When no positional args are given, the image entrypoint +(`docker/entrypoint.sh`) builds the server command from env vars: + +| Var | Effect | +|---|---| +| `OMNIGRAPH_TARGET_URI` | Graph URI, passed as the positional argument. | +| `OMNIGRAPH_CONFIG` | Path to an `omnigraph.yaml`, passed as `--config`. Used to supply a `policy.file` (Cedar authorization). The config file and any relative `policy.file` must be mounted into the container. | +| `OMNIGRAPH_TARGET` | Graph name to select from the config's `graphs:` block (with `OMNIGRAPH_CONFIG`, when no `OMNIGRAPH_TARGET_URI`). | +| `OMNIGRAPH_BIND` | Listen address (default `0.0.0.0:8080`). | + +`OMNIGRAPH_TARGET_URI` and `OMNIGRAPH_CONFIG` **compose**: set both to keep the +graph URI in the env var while loading policy from the config file (the +positional URI wins over any `graphs:` entry). To enable Cedar policy on a +container otherwise driven by `OMNIGRAPH_TARGET_URI`, mount the config dir and +add `OMNIGRAPH_CONFIG`: + +```bash +docker run --rm -p 8080:8080 \ + -e OMNIGRAPH_SERVER_BEARER_TOKEN="change-me" \ + -e OMNIGRAPH_TARGET_URI="s3://my-bucket/graphs/example/releases/2026-04-10-v0.1.0" \ + -e OMNIGRAPH_CONFIG="/etc/omnigraph/omnigraph.yaml" \ + -v "$PWD/config:/etc/omnigraph:ro" \ + omnigraph-server:local +# /etc/omnigraph/omnigraph.yaml contains `policy: { file: ./policy.yaml }`; +# policy.yaml (+ optional policy.tests.yaml) sit beside it in the mount. +``` + ## Auth The server can run unauthenticated for local development only when explicitly From 2d5c4b12021ff0bf4c7036ebba034674a6ba7bf3 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 30 May 2026 23:20:56 +0200 Subject: [PATCH 4/4] =?UTF-8?q?docs:=20rename=20runs.md/runs.rs=20?= =?UTF-8?q?=E2=86=92=20writes=20and=20repoint=20all=20references=20(#131)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Run state machine was removed in MR-771 (v0.4.0); `docs/dev/runs.md` and `crates/omnigraph/tests/runs.rs` have since documented and tested the direct-publish write path, so the "runs" name was misleading. - git mv docs/dev/runs.md → docs/dev/writes.md (reframe H1 + intro; keep MR-771 history note) - git mv crates/omnigraph/tests/runs.rs → tests/writes.rs (reframe header) - repoint every runs.md / runs.rs reference across docs, AGENTS.md, and source comments - fix four pre-existing broken `docs/runs.md` links (the file never lived at that path) to `docs/dev/writes.md` - fix the stale v0.4.0 anchor to the live section No behavior change: every source edit is a comment. Engine builds and the renamed test passes 25/25; scripts/check-agents-md.sh passes. The run-removal cleanup itself (run_registry.rs guard, __run__ prefix) is deferred to MR-770. --- AGENTS.md | 4 ++-- crates/omnigraph-cli/tests/system_local.rs | 2 +- crates/omnigraph/src/db/manifest/recovery.rs | 2 +- crates/omnigraph/src/loader/mod.rs | 2 +- crates/omnigraph/src/table_store.rs | 4 ++-- crates/omnigraph/tests/staged_writes.rs | 4 ++-- crates/omnigraph/tests/{runs.rs => writes.rs} | 8 ++++---- docs/dev/architecture.md | 4 ++-- docs/dev/execution.md | 2 +- docs/dev/index.md | 2 +- docs/dev/invariants.md | 12 +++++------ docs/dev/testing.md | 4 ++-- docs/dev/{runs.md => writes.md} | 9 ++++++--- docs/releases/v0.4.0.md | 2 +- docs/releases/v0.4.1.md | 20 +++++++++---------- docs/user/errors.md | 2 +- docs/user/query-language.md | 2 +- docs/user/transactions.md | 2 +- 18 files changed, 45 insertions(+), 42 deletions(-) rename crates/omnigraph/tests/{runs.rs => writes.rs} (99%) rename docs/dev/{runs.md => writes.md} (98%) diff --git a/AGENTS.md b/AGENTS.md index 3fc78f7..68de6b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,7 @@ Full diagram and concurrency model: [docs/dev/architecture.md](docs/dev/architec | Embeddings (compiler + engine clients, env vars, `@embed`) | [docs/user/embeddings.md](docs/user/embeddings.md) | | Branches, commit graph, snapshots, system branches | [docs/user/branches-commits.md](docs/user/branches-commits.md) | | Transactions and atomicity (per-query atomic; branches as multi-query transactions) | [docs/user/transactions.md](docs/user/transactions.md) | -| Direct-publish writes (the former Run state machine, now demoted to publisher CAS) | [docs/dev/runs.md](docs/dev/runs.md) | +| Direct-publish write path (staging, D2, recovery sidecars; the former Run state machine) | [docs/dev/writes.md](docs/dev/writes.md) | | Three-way merge and conflict kinds | [docs/dev/merge.md](docs/dev/merge.md) | | Diff / change feed (`diff_between`, `diff_commits`) | [docs/user/changes.md](docs/user/changes.md) | | Query execution, mutation execution, bulk loader, `load` vs `ingest` | [docs/dev/execution.md](docs/dev/execution.md) | @@ -176,7 +176,7 @@ cargo run -p omnigraph-server -- --bind 0.0.0.0:8080 # run the server fr # Run one crate / one test file / one test fn cargo test -p omnigraph-engine --test traversal # one integration-test file (see docs/dev/testing.md) -cargo test -p omnigraph-engine --test runs concurrent # one test fn by name substring +cargo test -p omnigraph-engine --test writes concurrent # one test fn by name substring cargo test -p omnigraph-engine some_inline_test -- --nocapture # show stdout # Feature-gated suites (each is its own job in CI, not part of the default run) diff --git a/crates/omnigraph-cli/tests/system_local.rs b/crates/omnigraph-cli/tests/system_local.rs index 074b203..08f653d 100644 --- a/crates/omnigraph-cli/tests/system_local.rs +++ b/crates/omnigraph-cli/tests/system_local.rs @@ -991,7 +991,7 @@ query vector_search($q: String) { // The publisher CAS conflict shape is verified end-to-end at the engine // level in -// `crates/omnigraph/tests/runs.rs::concurrent_writers_one_succeeds_one_gets_expected_version_mismatch` +// `crates/omnigraph/tests/writes.rs::concurrent_writers_one_succeeds_one_gets_expected_version_mismatch` // and at the HTTP boundary in // `crates/omnigraph-server/tests/server.rs::change_conflict_returns_manifest_conflict_409`. // A CLI-level race would be timing-dependent; with direct-publish the diff --git a/crates/omnigraph/src/db/manifest/recovery.rs b/crates/omnigraph/src/db/manifest/recovery.rs index 425499a..4c1b987 100644 --- a/crates/omnigraph/src/db/manifest/recovery.rs +++ b/crates/omnigraph/src/db/manifest/recovery.rs @@ -2,7 +2,7 @@ //! //! This module implements the building blocks of the per-sidecar recovery //! sweep that closes the documented Phase B → Phase C residual (see -//! `docs/dev/runs.md` "Open-time recovery sweep"). The high-level shape: +//! `docs/dev/writes.md` "Open-time recovery sweep"). The high-level shape: //! //! 1. Each writer that performs a multi-table commit writes a small JSON //! sidecar at `__recovery/{ulid}.json` BEFORE its per-table diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index cade1f4..46a46e2 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -613,7 +613,7 @@ async fn load_jsonl_reader( } else { // LoadMode::Overwrite keeps the legacy inline-commit path — // truncate-then-append doesn't fit the staged shape (see - // `docs/runs.md` "LoadMode::Overwrite residual"). The recovery + // `docs/dev/writes.md` "LoadMode::Overwrite residual"). The recovery // sidecar is not applicable here because the writer doesn't go // through MutationStaging; per-table inline commits + a final // manifest publish handle their own residual via the documented diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index ddab706..46b15b0 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -49,7 +49,7 @@ pub struct DeleteState { /// `exec/mutation.rs`) and the bulk loader (`loader/mod.rs`). The /// intent: defer Lance commits to end-of-query so a mid-query failure /// leaves the touched table at the pre-mutation HEAD instead of -/// drifting ahead. See `docs/runs.md` for the publisher-CAS contract +/// drifting ahead. See `docs/dev/writes.md` for the publisher-CAS contract /// this builds on. /// /// `transaction` is opaque from our side — Lance owns its semantics. We @@ -901,7 +901,7 @@ impl TableStore { /// Lift path: either a Lance API extension that lets /// `MergeInsertBuilder` accept additional staged fragments, or an /// in-memory pre-merge here that folds prior staged batches into the - /// input stream. See `docs/runs.md`. + /// input stream. See `docs/dev/writes.md`. pub async fn stage_merge_insert( &self, ds: Dataset, diff --git a/crates/omnigraph/tests/staged_writes.rs b/crates/omnigraph/tests/staged_writes.rs index 021b36e..5335057 100644 --- a/crates/omnigraph/tests/staged_writes.rs +++ b/crates/omnigraph/tests/staged_writes.rs @@ -2,7 +2,7 @@ //! exercise `stage_append`, `stage_merge_insert`, `scan_with_staged`, //! and `count_rows_with_staged` directly against a Lance dataset — no //! Omnigraph engine involved. The engine-level use of these primitives -//! is exercised by `tests/runs.rs`. +//! is exercised by `tests/writes.rs`. //! //! Test surface here: //! 1. `stage_append` + `scan_with_staged` shows committed + staged data @@ -709,7 +709,7 @@ async fn stage_create_inverted_index_does_not_advance_head_until_commit() { /// /// **When Lance #6658 lands**: this test will need to flip — replace /// the assertion with a `stage_delete` + `commit_staged` round-trip -/// and remove the residual line in `docs/runs.md`. +/// and remove the residual line in `docs/dev/writes.md`. #[tokio::test] async fn delete_where_advances_head_inline_documents_residual() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph/tests/runs.rs b/crates/omnigraph/tests/writes.rs similarity index 99% rename from crates/omnigraph/tests/runs.rs rename to crates/omnigraph/tests/writes.rs index cfff3fc..13cb10f 100644 --- a/crates/omnigraph/tests/runs.rs +++ b/crates/omnigraph/tests/writes.rs @@ -1,7 +1,7 @@ -//! Tests for the direct-to-target write path (Run state machine -//! removed). The Run/`__run__` staging branch / RunRecord state machine no -//! longer exists; mutations and loads write directly to target tables and -//! commit once via the publisher's `expected_table_versions` CAS. +//! Tests for the direct-publish write path: mutations and loads write +//! directly to target tables and commit once via the publisher's +//! `expected_table_versions` CAS. (History: this replaced the removed Run +//! state machine / `__run__` staging branches / RunRecord — MR-771.) //! //! What this file covers: //! - No `__run__*` branches are created by load or mutate. diff --git a/docs/dev/architecture.md b/docs/dev/architecture.md index 8b7fca2..813f30c 100644 --- a/docs/dev/architecture.md +++ b/docs/dev/architecture.md @@ -207,7 +207,7 @@ contracts: This pattern realizes read-your-writes within a multi-statement mutation and keeps failure scope bounded for inserts/updates by construction at the writer layer. See [docs/dev/invariants.md](invariants.md) and -[docs/dev/runs.md](runs.md) for the publisher CAS contract this builds on. +[docs/dev/writes.md](writes.md) for the publisher CAS contract this builds on. ### Storage trait — today vs. roadmap @@ -278,7 +278,7 @@ flowchart LR eng --> wq ``` -The server applies Cedar policy at the HTTP boundary today. The roadmap, called out in [docs/dev/invariants.md](invariants.md) as a known gap, is to push policy into the planner as predicates. After Cedar, mutating handlers go through `WorkloadController` (per-actor admission cap + byte budget; PR 2 / MR-686) before reaching the engine. The engine itself holds an `Arc` so concurrent mutations on the same `(table, branch)` serialize at the queue, while disjoint keys run in parallel — see [docs/user/server.md](../user/server.md) "Per-actor admission control" and [docs/dev/runs.md](runs.md). The CLI bypasses the HTTP layer (and admission) and calls the engine API directly. +The server applies Cedar policy at the HTTP boundary today. The roadmap, called out in [docs/dev/invariants.md](invariants.md) as a known gap, is to push policy into the planner as predicates. After Cedar, mutating handlers go through `WorkloadController` (per-actor admission cap + byte budget; PR 2 / MR-686) before reaching the engine. The engine itself holds an `Arc` so concurrent mutations on the same `(table, branch)` serialize at the queue, while disjoint keys run in parallel — see [docs/user/server.md](../user/server.md) "Per-actor admission control" and [docs/dev/writes.md](writes.md). The CLI bypasses the HTTP layer (and admission) and calls the engine API directly. Code paths: diff --git a/docs/dev/execution.md b/docs/dev/execution.md index f5c2840..3a108d7 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -147,7 +147,7 @@ sequenceDiagram - End-of-query Lance commit: `TableStore::stage_append`, `stage_merge_insert`, `commit_staged` at `crates/omnigraph/src/table_store.rs` - Manifest commit primitive: `commit_updates_on_branch_with_expected` at `crates/omnigraph/src/db/omnigraph/table_ops.rs` -Atomicity guarantee for multi-statement mutations: a mid-query failure leaves Lance HEAD untouched on staged tables (no inline commit happened during op execution), so the next mutation proceeds normally with no `ExpectedVersionMismatch`. The publisher CAS at the very end either succeeds (manifest advances atomically across all touched sub-tables) or fails with a typed `ManifestConflictDetails::ExpectedVersionMismatch` (no partial publish). See [docs/dev/invariants.md](invariants.md) and [docs/dev/runs.md](runs.md). +Atomicity guarantee for multi-statement mutations: a mid-query failure leaves Lance HEAD untouched on staged tables (no inline commit happened during op execution), so the next mutation proceeds normally with no `ExpectedVersionMismatch`. The publisher CAS at the very end either succeeds (manifest advances atomically across all touched sub-tables) or fails with a typed `ManifestConflictDetails::ExpectedVersionMismatch` (no partial publish). See [docs/dev/invariants.md](invariants.md) and [docs/dev/writes.md](writes.md). ## Bulk loader (`loader/mod.rs`) diff --git a/docs/dev/index.md b/docs/dev/index.md index 83df8c8..d9ba5e5 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -21,7 +21,7 @@ constraints. User-facing behavior should still be documented through |---|---| | System structure, L1/L2 framing, component diagrams | [architecture.md](architecture.md) | | On-disk layout, manifest schema, URI behavior | [storage.md](../user/storage.md) | -| Direct-publish writes, D2, staged writes, recovery sidecars | [runs.md](runs.md) | +| Direct-publish writes, D2, staged writes, recovery sidecars | [writes.md](writes.md) | | Query execution, mutation execution, loader flow | [execution.md](execution.md) | | Index lifecycle and graph topology indexes | [indexes.md](../user/indexes.md) | | Branch and commit internals | [branches-commits.md](../user/branches-commits.md) | diff --git a/docs/dev/invariants.md b/docs/dev/invariants.md index 958042f..70477d4 100644 --- a/docs/dev/invariants.md +++ b/docs/dev/invariants.md @@ -38,7 +38,7 @@ Use it this way: publishes one manifest update. Do not commit per statement. Delete-only queries are the documented inline residual; the parse-time D2 rule prevents mixing deletes with insert/update until Lance exposes two-phase delete. - Read [runs.md](runs.md) and [execution.md](execution.md). + Read [writes.md](writes.md) and [execution.md](execution.md). 5. **Recovery is part of the commit protocol.** Writers that can advance Lance HEAD before manifest publish must write `__recovery/{ulid}.json` sidecars. @@ -56,7 +56,7 @@ Use it this way: branch they read even when index coverage is partial. Expensive index work should converge from manifest state instead of extending the critical write path. Scalar staged index builds and vector inline residuals are documented - in [runs.md](runs.md) and [indexes.md](../user/indexes.md). + in [writes.md](writes.md) and [indexes.md](../user/indexes.md). 8. **Schema identity survives renames.** Accepted schema identity must remain stable across type and property renames. Rename support belongs in migration @@ -96,12 +96,12 @@ Use it this way: | Area | Current state | Source | |---|---|---| -| Multi-table commit | Manifest CAS plus recovery sidecars; not a single Lance primitive | [runs.md](runs.md), [architecture.md](architecture.md) | -| Constructive mutations | In-memory `MutationStaging`, one end-of-query table commit per touched table, then one manifest publish | [runs.md](runs.md), [execution.md](execution.md) | -| Deletes | Inline-commit residual; delete-only queries allowed, mixed insert/update/delete rejected by D2 | [query-language.md](../user/query-language.md), [runs.md](runs.md) | +| Multi-table commit | Manifest CAS plus recovery sidecars; not a single Lance primitive | [writes.md](writes.md), [architecture.md](architecture.md) | +| Constructive mutations | In-memory `MutationStaging`, one end-of-query table commit per touched table, then one manifest publish | [writes.md](writes.md), [execution.md](execution.md) | +| Deletes | Inline-commit residual; delete-only queries allowed, mixed insert/update/delete rejected by D2 | [query-language.md](../user/query-language.md), [writes.md](writes.md) | | Schema validation | Type checks, required fields, defaults, edge endpoint checks, and edge cardinality are enforced on write paths | [schema-language.md](../user/schema-language.md), [execution.md](execution.md) | | Unique constraints | Intra-batch and write-path checks exist; full cross-version uniqueness is still a gap | [schema-language.md](../user/schema-language.md) | -| Storage trait | `TableStorage` exists as the sealed staged-write surface; full call-site migration and capability/stat surfaces are incomplete | [runs.md](runs.md), [architecture.md](architecture.md) | +| Storage trait | `TableStorage` exists as the sealed staged-write surface; full call-site migration and capability/stat surfaces are incomplete | [writes.md](writes.md), [architecture.md](architecture.md) | | Index lifecycle | `ensure_indices` is explicit today; reconciler-based convergence is roadmap | [indexes.md](../user/indexes.md), [maintenance.md](../user/maintenance.md) | | Traversal IDs | Runtime still builds `TypeIndex`; Lance stable row-id based graph IDs are roadmap | [architecture.md](architecture.md), [query-language.md](../user/query-language.md) | | Auth | Bearer token hashing and server-side actor resolution are implemented at the HTTP boundary | [server.md](../user/server.md), [policy.md](../user/policy.md) | diff --git a/docs/dev/testing.md b/docs/dev/testing.md index e6989ba..425fcee 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -20,7 +20,7 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `end_to_end.rs` | Full init → load → query/mutate flow | | `branching.rs` | Branch create / list / delete, lazy fork | | `merge_truth_table.rs` | Merge-pair truth table (MR-786): all 9×9 `(left_op, right_op)` cells from `{noop, addNode, removeNode, addEdge, removeEdge, setProperty, dropProperty, addLabel, removeLabel}`. Adding a new op to `OpVariant` forces a compile error in `build_case` until the new row + column are dispositioned. 36 executable cells run through real `branch_merge` with a structured oracle (`MergeOutcome` / `MergeConflictKind` + graph-state assert); 45 cells involving `dropProperty`/`addLabel`/`removeLabel` are recorded as `Unsupported` until the mutation grammar grows. | -| `runs.rs` | Direct-publish writes: cancellation, concurrent-writer CAS, multi-statement atomicity, MR-794 staged-write rewire (D₂ rejection, insert+update coalesce, multi-append coalesce, partial-failure recovery, load RI/cardinality recovery) | +| `writes.rs` | Direct-publish writes: cancellation, concurrent-writer CAS, multi-statement atomicity, MR-794 staged-write rewire (D₂ rejection, insert+update coalesce, multi-append coalesce, partial-failure recovery, load RI/cardinality recovery) | | `staged_writes.rs` | TableStore staged-write primitives (`stage_append`, `stage_merge_insert`, `commit_staged`, `scan_with_staged`, `count_rows_with_staged`) — primitive-level only; engine code uses the in-memory `MutationStaging` accumulator instead | | `lifecycle.rs` | Graph lifecycle, schema state | | `point_in_time.rs` | Snapshots, time travel (`snapshot_at_version`, `entity_at`) | @@ -89,7 +89,7 @@ If introducing coverage tooling is in scope for your task, the natural first ste How to check: -1. **Map the change to an area** — use the engine integration-test table above (`branching.rs`, `runs.rs`, `search.rs`, etc.). The filename usually names the area. +1. **Map the change to an area** — use the engine integration-test table above (`branching.rs`, `writes.rs`, `search.rs`, etc.). The filename usually names the area. 2. **Open the file and skim every test fn name.** Test fn names are the index — read them all, not just the first few. 3. **Grep for the symbol or path you're changing.** `rg ` or `rg ` across all `tests/` directories surfaces existing coverage you might miss. 4. **Decide one of three outcomes**, in this order of preference: diff --git a/docs/dev/runs.md b/docs/dev/writes.md similarity index 98% rename from docs/dev/runs.md rename to docs/dev/writes.md index 816f2ac..974f7a6 100644 --- a/docs/dev/runs.md +++ b/docs/dev/writes.md @@ -1,7 +1,10 @@ -# Runs — REMOVED (MR-771) +# Direct-Publish Write Path -The Run state machine and `__run__` staging branches were removed in -MR-771. `mutate_as` and `load` now write **directly to the target table** +> History: the Run state machine and `__run__` staging branches were +> removed in MR-771 (shipped v0.4.0). Writes now go directly to the target +> table; this document specifies that direct-publish path. + +`mutate_as` and `load` write **directly to the target table** and call `ManifestBatchPublisher::publish` once at the end with `expected_table_versions` (the per-table manifest versions captured before the first write). Cross-table OCC is enforced inside the publisher; the diff --git a/docs/releases/v0.4.0.md b/docs/releases/v0.4.0.md index efb2da7..d3a8244 100644 --- a/docs/releases/v0.4.0.md +++ b/docs/releases/v0.4.0.md @@ -65,7 +65,7 @@ manifest. The next mutation against that table fails with `ExpectedVersionMismatch`. Most validation runs before any Lance write, so single-statement mutations are unaffected; the narrow path is multi-statement queries with late-op failures. Tracked as a follow-up; -see [docs/dev/runs.md](../dev/runs.md#known-limitation-mid-query-partial-failure-on-the-same-table) +see [docs/dev/writes.md](../dev/writes.md#mid-query-partial-failure-closed-by-mr-794) for the workaround. ## Upgrade notes diff --git a/docs/releases/v0.4.1.md b/docs/releases/v0.4.1.md index 78211e4..4983015 100644 --- a/docs/releases/v0.4.1.md +++ b/docs/releases/v0.4.1.md @@ -19,7 +19,7 @@ mutation proceeds normally. HEAD on every staged table is untouched and the next mutation proceeds normally. A narrowed residual remains at the finalize→publisher boundary (multi-table `commit_staged` is not - atomic with the manifest commit) — see [docs/dev/runs.md](../dev/runs.md) + atomic with the manifest commit) — see [docs/dev/writes.md](../dev/writes.md) "Finalize → publisher residual" for details. - **D₂ parse-time rule**: a single mutation query is either insert/update-only or delete-only. Mixed → rejected with a clear @@ -75,14 +75,14 @@ mutation proceeds normally. ## Tests added -- `tests/runs.rs::partial_failure_leaves_target_queryable_and_unblocks_next_mutation` +- `tests/writes.rs::partial_failure_leaves_target_queryable_and_unblocks_next_mutation` (replaces the old `partial_failure_observably_rolls_back_but_blocks_next_mutation_on_same_table`) -- `tests/runs.rs::mutation_rejects_mixed_insert_and_delete_at_parse_time` -- `tests/runs.rs::mixed_insert_and_update_on_same_person_coalesces_to_one_merge` -- `tests/runs.rs::multiple_appends_to_same_edge_coalesce_to_one_append` -- `tests/runs.rs::multi_statement_inserts_publish_exactly_once` -- `tests/runs.rs::load_with_bad_edge_reference_unblocks_next_load` -- `tests/runs.rs::load_with_cardinality_violation_unblocks_next_load` +- `tests/writes.rs::mutation_rejects_mixed_insert_and_delete_at_parse_time` +- `tests/writes.rs::mixed_insert_and_update_on_same_person_coalesces_to_one_merge` +- `tests/writes.rs::multiple_appends_to_same_edge_coalesce_to_one_append` +- `tests/writes.rs::multi_statement_inserts_publish_exactly_once` +- `tests/writes.rs::load_with_bad_edge_reference_unblocks_next_load` +- `tests/writes.rs::load_with_cardinality_violation_unblocks_next_load` ## Files changed @@ -105,7 +105,7 @@ mutation proceeds normally. - `Cargo.toml` (workspace) + `crates/omnigraph/Cargo.toml` — added `datafusion = "52"` direct dep (transitively pulled by Lance already; required for `MemTable`). -- `docs/dev/runs.md` — removed "Known limitation" section; documented +- `docs/dev/writes.md` — removed "Known limitation" section; documented the new accumulator + D₂ + LoadMode::Overwrite residual. - `docs/dev/invariants.md` — mutation atomicity / read-your-writes status flipped to `upheld for inserts/updates`. @@ -127,7 +127,7 @@ mutation proceeds normally. as legacy. - `docs/user/cli.md` — replaced the legacy `omnigraph run *` quickstart block with `omnigraph commit list/show`. -- `docs/dev/testing.md` — extended the `runs.rs` row to cover the new +- `docs/dev/testing.md` — extended the `writes.rs` row to cover the new staged-write contract tests; added the `staged_writes.rs` row. - `AGENTS.md` (CLAUDE.md symlink) — updated the atomic-per-query description and the L2 capability matrix row. diff --git a/docs/user/errors.md b/docs/user/errors.md index fd047eb..8373b0d 100644 --- a/docs/user/errors.md +++ b/docs/user/errors.md @@ -9,7 +9,7 @@ - `Manifest(ManifestError { kind: BadRequest|NotFound|Conflict|Internal, details: Option, … })` - `ManifestConflictDetails::ExpectedVersionMismatch { table_key, expected, actual }` — caller's `expected_table_versions` did not match the manifest's current latest non-tombstoned version (set by `OmniError::manifest_expected_version_mismatch`). - `ManifestConflictDetails::RowLevelCasContention` — Lance row-level CAS rejected the publish because a concurrent writer landed the same `object_id`. Retried internally by the publisher; only surfaces if the retry budget exhausts. - - **D₂ parse-time rejection** (MR-794): a single mutation query that mixes inserts/updates with deletes errors out *before any I/O* with kind `BadRequest`. Message: `mutation '' on the same query mixes inserts/updates and deletes; split into separate mutations: (1) inserts and updates, then (2) deletes`. See [docs/user/query-language.md](query-language.md) for the rule and [docs/dev/runs.md](../dev/runs.md) for the underlying staged-write rationale. + - **D₂ parse-time rejection** (MR-794): a single mutation query that mixes inserts/updates with deletes errors out *before any I/O* with kind `BadRequest`. Message: `mutation '' on the same query mixes inserts/updates and deletes; split into separate mutations: (1) inserts and updates, then (2) deletes`. See [docs/user/query-language.md](query-language.md) for the rule and [docs/dev/writes.md](../dev/writes.md) for the underlying staged-write rationale. - `MergeConflicts(Vec)` Compiler-side `NanoError` covers parse / catalog / type / storage / plan / execution / arrow / lance / IO / manifest / unique-constraint, each with structured spans (`SourceSpan { start, end }`) for ariadne-style diagnostics. diff --git a/docs/user/query-language.md b/docs/user/query-language.md index 94528af..6c7516f 100644 --- a/docs/user/query-language.md +++ b/docs/user/query-language.md @@ -70,7 +70,7 @@ A single mutation query must be **either insert/update-only or delete-only**. Mi > `mutation '' on the same query mixes inserts/updates and deletes; split into separate mutations: (1) inserts and updates, then (2) deletes. This restriction lifts when Lance exposes a two-phase delete API (tracked: MR-793 / Lance-upstream).` -Reason: under the staged-write rewire (MR-794), inserts and updates accumulate in memory and commit at end-of-query, while deletes still inline-commit (Lance 4.0.0 has no public two-phase delete). Mixing creates ordering hazards (same-row insert→delete becomes a no-op because the staged insert isn't visible to delete; cascading deletes of just-inserted edges break referential integrity by silent design). Until Lance exposes `DeleteJob::execute_uncommitted`, the parse-time rejection keeps both paths atomic and correct. See [docs/dev/runs.md](../dev/runs.md) and [docs/dev/invariants.md](../dev/invariants.md). +Reason: under the staged-write rewire (MR-794), inserts and updates accumulate in memory and commit at end-of-query, while deletes still inline-commit (Lance 4.0.0 has no public two-phase delete). Mixing creates ordering hazards (same-row insert→delete becomes a no-op because the staged insert isn't visible to delete; cascading deletes of just-inserted edges break referential integrity by silent design). Until Lance exposes `DeleteJob::execute_uncommitted`, the parse-time rejection keeps both paths atomic and correct. See [docs/dev/writes.md](../dev/writes.md) and [docs/dev/invariants.md](../dev/invariants.md). ## IR (Intermediate Representation) diff --git a/docs/user/transactions.md b/docs/user/transactions.md index e4ed485..d6c79f4 100644 --- a/docs/user/transactions.md +++ b/docs/user/transactions.md @@ -164,5 +164,5 @@ This is the workflow MR-797 / agentic loops are designed around: **branches are - [`docs/user/branches-commits.md`](branches-commits.md) — branch and commit-graph mechanics. - [`docs/dev/merge.md`](../dev/merge.md) — three-way merge details and conflict kinds. - [`docs/user/query-language.md`](query-language.md) — `.gq` syntax for the multi-statement queries used above. -- [`docs/dev/runs.md`](../dev/runs.md) — the per-query commit pipeline that gives single-query atomicity. +- [`docs/dev/writes.md`](../dev/writes.md) — the per-query commit pipeline that gives single-query atomicity. - [`docs/dev/invariants.md`](../dev/invariants.md) — the architectural rule.