diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d4ecfa5..e937724 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,9 +8,9 @@ # CI fails if this file drifts from its source, and rejects PRs that # edit this file directly without also editing the yml. -* @ragnorc +* @ragnorc @aaltshuler -crates/** @ragnorc +crates/** @ragnorc @aaltshuler docs/** @ragnorc README.md @ragnorc AGENTS.md @ragnorc diff --git a/.github/DISCUSSION_TEMPLATE/rfc.yml b/.github/DISCUSSION_TEMPLATE/rfc.yml new file mode 100644 index 0000000..2a63525 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/rfc.yml @@ -0,0 +1,34 @@ +labels: ["rfc"] +body: + - type: markdown + attributes: + value: | + Use this to **incubate an RFC** β€” socialize a design and reach rough + consensus before writing the formal document. When it's ready, graduate + it into a pull request that adds `docs/rfcs/NNNN-title.md` + (see [docs/rfcs/README.md](../blob/main/docs/rfcs/README.md)); a + maintainer merging that PR is acceptance. + + For a plain feature request or open-ended idea, use the **Ideas** + category instead. For bugs, open an [Issue](../../issues/new/choose). + - type: textarea + id: problem + attributes: + label: Problem / motivation + description: What needs solving, and why is it worth the long-run cost? + validations: + required: true + - type: textarea + id: sketch + attributes: + label: Proposed direction (sketch) + description: A rough shape of the design. Detail comes later in the RFC document. + validations: + required: true + - type: textarea + id: invariants + attributes: + label: Invariants touched + description: Which items in docs/dev/invariants.md does this affect or risk? Any deny-list brush? + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..8e19465 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,55 @@ +name: Bug report +description: Report a reproducible problem or wrong behavior in OmniGraph. +title: "bug: " +labels: ["bug", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Issues are for **reporting problems** β€” concrete, reproducible bugs. + For ideas, feature requests, or questions, please use + [Discussions](../../discussions) instead. + For a security vulnerability, follow [SECURITY.md](../../blob/main/SECURITY.md) β€” do **not** file it here. + + A maintainer will triage this; once labelled **`accepted`** it's open for a pull request + (see [GOVERNANCE.md](../../blob/main/GOVERNANCE.md)). + - type: textarea + id: what-happened + attributes: + label: What happened + description: What went wrong, and what you expected instead. + validations: + required: true + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: Minimal steps, commands, schema/query, or a failing snippet. + placeholder: | + 1. omnigraph init ... + 2. omnigraph ... + 3. observed: ... / expected: ... + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: Output of `omnigraph --version` (or the engine/crate version) and how you installed it. + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + description: OS, architecture, and storage backend (local FS / S3 / RustFS / MinIO). + validations: + required: false + - type: textarea + id: logs + attributes: + label: Logs / output + description: Relevant error text or logs. Will be rendered as code. + render: shell + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..50720b8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,13 @@ +# Issues are for problem reports only. Disable blank issues so everything is +# routed: bugs through the form, everything else to Discussions / SECURITY.md. +blank_issues_enabled: false +contact_links: + - name: πŸ’‘ Idea, feature request, or RFC + url: https://github.com/ModernRelay/omnigraph/discussions + about: Propose features and designs in Discussions. RFCs graduate from there into a docs/rfcs/ pull request. + - name: ❓ Question or help + url: https://github.com/ModernRelay/omnigraph/discussions + about: Ask in Discussions β€” questions are not tracked as Issues. + - name: πŸ”’ Security vulnerability + url: https://github.com/ModernRelay/omnigraph/blob/main/SECURITY.md + about: Report security issues privately per SECURITY.md β€” never as a public Issue. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..2a548c7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,29 @@ + + +## What & why + + + +## Backing issue / RFC + + + +- [ ] Fixes an **accepted** issue: Closes # +- [ ] Implements / is an **accepted** RFC: +- [ ] **Trivial fast-lane** (typo / docs / dependency bump / comment / one-line CI) β€” no issue/RFC required + +## Checklist + +- [ ] Change is focused (one logical change) +- [ ] Tests added/updated for behavior changes (or N/A) +- [ ] Public docs updated if user-facing surface changed (or N/A) +- [ ] Reviewed against [docs/dev/invariants.md](../blob/main/docs/dev/invariants.md) β€” no Hard Invariant weakened, no deny-list item hit (or justified) + +## Notes for reviewers + + diff --git a/.github/branch-protection.json b/.github/branch-protection.json index 61b7d33..c039e32 100644 --- a/.github/branch-protection.json +++ b/.github/branch-protection.json @@ -1,5 +1,5 @@ { - "_comment": "Branch protection policy for main. Applied via scripts/apply-branch-protection.sh. See docs/branch-protection.md for rationale.", + "_comment": "Branch protection policy for main. Applied via scripts/apply-branch-protection.sh. See docs/branch-protection.md for rationale. NOTE: bypass_pull_request_allowances.users must mirror the engineering owners in .github/codeowners-roles.yml β€” code owners merge their own PRs without a second review; non-owners still need a code-owner approval. (render-codeowners.py does NOT generate this list; keep it in sync by hand.)", "required_status_checks": { "strict": true, "contexts": [ @@ -7,8 +7,8 @@ "Check AGENTS.md Links", "Test Workspace", "Test omnigraph-server --features aws", - "CODEOWNERS / drift", - "CODEOWNERS / noedit" + "CODEOWNERS matches source", + "CODEOWNERS not hand-edited" ] }, "enforce_admins": false, @@ -17,7 +17,12 @@ "dismiss_stale_reviews": true, "require_code_owner_reviews": true, "required_approving_review_count": 1, - "require_last_push_approval": false + "require_last_push_approval": false, + "bypass_pull_request_allowances": { + "users": ["ragnorc", "aaltshuler"], + "teams": [], + "apps": [] + } }, "restrictions": null, "required_linear_history": true, diff --git a/.github/codeowners-roles.yml b/.github/codeowners-roles.yml index c5e36a9..ce4014d 100644 --- a/.github/codeowners-roles.yml +++ b/.github/codeowners-roles.yml @@ -22,6 +22,7 @@ roles: compiler. members: - ragnorc + - aaltshuler docs: description: > diff --git a/.github/scripts/render-codeowners.py b/.github/scripts/render-codeowners.py index f243d0c..5e96545 100755 --- a/.github/scripts/render-codeowners.py +++ b/.github/scripts/render-codeowners.py @@ -1,10 +1,14 @@ #!/usr/bin/env python3 -"""Render .github/CODEOWNERS from .github/codeowners-roles.yml. +"""Render .github/CODEOWNERS and the ownership tables in +docs/dev/codeowners.md from .github/codeowners-roles.yml. -The yml is the source of truth β€” editing CODEOWNERS directly is -rejected by CI (see .github/workflows/codeowners.yml). This script -expands the role-based yml into the flat pathβ†’owners format GitHub -expects. +The yml is the source of truth. This script expands the role-based yml +into (1) the flat pathβ†’owners format GitHub expects in +`.github/CODEOWNERS`, and (2) the "who owns what" markdown tables spliced +between the generated-region markers in `docs/dev/codeowners.md`. Both are +derived artifacts; CI re-renders them on every PR (see +.github/workflows/codeowners.yml) and auto-commits the result on same-repo +PRs, so the source of truth and the human-readable view never drift. Usage: python3 .github/scripts/render-codeowners.py @@ -16,6 +20,7 @@ Exits non-zero on: one owner; otherwise CODEOWNERS would assign nobody and GitHub would silently fall back to "no required reviewer", which defeats the purpose). + - Missing generated-region markers in docs/dev/codeowners.md. """ from __future__ import annotations @@ -34,6 +39,13 @@ except ImportError: REPO_ROOT = Path(__file__).resolve().parents[2] SOURCE = REPO_ROOT / ".github" / "codeowners-roles.yml" OUTPUT = REPO_ROOT / ".github" / "CODEOWNERS" +DOCS = REPO_ROOT / "docs" / "dev" / "codeowners.md" + +# The "who owns what" tables in docs/dev/codeowners.md are spliced between +# these markers so the human-readable view never drifts from the source of +# truth. Edit codeowners-roles.yml and re-render β€” never the table by hand. +DOCS_BEGIN = "" +DOCS_END = "" BANNER = """\ # AUTOGENERATED from .github/codeowners-roles.yml. Do not edit by hand. @@ -75,6 +87,62 @@ def owners_for(role_names: list[str], roles: dict) -> list[str]: return seen +def _oneline(text: str) -> str: + """Collapse a folded/multi-line YAML description into one cell of text.""" + return " ".join((text or "").split()) + + +def ownership_tables(spec: dict, roles: dict) -> str: + """Render the human-readable "who owns what" markdown β€” a pathβ†’owners + table (the operative view at PR time, in last-match-wins order with the + catch-all first) plus a roleβ†’members table. Spliced into the docs between + the markers so it is always current with the source of truth.""" + out: list[str] = [] + + out.append("**Path β†’ owners** (GitHub applies *last match wins*; the `*` " + "catch-all is listed first and is overridden by the specific " + "patterns below it):") + out.append("") + out.append("| Path | Owners | Role(s) |") + out.append("|---|---|---|") + if "default" in spec: + owners = " ".join(owners_for(spec["default"], roles)) + out.append(f"| `*` | {owners} | {', '.join(spec['default'])} |") + for pattern, role_names in (spec.get("paths") or {}).items(): + owners = " ".join(owners_for(role_names, roles)) + out.append(f"| `{pattern}` | {owners} | {', '.join(role_names)} |") + out.append("") + + out.append("**Roles**:") + out.append("") + out.append("| Role | Members | Description |") + out.append("|---|---|---|") + for name, role in roles.items(): + members = " ".join(f"@{m}" for m in (role.get("members") or [])) + out.append(f"| `{name}` | {members} | {_oneline(role.get('description', ''))} |") + out.append("") + + return "\n".join(out) + + +def splice_docs(table_md: str) -> None: + """Replace the region between DOCS_BEGIN/DOCS_END in the docs file with the + freshly generated tables, leaving surrounding prose untouched.""" + if not DOCS.exists(): + sys.exit(f"error: docs file not found: {DOCS}") + text = DOCS.read_text() + if DOCS_BEGIN not in text or DOCS_END not in text: + sys.exit( + f"error: ownership markers not found in {DOCS.relative_to(REPO_ROOT)}. " + f"Add the lines:\n {DOCS_BEGIN}\n {DOCS_END}\n" + f"around the generated table region." + ) + head, rest = text.split(DOCS_BEGIN, 1) + _, tail = rest.split(DOCS_END, 1) + new = f"{head}{DOCS_BEGIN}\n\n{table_md}\n{DOCS_END}{tail}" + DOCS.write_text(new) + + def main() -> int: if not SOURCE.exists(): sys.exit(f"error: source file not found: {SOURCE}") @@ -127,6 +195,9 @@ def main() -> int: OUTPUT.write_text(rendered) print(f"wrote {OUTPUT.relative_to(REPO_ROOT)}") + + splice_docs(ownership_tables(spec, roles)) + print(f"updated {DOCS.relative_to(REPO_ROOT)}") return 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b7b7b2..bbe5893 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,63 +261,6 @@ jobs: if: needs.classify_changes.outputs.run_full_ci == 'true' run: cargo test --locked -p omnigraph-server --features aws - test_windows_binaries: - name: Test Windows release binaries - needs: classify_changes - runs-on: windows-latest - timeout-minutes: 75 - permissions: - contents: read - env: - CARGO_TERM_COLOR: always - steps: - - name: Skip for text-only changes - if: needs.classify_changes.outputs.run_full_ci != 'true' - run: Write-Host "Text-only change detected; skipping Windows binary build." - - - name: Checkout source - if: needs.classify_changes.outputs.run_full_ci == 'true' - uses: actions/checkout@v5.0.1 - - - name: Install system dependencies - if: needs.classify_changes.outputs.run_full_ci == 'true' - run: choco install protoc -y - - - name: Install Rust stable - if: needs.classify_changes.outputs.run_full_ci == 'true' - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - - - name: Cache Rust build data - if: needs.classify_changes.outputs.run_full_ci == 'true' - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - . -> target - key: windows-release-binaries - - - name: Build Windows binaries - if: needs.classify_changes.outputs.run_full_ci == 'true' - run: cargo build --release --locked -p omnigraph-cli -p omnigraph-server - - - name: Smoke test Windows binaries - if: needs.classify_changes.outputs.run_full_ci == 'true' - run: | - & ./target/release/omnigraph.exe version - & ./target/release/omnigraph-server.exe --help - - - name: Check PowerShell installer syntax - if: needs.classify_changes.outputs.run_full_ci == 'true' - run: | - $tokens = $null - $errors = $null - [System.Management.Automation.Language.Parser]::ParseFile("scripts/install.ps1", [ref]$tokens, [ref]$errors) | Out-Null - if ($errors.Count -gt 0) { - $errors | Format-List - exit 1 - } - rustfs_integration: name: RustFS S3 Integration needs: diff --git a/.github/workflows/codeowners.yml b/.github/workflows/codeowners.yml index 19d5835..75b3515 100644 --- a/.github/workflows/codeowners.yml +++ b/.github/workflows/codeowners.yml @@ -1,19 +1,24 @@ name: CODEOWNERS +# Runs on EVERY pull request (no paths filter). The two jobs below are +# required status checks on `main`; a path-filtered required check never +# reports for PRs outside the filter and leaves them permanently "pending" +# (the trap that forced admin-override merges). Always-run + cheap +# short-circuit is what keeps them honest. on: pull_request: - paths: - - '.github/codeowners-roles.yml' - - '.github/CODEOWNERS' - - '.github/scripts/render-codeowners.py' - - '.github/workflows/codeowners.yml' workflow_dispatch: -# Read-only; we never push from this workflow. +# `drift` auto-commits the regenerated artifacts back to same-repo PR +# branches, so it needs write access. permissions: - contents: read + contents: write jobs: + # NOTE: the job `name:` values below ("CODEOWNERS matches source" / + # "CODEOWNERS not hand-edited") ARE the status-check contexts that + # .github/branch-protection.json must list verbatim. Renaming a job here + # is a branch-protection change β€” update the JSON and re-apply. drift: name: CODEOWNERS matches source runs-on: ubuntu-latest @@ -28,19 +33,56 @@ jobs: - name: Install PyYAML run: pip install pyyaml - - name: Re-render CODEOWNERS + - name: Re-render CODEOWNERS + ownership docs run: python3 .github/scripts/render-codeowners.py - - name: Reject drift + # Same-repo PR: push the regenerated artifacts back so contributors + # never have to run the script locally. Mirrors the openapi.json + # auto-commit in ci.yml (separate shallow clone of the head branch so + # the pushed commit carries only the regenerated files). + - name: Commit regenerated artifacts to PR branch + if: | + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if ! git diff --quiet .github/CODEOWNERS; then - echo "::error::.github/CODEOWNERS is out of sync with .github/codeowners-roles.yml." - echo "::error::Run \`python3 .github/scripts/render-codeowners.py\` locally and commit the result." + if git diff --quiet -- .github/CODEOWNERS docs/dev/codeowners.md; then + echo "CODEOWNERS and ownership docs already in sync." + exit 0 + fi + tmp=$(mktemp -d) + git clone --depth 1 --branch "${{ github.head_ref }}" \ + "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" \ + "$tmp" + cp .github/CODEOWNERS "$tmp/.github/CODEOWNERS" + cp docs/dev/codeowners.md "$tmp/docs/dev/codeowners.md" + cd "$tmp" + if git diff --quiet -- .github/CODEOWNERS docs/dev/codeowners.md; then + echo "Head branch already matches; nothing to push." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add .github/CODEOWNERS docs/dev/codeowners.md + git commit -m "chore: regenerate CODEOWNERS + ownership docs" + git push + + # Fork PR / workflow_dispatch: cannot push back, so enforce drift + # strictly. The contributor runs the script and commits the result. + - name: Verify in sync (forks / manual runs) + if: | + !(github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository) + run: | + if ! git diff --quiet -- .github/CODEOWNERS docs/dev/codeowners.md; then + echo "::error::Generated CODEOWNERS / ownership docs are out of sync with .github/codeowners-roles.yml." + echo "::error::Run \`python3 .github/scripts/render-codeowners.py\` and commit the result." echo "--- diff ---" - git --no-pager diff .github/CODEOWNERS + git --no-pager diff -- .github/CODEOWNERS docs/dev/codeowners.md exit 1 fi - echo "CODEOWNERS is in sync with its source." + echo "Generated artifacts are in sync with their source." noedit: name: CODEOWNERS not hand-edited @@ -52,6 +94,8 @@ jobs: fetch-depth: 0 - name: Reject hand-edits to generated file + # Only meaningful for PRs (needs a base to diff against). + if: github.event_name == 'pull_request' run: | base="origin/${{ github.base_ref }}" git fetch origin "${{ github.base_ref }}" --quiet diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3a66ff2..a265c40 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,16 +121,30 @@ jobs: run: | ./scripts/update-homebrew-formula.sh "${GITHUB_REF_NAME}" homebrew-tap/Formula/omnigraph.rb + # Diagnostic only: brew is not on PATH on the ubuntu runner by default, so + # set it up explicitly. Both this setup and the audit below are best-effort + # canaries, not gates β€” continue-on-error on each keeps a failed/flaky brew + # (the action is pinned to a moving @master ref) from skipping the actual + # tap publish below. The formula is correct by construction + # (update-homebrew-formula.sh), so brew tooling must never block the push. + - name: Set up Homebrew + if: env.HOMEBREW_TAP_SKIP != '1' + continue-on-error: true + uses: Homebrew/actions/setup-homebrew@master + - name: Audit generated formula if: env.HOMEBREW_TAP_SKIP != '1' + continue-on-error: true run: | # Audit the checked-out tap by name (brew audit rejects bare paths # and needs tap context). Symlink the checkout into Homebrew's Taps - # tree so `modernrelay/tap/omnigraph` resolves to it. + # tree so `modernrelay/tap/omnigraph` resolves to it. Offline audit + # (no --online) keeps it deterministic; it still catches the + # ComponentsOrder/structure class of problems. tap_dir="$(brew --repository)/Library/Taps/modernrelay/homebrew-tap" mkdir -p "$(dirname "$tap_dir")" ln -sfn "$PWD/homebrew-tap" "$tap_dir" - brew audit --strict --online modernrelay/tap/omnigraph + brew audit --strict modernrelay/tap/omnigraph - name: Commit and push formula update if: env.HOMEBREW_TAP_SKIP != '1' diff --git a/AGENTS.md b/AGENTS.md index 64a2699..5799f04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,8 +16,8 @@ Tools that support `@`-imports (Claude Code) auto-include all three files via th `CLAUDE.md` is a symlink to this file β€” there is exactly one source of truth. Edit `AGENTS.md`. -**Version surveyed:** 0.6.1 -**Workspace crates:** `omnigraph-compiler`, `omnigraph` (engine), `omnigraph-policy`, `omnigraph-config`, `omnigraph-queries`, `omnigraph-api-types`, `omnigraph-cli`, `omnigraph-server` +**Version surveyed:** 0.6.2 +**Workspace crates:** `omnigraph-compiler`, `omnigraph` (engine), `omnigraph-policy`, `omnigraph-config`, `omnigraph-queries`, `omnigraph-api-types`, `omnigraph-cluster`, `omnigraph-cli`, `omnigraph-server` **Storage substrate:** Lance 6.x (columnar, versioned, branchable) **License:** MIT **Toolchain:** Rust stable, edition 2024 @@ -214,8 +214,12 @@ omnigraph schema apply --schema ./next.pg s3://my-bucket/graph.omni --json # Merge review branch back omnigraph branch merge review/2026-04-25 --into main s3://my-bucket/graph.omni -# Compact + GC (preview, then confirm) +# Compact, preview any uncovered drift, then repair/GC after review omnigraph optimize s3://my-bucket/graph.omni +omnigraph repair s3://my-bucket/graph.omni +omnigraph repair --confirm s3://my-bucket/graph.omni +# For suspicious/unverifiable drift only after deliberate review: +# omnigraph repair --force --confirm s3://my-bucket/graph.omni omnigraph cleanup --keep 10 --older-than 7d s3://my-bucket/graph.omni omnigraph cleanup --keep 10 --older-than 7d --confirm s3://my-bucket/graph.omni @@ -236,8 +240,9 @@ omnigraph policy explain --actor act-alice --action change --branch main | Columnar storage on object store | βœ… Arrow/Lance | URI normalization, S3 env-var plumbing | | Per-dataset versioning + time travel | βœ… | `snapshot_at_version`, `entity_at`, snapshot-pinned reads across many tables | | Per-dataset branches | βœ… | **Graph-level** branches (atomic across all sub-tables), lazy fork, system branch filtering | -| Atomic single-dataset commits | βœ… | **Multi-table publish via three layers**, NOT a single Lance primitive: (1) per-table Lance `commit_staged` for the data write, (2) `__manifest` row-level CAS via `ManifestBatchPublisher` for cross-table ordering, (3) the open-time recovery sweep for the residual gap between (1) and (2). All three layers ship; the four migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`) write a `__recovery/{ulid}.json` sidecar before Phase B and delete it after Phase C. The next `Omnigraph::open` (gated on `OpenMode::ReadWrite`) runs the sweep in `db/manifest/recovery.rs`: classify, decide all-or-nothing per sidecar, roll forward via single `ManifestBatchPublisher::publish` or roll back via `Dataset::restore`, and record an audit row in `_graph_commit_recoveries.lance` (queryable via `omnigraph commit list --filter actor=omnigraph:recovery`). Continuous in-process recovery (no restart needed between Phase B failure and recovery) is the goal of a future background reconciler. Engine writes route through a sealed `TableStorage` trait exposing `stage_*` + `commit_staged` as the canonical staged-write surface; documented inline-commit residuals (`delete_where`, `create_vector_index`, plus legacy `append_batch` / `merge_insert_batches` / `overwrite_batch` / `create_*_index`) remain on the trait until upstream Lance ships a public two-phase API ([#6658](https://github.com/lance-format/lance/issues/6658), [#6666](https://github.com/lance-format/lance/issues/6666)) and the migration of every call site completes. | -| Compaction (`compact_files`) | βœ… | `omnigraph optimize` orchestrates over all node/edge tables, bounded concurrency; **skips blob-bearing tables** (reported via `TableOptimizeStats.skipped`, not silent), gated on `LANCE_SUPPORTS_BLOB_COMPACTION` until the upstream blob-v2 compaction-decode bug is fixed (see [docs/dev/invariants.md](docs/dev/invariants.md) Known Gaps) | +| Atomic single-dataset commits | βœ… | **Multi-table publish via three layers**, NOT a single Lance primitive: (1) per-table Lance `commit_staged` for the data write, (2) `__manifest` row-level CAS via `ManifestBatchPublisher` for cross-table ordering, (3) the open-time recovery sweep for the residual gap between (1) and (2). All three layers ship; the five migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`, `optimize_all_tables`) write a `__recovery/{ulid}.json` sidecar before Phase B and delete it after Phase C. The next `Omnigraph::open` (gated on `OpenMode::ReadWrite`) runs the sweep in `db/manifest/recovery.rs`: classify, decide all-or-nothing per sidecar, roll forward via single `ManifestBatchPublisher::publish` or roll back via `Dataset::restore` followed by a manifest publish of the restored version (so both directions converge to `manifest == HEAD` β€” no residual drift), and record an audit row in `_graph_commit_recoveries.lance` (queryable via `omnigraph commit list --filter actor=omnigraph:recovery`). Continuous in-process recovery (no restart needed between Phase B failure and recovery) is the goal of a future background reconciler. Engine writes route through a sealed `TableStorage` trait exposing `stage_*` + `commit_staged` as the canonical staged-write surface; documented inline-commit residuals (`delete_where`, `create_vector_index`, plus legacy `append_batch` / `merge_insert_batches` / `overwrite_batch` / `create_*_index`) remain on the trait until upstream Lance ships a public two-phase API ([#6658](https://github.com/lance-format/lance/issues/6658), [#6666](https://github.com/lance-format/lance/issues/6666)) and the migration of every call site completes. | +| Compaction (`compact_files`) | βœ… | `omnigraph optimize` orchestrates over all node/edge tables, bounded concurrency; **publishes each compacted table's new version to `__manifest`** (so the manifest tracks the Lance HEAD β€” required for reads to observe compaction and for schema apply / strict writes to pass their HEAD-vs-manifest precondition), under the per-`(table, main)` write queue with `SidecarKind::Optimize` recovery coverage; **refuses on an unrecovered graph** (errors if a `__recovery` sidecar is pending); **skips uncovered HEAD > manifest drift** with `DriftNeedsRepair` instead of interpreting it; **skips blob-bearing tables** (reported via `TableOptimizeStats.skipped`, not silent), gated on `LANCE_SUPPORTS_BLOB_COMPACTION` until the upstream blob-v2 compaction-decode bug is fixed (see [docs/dev/invariants.md](docs/dev/invariants.md) Known Gaps) | +| Repair uncovered drift | β€” | `omnigraph repair` explicitly classifies uncovered table `HEAD > manifest` drift: verified maintenance drift (`ReserveFragments`/`Rewrite`) can be published with `--confirm`; suspicious or unverifiable drift requires `--force --confirm`. Sidecar-covered crash residuals still recover automatically on open. | | Cleanup (`cleanup_old_versions`) | βœ… | `omnigraph cleanup` with `--keep` / `--older-than` policy | | BTREE / inverted (FTS) / vector indexes | βœ… | `ensure_indices` builds them on every relevant column; idempotent; lazy across branches | | `merge_insert` upsert | βœ… | `LoadMode::Merge`, mutation `update`/`insert`/`delete` lowering | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d9c687..2d77ef0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,29 @@ # Contributing -Small bug fixes and documentation improvements are welcome directly through pull -requests. +Thanks for your interest in OmniGraph. This page is the practical how-to; the +rules and decision authority behind it live in [GOVERNANCE.md](GOVERNANCE.md). -For larger changes, please open an issue or design discussion first so the -proposed direction is clear before implementation starts. +## Start in the right place + +| I want to… | Go to | Notes | +|---|---|---| +| **Report a bug** or wrong behavior | **[Open an Issue](../../issues/new/choose)** | Concrete and reproducible. A maintainer triages it; once labelled **`accepted`** it's open for a PR. | +| **Suggest a feature / share an idea / ask** | **[Start a Discussion](../../discussions)** | Ideas and questions live here, not in Issues. | +| **Propose a design / RFC** | **An RFC pull request** | Anyone can author one β€” see [docs/rfcs/README.md](docs/rfcs/README.md). A maintainer merging it is acceptance. | +| **Fix something / implement a change** | **A pull request** | Must link an `accepted` issue or an accepted RFC β€” unless it's trivial (below). | +| **Report a security vulnerability** | **[SECURITY.md](SECURITY.md)** | Do **not** open a public Issue. | + +### When can I just open a PR? +The **trivial fast-lane** β€” open directly, no prior issue/RFC needed: typo and +wording fixes, doc corrections, dependency bumps, comment fixes, obvious +one-line CI tweaks. Anything more substantial needs a backing `accepted` issue +or accepted RFC first, so the *why* is agreed before the *how* is reviewed. A PR +that turns out to be non-trivial will be redirected β€” that's about process, not +the merit of the change. + +> **Maintainers (ModernRelay team)** follow a separate internal process and are +> not bound by the intake rules above. Everyone is bound by review, CODEOWNERS, +> branch protection, and CI. ## Development @@ -49,6 +68,11 @@ CI runs both. ## Pull Requests -- keep changes focused -- include tests for behavior changes when practical -- update public docs when the user-facing surface changes +- **Link the backing issue or RFC** (`Closes #123`, or reference the RFC) β€” or + mark the PR as trivial per the fast-lane. +- Keep changes focused; one logical change per PR. +- Include tests for behavior changes when practical. +- Update public docs when the user-facing surface changes. + +New to the codebase? Read [AGENTS.md](AGENTS.md) β€” the architecture map and the +always-on invariants every change is reviewed against. diff --git a/Cargo.lock b/Cargo.lock index ba20563..ab7542a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4543,7 +4543,7 @@ dependencies = [ [[package]] name = "omnigraph-api-types" -version = "0.6.1" +version = "0.6.2" dependencies = [ "omnigraph-compiler", "omnigraph-engine", @@ -4555,7 +4555,7 @@ dependencies = [ [[package]] name = "omnigraph-cli" -version = "0.6.1" +version = "0.6.2" dependencies = [ "assert_cmd", "clap", @@ -4563,6 +4563,7 @@ dependencies = [ "lance", "lance-index", "omnigraph-api-types", + "omnigraph-cluster", "omnigraph-compiler", "omnigraph-config", "omnigraph-engine", @@ -4577,9 +4578,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "omnigraph-cluster" +version = "0.6.2" +dependencies = [ + "omnigraph-compiler", + "omnigraph-engine", + "serde", + "serde_json", + "serde_yaml", + "sha2", + "tempfile", + "thiserror", + "time", + "tokio", + "ulid", +] + [[package]] name = "omnigraph-compiler" -version = "0.6.1" +version = "0.6.2" dependencies = [ "ahash", "arrow-array", @@ -4600,7 +4618,7 @@ dependencies = [ [[package]] name = "omnigraph-config" -version = "0.6.1" +version = "0.6.2" dependencies = [ "clap", "color-eyre", @@ -4613,7 +4631,7 @@ dependencies = [ [[package]] name = "omnigraph-engine" -version = "0.6.1" +version = "0.6.2" dependencies = [ "arc-swap", "arrow-array", @@ -4638,6 +4656,7 @@ dependencies = [ "object_store 0.12.5", "omnigraph-compiler", "omnigraph-policy", + "proptest", "regex", "reqwest", "serde", @@ -4654,7 +4673,7 @@ dependencies = [ [[package]] name = "omnigraph-policy" -version = "0.6.1" +version = "0.6.2" dependencies = [ "cedar-policy", "clap", @@ -4667,7 +4686,7 @@ dependencies = [ [[package]] name = "omnigraph-queries" -version = "0.6.1" +version = "0.6.2" dependencies = [ "omnigraph-compiler", "omnigraph-config", @@ -4676,7 +4695,7 @@ dependencies = [ [[package]] name = "omnigraph-server" -version = "0.6.1" +version = "0.6.2" dependencies = [ "arc-swap", "async-trait", @@ -5164,6 +5183,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand 0.9.2", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.14.3" @@ -5225,6 +5263,12 @@ dependencies = [ "cc", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.37.5" @@ -5396,6 +5440,15 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -5795,6 +5848,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -6792,6 +6857,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.9.0" diff --git a/Cargo.toml b/Cargo.toml index 34c63b8..8c955e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/omnigraph-compiler", "crates/omnigraph", "crates/omnigraph-cli", + "crates/omnigraph-cluster", "crates/omnigraph-policy", "crates/omnigraph-server", "crates/omnigraph-config", diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..5878f1f --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,106 @@ +# Governance + +This document describes how **external contributions** to OmniGraph are +proposed, accepted, and merged. It exists so an outside contributor can answer, +without asking: *where does my report/idea/change go, who decides, and what has +to happen before code lands?* + +> **Scope.** This governs the public contribution surface β€” Issues, +> Discussions, RFCs, and pull requests from people outside the ModernRelay +> team. **Maintainers operate under a separate internal process** and are not +> bound by the intake gates below. Everyone, maintainer or not, is still bound +> by the universal gates: branch protection on `main` and CODEOWNERS review +> (see [docs/dev/branch-protection.md](docs/dev/branch-protection.md) and +> [docs/dev/codeowners.md](docs/dev/codeowners.md)). + +## Roles + +| Role | Who | Authority | +|---|---|---| +| **Maintainer** | The code owners in [`.github/CODEOWNERS`](.github/CODEOWNERS) (generated from [`.github/codeowners-roles.yml`](.github/codeowners-roles.yml)) | Validate issues, accept/reject RFCs, review and merge PRs, set direction. Final decision authority. | +| **Contributor** | Anyone else | Report problems (Issues), propose ideas (Discussions), author RFCs, and open pull requests. | + +Decision authority rests with the maintainers. CODEOWNERS is the single source +of truth for who that is; this document does not duplicate the list. + +## The three channels + +Each channel has one job. Using the right one is the first thing we ask of a +contribution. + +| Channel | Purpose | Not for | +|---|---|---| +| **[Issues](../../issues)** | **Report a problem** β€” a bug, a regression, a documented behavior that's wrong. Something concrete and reproducible. | Feature requests, ideas, questions, or design proposals (β†’ Discussions). | +| **[Discussions](../../discussions)** | **Propose and explore** β€” new ideas, feature requests, questions, and the incubation of RFCs. | Bug reports (β†’ Issues). | +| **Pull requests** | **Land a sanctioned change** β€” a fix for a *validated* issue, an *accepted* RFC, or a trivial change (see fast-lane). | Substantive change with no backing issue/RFC β€” it will be redirected. | + +## How a change becomes mergeable + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ bug ───────────┐ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€ idea / feature ────────┐ + β–Ό β”‚ β–Ό β”‚ + Issue (problem report) β”‚ Discussion (idea / RFC incubation) β”‚ + β”‚ β”‚ β”‚ β”‚ + maintainer triage β”‚ rough consensus β”‚ + β”‚ β”‚ β”‚ graduate β”‚ + β–Ό β”‚ β–Ό β”‚ + label: accepted ──────────┐ β”‚ RFC PR (docs/rfcs/NNNN-*.md) β”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ β”‚ maintainer review β”‚ + β–Ό β–Ό β”‚ β–Ό β”‚ + Pull request ◀──────────┴──────────│── merged == accepted β”‚ + (links the issue or the accepted RFC) β—€β”€β”€β”€β”€β”€β”€β”€β”˜ (implementation PRs reference it) β”‚ + β”‚ + review + CODEOWNERS + branch protection + β–Ό + merged +``` + +### Issues β†’ validated +A new issue starts unlabeled. A maintainer triages it and, if it's a real, +in-scope problem, applies the **`accepted`** label. **Only `accepted` issues are +open for a contributor PR.** This prevents the "I fixed an issue you hadn't +agreed was a problem" rejection. Want to fix something? Get the issue accepted +first, or pick one already labelled `accepted` / `help wanted`. + +### Discussions β†’ RFCs β†’ accepted +Ideas and feature requests start in **Discussions**. Anyone β€” including external +contributors β€” may then **author an RFC** by opening a pull request that adds +`docs/rfcs/NNNN-title.md` (see [docs/rfcs/README.md](docs/rfcs/README.md)). The +RFC is reviewed as code; **a maintainer merging it is the act of acceptance** +(it becomes the durable decision record). Implementation PRs then reference the +accepted RFC. + +Authoring an RFC is open to everyone; **accepting one is a maintainer +decision.** Maintainers may also decline an RFC, with rationale, by closing it. + +### Pull requests β†’ sanctioned +A contributor PR must do one of: +1. link a maintainer-**`accepted`** issue it fixes, or +2. be (or reference) an **accepted RFC**, or +3. qualify for the **trivial fast-lane**. + +**Trivial fast-lane** β€” these may be opened directly, no prior issue/RFC: +typo and wording fixes, documentation corrections, dependency bumps, comment +fixes, and obviously-correct one-line CI tweaks. When in doubt, open an Issue or +Discussion first; a PR that turns out to be non-trivial will be asked to. + +A substantive PR with no backing issue/RFC will be closed with a pointer to the +right channel β€” not as a judgment of the idea, but to keep design discussion +where it's reviewable. + +## What maintainers do *not* gate +Maintainers' own changes do not pass through the intake gates above β€” the team +runs a separate internal process. The universal gates (review, CODEOWNERS, +branch protection, CI) apply to everyone. Enforcement of the intake rules is, to +start, **by convention and review** (PR template + labels); an automated check +keyed to author association may be added later if volume warrants. + +## Code of conduct & security +- Conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). +- Security issues are **not** public Issues β€” see [SECURITY.md](SECURITY.md). + +## Changing this document +Governance changes the same way code does: a pull request, reviewed by +maintainers. This file describes the external surface; the internal maintainer +process is intentionally out of scope here. diff --git a/crates/omnigraph-api-types/Cargo.toml b/crates/omnigraph-api-types/Cargo.toml index d3f965c..afaefe4 100644 --- a/crates/omnigraph-api-types/Cargo.toml +++ b/crates/omnigraph-api-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-api-types" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "HTTP request/response types (OpenAPI schemas) for the Omnigraph graph database." license = "MIT" @@ -9,9 +9,9 @@ homepage = "https://github.com/ModernRelay/omnigraph" documentation = "https://docs.rs/omnigraph-api-types" [dependencies] -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.1" } -omnigraph-queries = { path = "../omnigraph-queries", version = "0.6.1" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.2" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } +omnigraph-queries = { path = "../omnigraph-queries", version = "0.6.2" } serde = { workspace = true } serde_json = { workspace = true } utoipa = { workspace = true } diff --git a/crates/omnigraph-cli/Cargo.toml b/crates/omnigraph-cli/Cargo.toml index 68321dd..d49cbd3 100644 --- a/crates/omnigraph-cli/Cargo.toml +++ b/crates/omnigraph-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-cli" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "CLI for the Omnigraph graph database." license = "MIT" @@ -13,12 +13,13 @@ name = "omnigraph" path = "src/main.rs" [dependencies] -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.1" } -omnigraph-policy = { path = "../omnigraph-policy", version = "0.6.1" } -omnigraph-config = { path = "../omnigraph-config", version = "0.6.1" } -omnigraph-queries = { path = "../omnigraph-queries", version = "0.6.1" } -omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.6.1" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.2" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } +omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.6.2" } +omnigraph-policy = { path = "../omnigraph-policy", version = "0.6.2" } +omnigraph-config = { path = "../omnigraph-config", version = "0.6.2" } +omnigraph-queries = { path = "../omnigraph-queries", version = "0.6.2" } +omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.6.2" } clap = { workspace = true } color-eyre = { workspace = true } serde = { workspace = true } diff --git a/crates/omnigraph-cli/src/main.rs b/crates/omnigraph-cli/src/main.rs index 748d44d..2c65004 100644 --- a/crates/omnigraph-cli/src/main.rs +++ b/crates/omnigraph-cli/src/main.rs @@ -18,6 +18,10 @@ use omnigraph_api_types::{ SnapshotTableOutput, commit_output, ingest_output, read_output, schema_apply_output, snapshot_payload, }; +use omnigraph_cluster::{ + DiagnosticSeverity, PlanOutput, StateSyncOutput, StatusOutput, ValidateOutput, + import_config_dir, plan_config_dir, refresh_config_dir, status_config_dir, validate_config_dir, +}; use omnigraph_compiler::query::parser::parse_query; use omnigraph_compiler::schema::parser::parse_schema; use omnigraph_compiler::{ @@ -286,6 +290,25 @@ enum Command { #[arg(long)] json: bool, }, + /// Classify and explicitly repair manifest/head drift + Repair { + /// Graph URI + uri: Option, + #[arg(long)] + target: Option, + #[arg(long)] + config: Option, + /// Publish verified maintenance drift. Without this flag, repair only + /// previews what it would do. + #[arg(long)] + confirm: bool, + /// Also publish suspicious or unverifiable drift. Requires + /// `--confirm`; use only after operator review. + #[arg(long, requires = "confirm")] + force: bool, + #[arg(long)] + json: bool, + }, /// Remove old Lance versions from every table of the graph (destructive) Cleanup { /// Graph URI @@ -308,6 +331,11 @@ enum Command { #[arg(long)] json: bool, }, + /// Validate and plan read-only cluster configuration. + Cluster { + #[command(subcommand)] + command: ClusterCommand, + }, /// Manage graphs on a multi-graph server (MR-668) Graphs { #[command(subcommand)] @@ -328,6 +356,55 @@ enum Command { }, } +#[derive(Debug, Subcommand)] +enum ClusterCommand { + /// Validate cluster.yaml and referenced schemas, queries, and policy files. + Validate { + /// Cluster config directory containing cluster.yaml. + #[arg(long, default_value = ".")] + config: PathBuf, + /// Emit JSON instead of human text. + #[arg(long)] + json: bool, + }, + /// Produce a read-only plan by diffing cluster.yaml against __cluster/state.json. + Plan { + /// Cluster config directory containing cluster.yaml. + #[arg(long, default_value = ".")] + config: PathBuf, + /// Emit JSON instead of human text. + #[arg(long)] + json: bool, + }, + /// Read the local JSON state ledger without scanning live graph resources. + Status { + /// Cluster config directory containing cluster.yaml. + #[arg(long, default_value = ".")] + config: PathBuf, + /// Emit JSON instead of human text. + #[arg(long)] + json: bool, + }, + /// Refresh existing local JSON state from declared graph observations. + Refresh { + /// Cluster config directory containing cluster.yaml. + #[arg(long, default_value = ".")] + config: PathBuf, + /// Emit JSON instead of human text. + #[arg(long)] + json: bool, + }, + /// Import initial local JSON state from declared graph observations. + Import { + /// Cluster config directory containing cluster.yaml. + #[arg(long, default_value = ".")] + config: PathBuf, + /// Emit JSON instead of human text. + #[arg(long)] + json: bool, + }, +} + /// Operations on the graph registry of a multi-graph server (MR-668). /// /// All operations target a remote multi-graph server URL (http:// or @@ -720,6 +797,159 @@ fn print_json(value: &T) -> Result<()> { Ok(()) } +fn print_cluster_validate_human(output: &ValidateOutput) { + if output.ok { + println!( + "cluster config valid: {} resource(s), {} dependency edge(s)", + output.resources.len(), + output.dependencies.len() + ); + } else { + println!("cluster config invalid"); + } + print_cluster_diagnostics(&output.diagnostics); +} + +fn print_cluster_plan_human(output: &PlanOutput) { + if output.ok { + println!( + "cluster plan: {} change(s), {} approval gate(s)", + output.changes.len(), + output.approvals_required.len() + ); + for change in &output.changes { + println!(" {:?} {}", change.operation, change.resource); + } + if output.changes.is_empty() { + println!(" no changes"); + } + } else { + println!("cluster plan failed"); + } + print_cluster_diagnostics(&output.diagnostics); +} + +fn print_cluster_status_human(output: &StatusOutput) { + if output.ok { + let state = &output.state_observations; + if state.state_found { + println!( + "cluster state: revision {}, {} resource(s)", + state.state_revision, state.resource_count + ); + if let Some(digest) = state.applied_config_digest.as_deref() { + println!(" applied config: {digest}"); + } + if state.locked { + match state.lock_id.as_deref() { + Some(lock_id) => println!(" lock: held ({lock_id})"), + None => println!(" lock: held"), + } + } else { + println!(" lock: not held"); + } + } else { + println!("cluster state missing"); + } + } else { + println!("cluster status failed"); + } + print_cluster_diagnostics(&output.diagnostics); +} + +fn print_cluster_state_sync_human(output: &StateSyncOutput) { + let operation = match output.operation { + omnigraph_cluster::StateSyncOperation::Refresh => "refresh", + omnigraph_cluster::StateSyncOperation::Import => "import", + }; + if output.ok { + let state = &output.state_observations; + println!( + "cluster {operation}: revision {}, {} resource(s)", + state.state_revision, state.resource_count + ); + if let Some(cas) = state.state_cas.as_deref() { + println!(" state_cas: {cas}"); + } + if state.locked { + match state.lock_id.as_deref() { + Some(lock_id) => println!(" lock: acquired ({lock_id})"), + None => println!(" lock: acquired"), + } + } else { + println!(" lock: not acquired"); + } + } else { + println!("cluster {operation} failed"); + } + print_cluster_diagnostics(&output.diagnostics); +} + +fn print_cluster_diagnostics(diagnostics: &[omnigraph_cluster::Diagnostic]) { + for diagnostic in diagnostics { + let label = match diagnostic.severity { + DiagnosticSeverity::Error => "ERROR", + DiagnosticSeverity::Warning => "WARN ", + }; + println!( + "{label} {} {}: {}", + diagnostic.code, diagnostic.path, diagnostic.message + ); + } +} + +fn finish_cluster_validate(output: &ValidateOutput, json: bool) -> Result<()> { + if json { + print_json(output)?; + } else { + print_cluster_validate_human(output); + } + if !output.ok { + io::stdout().flush()?; + std::process::exit(1); + } + Ok(()) +} + +fn finish_cluster_plan(output: &PlanOutput, json: bool) -> Result<()> { + if json { + print_json(output)?; + } else { + print_cluster_plan_human(output); + } + if !output.ok { + io::stdout().flush()?; + std::process::exit(1); + } + Ok(()) +} + +fn finish_cluster_status(output: &StatusOutput, json: bool) -> Result<()> { + if json { + print_json(output)?; + } else { + print_cluster_status_human(output); + } + if !output.ok { + io::stdout().flush()?; + std::process::exit(1); + } + Ok(()) +} + +fn finish_cluster_state_sync(output: &StateSyncOutput, json: bool) -> Result<()> { + if json { + print_json(output)?; + } else { + print_cluster_state_sync_human(output); + } + if !output.ok { + io::stdout().flush()?; + std::process::exit(1); + } + Ok(()) +} + fn is_remote_uri(uri: &str) -> bool { uri.starts_with("http://") || uri.starts_with("https://") } @@ -3188,6 +3418,8 @@ async fn main() -> Result<()> { "fragments_added": s.fragments_added, "committed": s.committed, "skipped": s.skipped.map(|r| r.as_str()), + "manifest_version": s.manifest_version, + "lance_head_version": s.lance_head_version, })).collect::>(), }); print_json(&value)?; @@ -3207,6 +3439,89 @@ async fn main() -> Result<()> { } } } + Command::Repair { + uri, + target, + config, + confirm, + force, + json, + } => { + let config = load_cli_config(config.as_ref())?; + let uri = resolve_uri(&config, uri, target.as_deref())?; + let db = Omnigraph::open(&uri).await?; + let stats = db + .repair(omnigraph::db::RepairOptions { confirm, force }) + .await?; + let refused_count = stats + .tables + .iter() + .filter(|s| matches!(s.action, omnigraph::db::RepairAction::Refused)) + .count(); + if json { + let value = serde_json::json!({ + "uri": uri, + "confirm": confirm, + "force": force, + "manifest_version": stats.manifest_version, + "tables": stats.tables.iter().map(|s| serde_json::json!({ + "table_key": s.table_key, + "manifest_version": s.manifest_version, + "lance_head_version": s.lance_head_version, + "classification": s.classification.as_str(), + "action": s.action.as_str(), + "operations": s.operations, + "error": s.error, + })).collect::>(), + }); + print_json(&value)?; + } else { + let mode = if confirm { "confirm" } else { "preview" }; + println!( + "repair {} β€” {} mode, {} tables", + uri, + mode, + stats.tables.len() + ); + for s in &stats.tables { + let drift = if s.manifest_version == s.lance_head_version { + format!("{}", s.manifest_version) + } else { + format!("{} β†’ {}", s.manifest_version, s.lance_head_version) + }; + let ops = if s.operations.is_empty() { + String::new() + } else { + format!(" [{}]", s.operations.join(", ")) + }; + let err = s + .error + .as_ref() + .map(|err| format!(" ({err})")) + .unwrap_or_default(); + println!( + " {:<40} {:<12} {:<22} {}{}{}", + s.table_key, + s.action.as_str(), + s.classification.as_str(), + drift, + ops, + err + ); + } + if !confirm { + println!("rerun with --confirm to publish verified maintenance drift"); + } + } + if refused_count > 0 { + bail!( + "repair refused {} suspicious or unverifiable table(s); review the preview \ + output and rerun with --force --confirm only if publishing that drift is \ + intentional", + refused_count + ); + } + } Command::Cleanup { uri, target, @@ -3287,6 +3602,28 @@ async fn main() -> Result<()> { } } } + Command::Cluster { command } => match command { + ClusterCommand::Validate { config, json } => { + let output = validate_config_dir(config); + finish_cluster_validate(&output, json)?; + } + ClusterCommand::Plan { config, json } => { + let output = plan_config_dir(config); + finish_cluster_plan(&output, json)?; + } + ClusterCommand::Status { config, json } => { + let output = status_config_dir(config); + finish_cluster_status(&output, json)?; + } + ClusterCommand::Refresh { config, json } => { + let output = refresh_config_dir(config).await; + finish_cluster_state_sync(&output, json)?; + } + ClusterCommand::Import { config, json } => { + let output = import_config_dir(config).await; + finish_cluster_state_sync(&output, json)?; + } + }, Command::Graphs { command } => match command { GraphsCommand::List { uri, diff --git a/crates/omnigraph-cli/tests/cli.rs b/crates/omnigraph-cli/tests/cli.rs index 8dba27e..6e664cf 100644 --- a/crates/omnigraph-cli/tests/cli.rs +++ b/crates/omnigraph-cli/tests/cli.rs @@ -1,5 +1,6 @@ use std::fs; +use lance::Dataset; use lance::index::DatasetIndexExt; use omnigraph::db::{Omnigraph, ReadTarget}; use serde_json::Value; @@ -60,6 +61,25 @@ fn manifest_dataset_version(graph: &std::path::Path) -> u64 { }) } +fn forge_person_delete_drift(graph: &std::path::Path) -> (u64, u64) { + tokio::runtime::Runtime::new().unwrap().block_on(async { + let uri = graph.to_string_lossy(); + let db = Omnigraph::open(uri.as_ref()).await.unwrap(); + let snap = db + .snapshot_of(ReadTarget::branch("main")) + .await + .unwrap(); + let entry = snap.entry("node:Person").unwrap(); + let full_path = format!("{}/{}", uri.trim_end_matches('/'), entry.table_path); + let mut ds = Dataset::open(&full_path).await.unwrap(); + let deleted = ds.delete("name = 'Alice'").await.unwrap(); + assert_eq!(deleted.num_deleted_rows, 1); + let head = deleted.new_dataset.version().version; + assert!(head > entry.table_version); + (entry.table_version, head) + }) +} + fn write_policy_config_fixture(root: &std::path::Path) -> (std::path::PathBuf, std::path::PathBuf) { let config = root.join("omnigraph.yaml"); let policy = root.join("policy.yaml"); @@ -78,6 +98,64 @@ policy: (config, policy) } +fn write_cluster_config_fixture(root: &std::path::Path) { + fs::write( + root.join("people.pg"), + r#" +node Person { + name: String @key + age: I32? +} +"#, + ) + .unwrap(); + fs::write( + root.join("people.gq"), + r#" +query find_person($name: String) { + match { $p: Person { name: $name } } + return { $p.name, $p.age } +} +"#, + ) + .unwrap(); + fs::write(root.join("base.policy.yaml"), "rules: []\n").unwrap(); + fs::write( + root.join("cluster.yaml"), + r#" +version: 1 +metadata: + name: company-brain +state: + backend: cluster + lock: true +graphs: + knowledge: + schema: ./people.pg + queries: + find_person: + file: ./people.gq +policies: + base: + file: ./base.policy.yaml + applies_to: [knowledge] +"#, + ) + .unwrap(); +} + +fn init_cluster_derived_graph(root: &std::path::Path) { + let graph_dir = root.join("graphs"); + fs::create_dir_all(&graph_dir).unwrap(); + output_success( + cli() + .arg("init") + .arg("--schema") + .arg(root.join("people.pg")) + .arg(graph_dir.join("knowledge.omni")), + ); +} + #[test] fn version_command_prints_current_cli_version() { let output = output_success(cli().arg("version")); @@ -89,6 +167,470 @@ fn version_command_prints_current_cli_version() { ); } +#[test] +fn cluster_validate_config_success() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + + let output = output_success( + cli() + .arg("cluster") + .arg("validate") + .arg("--config") + .arg(temp.path()), + ); + let stdout = stdout_string(&output); + assert!(stdout.contains("cluster config valid"), "{stdout}"); +} + +#[test] +fn cluster_validate_json_is_stable() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("validate") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert!(json["resource_digests"]["graph.knowledge"].is_string()); + assert!(json["resource_digests"]["query.knowledge.find_person"].is_string()); + assert_eq!(json["dependencies"][0]["from"], "policy.base"); + assert_eq!(json["dependencies"][0]["to"], "graph.knowledge"); +} + +#[test] +fn cluster_plan_json_reads_inferred_local_state() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#" +{ + "version": 1, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": "old-graph" }, + "policy.old": { "digest": "old-policy" } + } + } +} +"#, + ) + .unwrap(); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("plan") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert_eq!(json["state_observations"]["state_found"], true); + assert!( + json["changes"] + .as_array() + .unwrap() + .iter() + .any(|change| change["resource"] == "policy.old" && change["operation"] == "delete"), + "plan should read state and delete stale resources: {json}" + ); +} + +#[test] +fn cluster_status_json_reports_missing_state() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("status") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert_eq!(json["state_observations"]["state_found"], false); + assert!( + json["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|diagnostic| diagnostic["code"] == "state_missing"), + "missing state should be a warning diagnostic: {json}" + ); +} + +#[test] +fn cluster_status_json_reports_extended_state() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#" +{ + "version": 1, + "state_revision": 5, + "applied_revision": { + "config_digest": "applied", + "resources": { + "graph.knowledge": { "digest": "graph-digest" } + } + }, + "resource_statuses": { + "graph.knowledge": { "status": "applied", "conditions": ["healthy"] } + }, + "approval_records": {}, + "recovery_records": {}, + "observations": {} +} +"#, + ) + .unwrap(); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("status") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert_eq!(json["state_observations"]["state_revision"], 5); + assert!( + json["state_observations"]["state_cas"] + .as_str() + .unwrap() + .starts_with("sha256:") + ); + assert_eq!(json["resource_digests"]["graph.knowledge"], "graph-digest"); + assert_eq!( + json["resource_statuses"]["graph.knowledge"]["status"], + "applied" + ); +} + +#[test] +fn cluster_plan_json_includes_state_cas_revision_and_lock_observation() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#" +{ + "version": 1, + "state_revision": 9, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": "old-graph" } + } + } +} +"#, + ) + .unwrap(); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("plan") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert_eq!(json["state_observations"]["state_revision"], 9); + assert!( + json["state_observations"]["state_cas"] + .as_str() + .unwrap() + .starts_with("sha256:") + ); + assert_eq!(json["state_observations"]["locked"], false); + assert_eq!(json["state_observations"]["lock_acquired"], true); + assert!(json["state_observations"]["acquired_lock_id"].is_string()); + assert!(!state_dir.join("lock.json").exists()); +} + +#[test] +fn cluster_plan_locked_state_exits_nonzero() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("lock.json"), + r#" +{ + "version": 1, + "lock_id": "held-lock", + "operation": "plan", + "created_at": "2026-06-08T00:00:00Z", + "pid": 123 +} +"#, + ) + .unwrap(); + + let output = output_failure( + cli() + .arg("cluster") + .arg("plan") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + ); + let json = parse_stdout_json(&output); + assert_eq!(json["ok"], false); + assert_eq!(json["state_observations"]["locked"], true); + assert_eq!(json["state_observations"]["lock_acquired"], false); + assert_eq!(json["state_observations"]["lock_id"], "held-lock"); + assert!( + json["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|diagnostic| diagnostic["code"] == "state_lock_held"), + "locked state should produce a useful diagnostic: {json}" + ); +} + +#[test] +fn cluster_import_json_bootstraps_missing_state() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + init_cluster_derived_graph(temp.path()); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("import") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert_eq!(json["operation"], "import"); + assert_eq!(json["state_observations"]["state_revision"], 1); + assert!( + json["state_observations"]["state_cas"] + .as_str() + .unwrap() + .starts_with("sha256:") + ); + assert_eq!(json["state_observations"]["locked"], false); + assert_eq!(json["state_observations"]["lock_acquired"], true); + assert!(json["state_observations"]["acquired_lock_id"].is_string()); + assert!(json["observations"]["graph.knowledge"]["manifest_version"].is_number()); + assert_eq!( + json["resource_statuses"]["graph.knowledge"]["status"], + "applied" + ); + assert!(temp.path().join("__cluster/state.json").exists()); + assert!(!temp.path().join("__cluster/lock.json").exists()); +} + +#[test] +fn cluster_refresh_json_updates_revision_cas_and_removes_lock() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + init_cluster_derived_graph(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#" +{ + "version": 1, + "state_revision": 2, + "applied_revision": { "resources": {} } +} +"#, + ) + .unwrap(); + + let json = parse_stdout_json(&output_success( + cli() + .arg("cluster") + .arg("refresh") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(json["ok"], true); + assert_eq!(json["operation"], "refresh"); + assert_eq!(json["state_observations"]["state_revision"], 3); + assert!( + json["state_observations"]["state_cas"] + .as_str() + .unwrap() + .starts_with("sha256:") + ); + assert_eq!(json["state_observations"]["locked"], false); + assert_eq!(json["state_observations"]["lock_acquired"], true); + assert!(json["state_observations"]["acquired_lock_id"].is_string()); + assert!(!state_dir.join("lock.json").exists()); +} + +#[test] +fn cluster_refresh_missing_state_exits_nonzero() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + + let output = output_failure( + cli() + .arg("cluster") + .arg("refresh") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + ); + let json = parse_stdout_json(&output); + assert_eq!(json["ok"], false); + assert!( + json["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|diagnostic| diagnostic["code"] == "state_missing"), + "missing state should produce a useful diagnostic: {json}" + ); +} + +#[test] +fn cluster_import_existing_state_exits_nonzero() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let output = output_failure( + cli() + .arg("cluster") + .arg("import") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + ); + let json = parse_stdout_json(&output); + assert_eq!(json["ok"], false); + assert!( + json["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|diagnostic| diagnostic["code"] == "state_already_exists"), + "existing state should produce a useful diagnostic: {json}" + ); +} + +#[test] +fn cluster_refresh_and_import_locked_state_exit_nonzero() { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{"version":1,"lock_id":"held-lock","operation":"refresh","created_at":"2026-06-08T00:00:00Z","pid":123}"#, + ) + .unwrap(); + + let refresh = parse_stdout_json(&output_failure( + cli() + .arg("cluster") + .arg("refresh") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(refresh["state_observations"]["locked"], true); + assert_eq!(refresh["state_observations"]["lock_id"], "held-lock"); + assert_eq!(refresh["state_observations"]["lock_acquired"], false); + assert!( + refresh["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|diagnostic| diagnostic["code"] == "state_lock_held") + ); + + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let state_dir = temp.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{"version":1,"lock_id":"held-lock","operation":"import","created_at":"2026-06-08T00:00:00Z","pid":123}"#, + ) + .unwrap(); + + let imported = parse_stdout_json(&output_failure( + cli() + .arg("cluster") + .arg("import") + .arg("--config") + .arg(temp.path()) + .arg("--json"), + )); + assert_eq!(imported["state_observations"]["locked"], true); + assert_eq!(imported["state_observations"]["lock_id"], "held-lock"); + assert_eq!(imported["state_observations"]["lock_acquired"], false); + assert!( + imported["diagnostics"] + .as_array() + .unwrap() + .iter() + .any(|diagnostic| diagnostic["code"] == "state_lock_held") + ); +} + +#[test] +fn cluster_validate_invalid_config_exits_nonzero() { + let temp = tempdir().unwrap(); + fs::write( + temp.path().join("cluster.yaml"), + "version: 1\ngraphs: {}\npipelines: {}\n", + ) + .unwrap(); + + let output = output_failure( + cli() + .arg("cluster") + .arg("validate") + .arg("--config") + .arg(temp.path()), + ); + let stdout = stdout_string(&output); + assert!(stdout.contains("future_phase_field"), "{stdout}"); +} + #[test] fn short_version_flag_prints_current_cli_version() { let output = output_success(cli().arg("-v")); @@ -450,6 +992,83 @@ fn explicit_omnigraph_config_pointing_at_missing_file_errors() { ); } +#[test] +fn repair_json_reports_noop_on_clean_graph() { + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + load_fixture(&graph); + + let output = output_success(cli().arg("repair").arg("--json").arg(&graph)); + let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); + + assert_eq!(payload["confirm"], false); + assert_eq!(payload["force"], false); + assert_eq!(payload["manifest_version"], Value::Null); + let tables = payload["tables"].as_array().unwrap(); + assert_eq!(tables.len(), 4); + assert!(tables.iter().all(|table| { + table["classification"] == "no_drift" && table["action"] == "no_op" + })); +} + +#[test] +fn repair_confirm_json_refuses_suspicious_drift_with_nonzero_exit_then_force_succeeds() { + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + load_fixture(&graph); + let graph_manifest_before = manifest_dataset_version(&graph); + let (table_manifest_before, table_head_before) = forge_person_delete_drift(&graph); + + let refused = output_failure( + cli() + .arg("repair") + .arg("--confirm") + .arg("--json") + .arg(&graph), + ); + let refused_payload: Value = serde_json::from_slice(&refused.stdout).unwrap(); + assert_eq!(refused_payload["manifest_version"], Value::Null); + let person = refused_payload["tables"] + .as_array() + .unwrap() + .iter() + .find(|table| table["table_key"] == "node:Person") + .unwrap(); + assert_eq!(person["classification"], "suspicious"); + assert_eq!(person["action"], "refused"); + assert!( + String::from_utf8_lossy(&refused.stderr).contains("repair refused"), + "stderr should explain the non-zero exit; got: {}", + String::from_utf8_lossy(&refused.stderr) + ); + assert_eq!(manifest_dataset_version(&graph), graph_manifest_before); + + let forced = output_success( + cli() + .arg("repair") + .arg("--force") + .arg("--confirm") + .arg("--json") + .arg(&graph), + ); + let forced_payload: Value = serde_json::from_slice(&forced.stdout).unwrap(); + let forced_manifest = forced_payload["manifest_version"].as_u64().unwrap(); + assert!(forced_manifest > graph_manifest_before); + let person = forced_payload["tables"] + .as_array() + .unwrap() + .iter() + .find(|table| table["table_key"] == "node:Person") + .unwrap(); + assert_eq!(person["classification"], "suspicious"); + assert_eq!(person["action"], "forced"); + assert_eq!(person["manifest_version"], table_manifest_before); + assert_eq!(person["lance_head_version"], table_head_before); + assert_eq!(manifest_dataset_version(&graph), forced_manifest); +} + #[test] fn schema_plan_json_reports_supported_additive_change() { let temp = tempdir().unwrap(); diff --git a/crates/omnigraph-cluster/Cargo.toml b/crates/omnigraph-cluster/Cargo.toml new file mode 100644 index 0000000..9280c42 --- /dev/null +++ b/crates/omnigraph-cluster/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "omnigraph-cluster" +version = "0.6.2" +edition = "2024" +description = "Read-only cluster configuration validation and planning for Omnigraph." +license = "MIT" +repository = "https://github.com/ModernRelay/omnigraph" +homepage = "https://github.com/ModernRelay/omnigraph" +documentation = "https://docs.rs/omnigraph-cluster" + +[dependencies] +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.2" } +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +ulid = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true } diff --git a/crates/omnigraph-cluster/src/lib.rs b/crates/omnigraph-cluster/src/lib.rs new file mode 100644 index 0000000..9a6ea78 --- /dev/null +++ b/crates/omnigraph-cluster/src/lib.rs @@ -0,0 +1,2784 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, OpenOptions}; +use std::io::{ErrorKind, Write}; +use std::path::{Path, PathBuf}; +use std::process; + +use omnigraph::db::{Omnigraph, ReadTarget}; +use omnigraph_compiler::build_catalog; +use omnigraph_compiler::query::parser::parse_query; +use omnigraph_compiler::query::typecheck::typecheck_query_decl; +use omnigraph_compiler::schema::parser::parse_schema; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use ulid::Ulid; + +pub const CLUSTER_CONFIG_FILE: &str = "cluster.yaml"; +pub const CLUSTER_GRAPHS_DIR: &str = "graphs"; +pub const CLUSTER_STATE_DIR: &str = "__cluster"; +pub const CLUSTER_STATE_FILE: &str = "__cluster/state.json"; +pub const CLUSTER_LOCK_FILE: &str = "__cluster/lock.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DiagnosticSeverity { + Error, + Warning, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Diagnostic { + pub code: String, + pub severity: DiagnosticSeverity, + pub path: String, + pub message: String, +} + +impl Diagnostic { + fn error(code: impl Into, path: impl Into, message: impl Into) -> Self { + Self { + code: code.into(), + severity: DiagnosticSeverity::Error, + path: path.into(), + message: message.into(), + } + } + + fn warning( + code: impl Into, + path: impl Into, + message: impl Into, + ) -> Self { + Self { + code: code.into(), + severity: DiagnosticSeverity::Warning, + path: path.into(), + message: message.into(), + } + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct ResourceSummary { + pub address: String, + pub kind: String, + pub digest: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Dependency { + pub from: String, + pub to: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ValidateOutput { + pub ok: bool, + pub config_dir: String, + pub config_file: String, + pub resource_digests: BTreeMap, + pub resources: Vec, + pub dependencies: Vec, + pub diagnostics: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DesiredRevision { + #[serde(skip_serializing_if = "Option::is_none")] + pub config_digest: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct StateObservations { + pub state_path: String, + pub lock_path: String, + pub state_found: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub applied_config_digest: Option, + pub state_revision: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub state_cas: Option, + pub resource_count: usize, + pub locked: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub lock_id: Option, + pub lock_acquired: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub acquired_lock_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ResourceLifecycleStatus { + Pending, + Planned, + Applying, + Applied, + Drifted, + Blocked, + Error, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ResourceStatusRecord { + pub status: ResourceLifecycleStatus, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PlanOperation { + Create, + Update, + Delete, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct PlanChange { + pub resource: String, + pub operation: PlanOperation, + #[serde(skip_serializing_if = "Option::is_none")] + pub before_digest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub after_digest: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct BlastRadius { + pub resource: String, + pub affected: Vec, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct ApprovalRequirement { + pub resource: String, + pub reason: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct PlanOutput { + pub ok: bool, + pub config_dir: String, + pub desired_revision: DesiredRevision, + pub resource_digests: BTreeMap, + pub dependencies: Vec, + pub state_observations: StateObservations, + pub changes: Vec, + pub blast_radius: Vec, + pub approvals_required: Vec, + pub diagnostics: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct StatusOutput { + pub ok: bool, + pub config_dir: String, + pub state_observations: StateObservations, + pub resource_digests: BTreeMap, + pub resource_statuses: BTreeMap, + pub observations: BTreeMap, + pub diagnostics: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum StateSyncOperation { + Refresh, + Import, +} + +#[derive(Debug, Clone, Serialize)] +pub struct StateSyncOutput { + pub ok: bool, + pub operation: StateSyncOperation, + pub config_dir: String, + pub state_observations: StateObservations, + pub resource_digests: BTreeMap, + pub resource_statuses: BTreeMap, + pub observations: BTreeMap, + pub diagnostics: Vec, +} + +#[derive(Debug, Clone)] +struct DesiredCluster { + config_dir: PathBuf, + config_digest: String, + state_lock: bool, + graphs: Vec, + resource_digests: BTreeMap, + resources: Vec, + dependencies: Vec, +} + +#[derive(Debug, Clone)] +struct DesiredGraph { + id: String, + schema_digest: String, +} + +#[derive(Debug)] +struct ParsedConfig { + raw: Option, + diagnostics: Vec, + config_dir: PathBuf, + config_file: PathBuf, +} + +#[derive(Debug, Clone, Copy)] +struct ClusterSettings { + state_lock: bool, +} + +#[derive(Debug)] +struct LoadOutcome { + desired: Option, + diagnostics: Vec, + config_dir: PathBuf, + config_file: PathBuf, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawClusterConfig { + version: u32, + #[serde(default)] + metadata: Metadata, + #[serde(default)] + state: StateConfig, + #[serde(default)] + graphs: BTreeMap, + #[serde(default)] + policies: BTreeMap, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Metadata { + name: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StateConfig { + backend: Option, + lock: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct GraphConfig { + schema: PathBuf, + #[serde(default)] + queries: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct QueryConfig { + file: PathBuf, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PolicyConfig { + file: PathBuf, + applies_to: Vec, +} + +// Stage 2A/2B accept these forward-compatible state sections so existing +// ledgers won't churn while approval/recovery semantics are staged later. +#[allow(dead_code)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ClusterState { + version: u32, + #[serde(default)] + state_revision: u64, + applied_revision: AppliedRevisionState, + #[serde(default)] + resource_statuses: BTreeMap, + #[serde(default)] + approval_records: BTreeMap, + #[serde(default)] + recovery_records: BTreeMap, + #[serde(default)] + observations: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AppliedRevisionState { + #[serde(default)] + config_digest: Option, + #[serde(default)] + resources: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StateResource { + digest: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StateLockFile { + version: u32, + lock_id: String, + operation: String, + created_at: String, + pid: u32, +} + +#[derive(Debug)] +struct LocalStateBackend { + state_dir: PathBuf, + state_path: PathBuf, + lock_path: PathBuf, +} + +#[derive(Debug)] +struct StateSnapshot { + state: Option, + state_cas: Option, +} + +#[derive(Debug)] +struct StateLockGuard { + path: PathBuf, +} + +pub fn validate_config_dir(config_dir: impl AsRef) -> ValidateOutput { + let outcome = load_desired(config_dir.as_ref()); + let (resource_digests, resources, dependencies) = match outcome.desired { + Some(desired) => ( + desired.resource_digests, + desired.resources, + desired.dependencies, + ), + None => (BTreeMap::new(), Vec::new(), Vec::new()), + }; + let ok = !has_errors(&outcome.diagnostics); + + ValidateOutput { + ok, + config_dir: display_path(&outcome.config_dir), + config_file: display_path(&outcome.config_file), + resource_digests, + resources, + dependencies, + diagnostics: outcome.diagnostics, + } +} + +pub fn plan_config_dir(config_dir: impl AsRef) -> PlanOutput { + let outcome = load_desired(config_dir.as_ref()); + let mut diagnostics = outcome.diagnostics; + let backend = LocalStateBackend::new(&outcome.config_dir); + let mut observations = backend.observations(); + + let Some(desired) = outcome.desired else { + return PlanOutput { + ok: false, + config_dir: display_path(&outcome.config_dir), + desired_revision: DesiredRevision { + config_digest: None, + }, + resource_digests: BTreeMap::new(), + dependencies: Vec::new(), + state_observations: observations, + changes: Vec::new(), + blast_radius: Vec::new(), + approvals_required: Vec::new(), + diagnostics, + }; + }; + + if has_errors(&diagnostics) { + return PlanOutput { + ok: false, + config_dir: display_path(&desired.config_dir), + desired_revision: DesiredRevision { + config_digest: Some(desired.config_digest), + }, + resource_digests: desired.resource_digests, + dependencies: desired.dependencies, + state_observations: observations, + changes: Vec::new(), + blast_radius: Vec::new(), + approvals_required: Vec::new(), + diagnostics, + }; + } + + let _lock_guard = if desired.state_lock { + match backend.acquire_lock("plan", &mut observations) { + Ok(guard) => Some(guard), + Err(diagnostic) => { + diagnostics.push(diagnostic); + None + } + } + } else { + diagnostics.push(Diagnostic::warning( + "state_lock_disabled", + "state.lock", + "state.lock is false; plan read state without acquiring the cluster state lock", + )); + None + }; + + let mut prior_resources = BTreeMap::new(); + if !has_errors(&diagnostics) { + match backend.read_state(&mut observations) { + Ok(snapshot) => { + if let Some(state) = snapshot.state { + prior_resources = state_resource_digests(&state); + } + } + Err(diagnostic) => diagnostics.push(diagnostic), + } + } + + let changes = if has_errors(&diagnostics) { + Vec::new() + } else { + diff_resources(&prior_resources, &desired.resource_digests) + }; + let blast_radius = compute_blast_radius(&changes, &desired.dependencies); + let approvals_required = compute_approvals(&changes); + let ok = !has_errors(&diagnostics); + + PlanOutput { + ok, + config_dir: display_path(&desired.config_dir), + desired_revision: DesiredRevision { + config_digest: Some(desired.config_digest), + }, + resource_digests: desired.resource_digests, + dependencies: desired.dependencies, + state_observations: observations, + changes, + blast_radius, + approvals_required, + diagnostics, + } +} + +pub fn status_config_dir(config_dir: impl AsRef) -> StatusOutput { + let parsed = parse_cluster_config(config_dir.as_ref()); + let mut diagnostics = parsed.diagnostics; + let backend = LocalStateBackend::new(&parsed.config_dir); + let mut observations = backend.observations(); + backend.observe_lock(&mut observations, &mut diagnostics); + + let mut resource_digests = BTreeMap::new(); + let mut resource_statuses = BTreeMap::new(); + let mut state_observation_records = BTreeMap::new(); + + if let Some(raw) = parsed.raw.as_ref() { + let _settings = validate_cluster_header(raw, &mut diagnostics); + if !has_errors(&diagnostics) { + match backend.read_state(&mut observations) { + Ok(snapshot) => { + if let Some(state) = snapshot.state { + resource_digests = state_resource_digests(&state); + resource_statuses = state.resource_statuses; + state_observation_records = state.observations; + } else { + diagnostics.push(Diagnostic::warning( + "state_missing", + CLUSTER_STATE_FILE, + "state.json is missing; no applied cluster revision has been recorded", + )); + } + } + Err(diagnostic) => diagnostics.push(diagnostic), + } + } + } + + StatusOutput { + ok: !has_errors(&diagnostics), + config_dir: display_path(&parsed.config_dir), + state_observations: observations, + resource_digests, + resource_statuses, + observations: state_observation_records, + diagnostics, + } +} + +pub async fn refresh_config_dir(config_dir: impl AsRef) -> StateSyncOutput { + sync_config_dir(config_dir.as_ref(), StateSyncOperation::Refresh).await +} + +pub async fn import_config_dir(config_dir: impl AsRef) -> StateSyncOutput { + sync_config_dir(config_dir.as_ref(), StateSyncOperation::Import).await +} + +async fn sync_config_dir(config_dir: &Path, operation: StateSyncOperation) -> StateSyncOutput { + let outcome = load_desired(config_dir); + let mut diagnostics = outcome.diagnostics; + let backend = LocalStateBackend::new(&outcome.config_dir); + let mut observations = backend.observations(); + + let Some(desired) = outcome.desired else { + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&outcome.config_dir), + state_observations: observations, + resource_digests: BTreeMap::new(), + resource_statuses: BTreeMap::new(), + observations: BTreeMap::new(), + diagnostics, + }; + }; + + if has_errors(&diagnostics) { + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests: desired.resource_digests, + resource_statuses: BTreeMap::new(), + observations: BTreeMap::new(), + diagnostics, + }; + } + + let operation_label = state_sync_operation_label(operation); + let _lock_guard = if desired.state_lock { + match backend.acquire_lock(operation_label, &mut observations) { + Ok(guard) => Some(guard), + Err(diagnostic) => { + diagnostics.push(diagnostic); + None + } + } + } else { + diagnostics.push(Diagnostic::warning( + "state_lock_disabled", + "state.lock", + format!( + "state.lock is false; {operation_label} wrote state without acquiring the cluster state lock" + ), + )); + None + }; + + if has_errors(&diagnostics) { + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests: desired.resource_digests, + resource_statuses: BTreeMap::new(), + observations: BTreeMap::new(), + diagnostics, + }; + } + + let snapshot = match backend.read_state(&mut observations) { + Ok(snapshot) => snapshot, + Err(diagnostic) => { + diagnostics.push(diagnostic); + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests: desired.resource_digests, + resource_statuses: BTreeMap::new(), + observations: BTreeMap::new(), + diagnostics, + }; + } + }; + + let expected_cas = snapshot.state_cas; + let mut state = match (operation, snapshot.state) { + (StateSyncOperation::Refresh, Some(state)) => state, + (StateSyncOperation::Refresh, None) => { + diagnostics.push(Diagnostic::error( + "state_missing", + CLUSTER_STATE_FILE, + "refresh requires an existing state.json; run `cluster import` to bootstrap state", + )); + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests: BTreeMap::new(), + resource_statuses: BTreeMap::new(), + observations: BTreeMap::new(), + diagnostics, + }; + } + (StateSyncOperation::Import, Some(state)) => { + diagnostics.push(Diagnostic::error( + "state_already_exists", + CLUSTER_STATE_FILE, + "import creates initial state only when state.json is missing; use `cluster refresh` for an existing state ledger", + )); + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests: state_resource_digests(&state), + resource_statuses: state.resource_statuses, + observations: state.observations, + diagnostics, + }; + } + (StateSyncOperation::Import, None) => initial_import_state(&desired), + }; + + let graph_error_count = observe_declared_graphs(&desired, &mut state).await; + if graph_error_count > 0 { + diagnostics.push(Diagnostic::error( + "graph_observation_error", + CLUSTER_GRAPHS_DIR, + format!("{graph_error_count} graph observation(s) failed"), + )); + } + + if operation == StateSyncOperation::Import && has_errors(&diagnostics) { + return StateSyncOutput { + ok: false, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests: state_resource_digests(&state), + resource_statuses: state.resource_statuses, + observations: state.observations, + diagnostics, + }; + } + + if operation == StateSyncOperation::Import { + state.state_revision = 1; + } else { + state.state_revision = state.state_revision.saturating_add(1); + } + + match backend.write_state(&state, expected_cas.as_deref(), &mut observations) { + Ok(()) => {} + Err(diagnostic) => diagnostics.push(diagnostic), + } + + let resource_digests = state_resource_digests(&state); + let ok = !has_errors(&diagnostics); + + StateSyncOutput { + ok, + operation, + config_dir: display_path(&desired.config_dir), + state_observations: observations, + resource_digests, + resource_statuses: state.resource_statuses, + observations: state.observations, + diagnostics, + } +} + +fn parse_cluster_config(config_dir: &Path) -> ParsedConfig { + let config_dir = config_dir.to_path_buf(); + let config_file = config_dir.join(CLUSTER_CONFIG_FILE); + let mut diagnostics = Vec::new(); + + if !config_dir.is_dir() { + diagnostics.push(Diagnostic::error( + "config_dir_not_found", + display_path(&config_dir), + "`--config` must point at a directory containing cluster.yaml", + )); + return ParsedConfig { + raw: None, + diagnostics, + config_dir, + config_file, + }; + } + + let text = match fs::read_to_string(&config_file) { + Ok(text) => text, + Err(err) => { + diagnostics.push(Diagnostic::error( + "cluster_config_read_error", + CLUSTER_CONFIG_FILE, + format!("could not read cluster.yaml: {err}"), + )); + return ParsedConfig { + raw: None, + diagnostics, + config_dir, + config_file, + }; + } + }; + + diagnostics.extend(duplicate_key_diagnostics(&text)); + diagnostics.extend(future_field_diagnostics(&text)); + if has_errors(&diagnostics) { + return ParsedConfig { + raw: None, + diagnostics, + config_dir, + config_file, + }; + } + + let raw = match serde_yaml::from_str::(&text) { + Ok(raw) => Some(raw), + Err(err) => { + diagnostics.push(Diagnostic::error( + "invalid_cluster_yaml", + CLUSTER_CONFIG_FILE, + format!("could not parse cluster.yaml: {err}"), + )); + None + } + }; + + ParsedConfig { + raw, + diagnostics, + config_dir, + config_file, + } +} + +fn validate_cluster_header( + raw: &RawClusterConfig, + diagnostics: &mut Vec, +) -> ClusterSettings { + if raw.version != 1 { + diagnostics.push(Diagnostic::error( + "unsupported_cluster_config_version", + "version", + format!( + "unsupported cluster config version {}; this build supports version 1", + raw.version + ), + )); + } + if let Some(name) = raw.metadata.name.as_deref() { + if name.trim().is_empty() { + diagnostics.push(Diagnostic::error( + "empty_metadata_name", + "metadata.name", + "metadata.name must not be empty when provided", + )); + } + } + if let Some(backend) = raw.state.backend.as_deref() { + if backend != "cluster" { + diagnostics.push(Diagnostic::error( + "unsupported_state_backend", + "state.backend", + "Stage 2B supports only omitted state.backend or `cluster`", + )); + } + } + + ClusterSettings { + state_lock: raw.state.lock.unwrap_or(true), + } +} + +impl LocalStateBackend { + fn new(config_dir: &Path) -> Self { + let state_dir = config_dir.join(CLUSTER_STATE_DIR); + Self { + state_path: config_dir.join(CLUSTER_STATE_FILE), + lock_path: config_dir.join(CLUSTER_LOCK_FILE), + state_dir, + } + } + + fn observations(&self) -> StateObservations { + StateObservations { + state_path: display_path(&self.state_path), + lock_path: display_path(&self.lock_path), + state_found: false, + applied_config_digest: None, + state_revision: 0, + state_cas: None, + resource_count: 0, + locked: false, + lock_id: None, + lock_acquired: false, + acquired_lock_id: None, + } + } + + fn read_state( + &self, + observations: &mut StateObservations, + ) -> Result { + let text = match fs::read_to_string(&self.state_path) { + Ok(text) => text, + Err(err) if err.kind() == ErrorKind::NotFound => { + return Ok(StateSnapshot { + state: None, + state_cas: None, + }); + } + Err(err) => { + return Err(Diagnostic::error( + "state_read_error", + CLUSTER_STATE_FILE, + format!("could not read state file: {err}"), + )); + } + }; + + observations.state_found = true; + let state_cas = format!("sha256:{}", sha256_hex(text.as_bytes())); + observations.state_cas = Some(state_cas.clone()); + + let state = serde_json::from_str::(&text).map_err(|err| { + Diagnostic::error( + "invalid_state_json", + CLUSTER_STATE_FILE, + format!("could not parse state JSON: {err}"), + ) + })?; + + if state.version != 1 { + return Err(Diagnostic::error( + "unsupported_state_version", + "state.version", + format!( + "unsupported cluster state version {}; this build supports version 1", + state.version + ), + )); + } + + observations.applied_config_digest = state.applied_revision.config_digest.clone(); + observations.state_revision = state.state_revision; + observations.resource_count = state.applied_revision.resources.len(); + + Ok(StateSnapshot { + state: Some(state), + state_cas: Some(state_cas), + }) + } + + fn write_state( + &self, + state: &ClusterState, + expected_cas: Option<&str>, + observations: &mut StateObservations, + ) -> Result<(), Diagnostic> { + fs::create_dir_all(&self.state_dir).map_err(|err| { + Diagnostic::error( + "state_write_error", + CLUSTER_STATE_DIR, + format!("could not create cluster state directory: {err}"), + ) + })?; + + let current_cas = self.current_state_cas()?; + if current_cas.as_deref() != expected_cas { + return Err(Diagnostic::error( + "state_cas_mismatch", + CLUSTER_STATE_FILE, + "state.json changed while the command was running; re-run the command against the latest state", + )); + } + + let mut payload = serde_json::to_string_pretty(state).map_err(|err| { + Diagnostic::error( + "state_write_error", + CLUSTER_STATE_FILE, + format!("could not encode state JSON: {err}"), + ) + })?; + payload.push('\n'); + + let tmp_path = self + .state_dir + .join(format!("state.json.tmp.{}", Ulid::new())); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path) + .map_err(|err| { + Diagnostic::error( + "state_write_error", + display_path(&tmp_path), + format!("could not create temporary state file: {err}"), + ) + })?; + file.write_all(payload.as_bytes()).map_err(|err| { + Diagnostic::error( + "state_write_error", + display_path(&tmp_path), + format!("could not write temporary state file: {err}"), + ) + })?; + file.sync_all().map_err(|err| { + Diagnostic::error( + "state_write_error", + display_path(&tmp_path), + format!("could not sync temporary state file: {err}"), + ) + })?; + drop(file); + + if let Err(err) = fs::rename(&tmp_path, &self.state_path) { + let _ = fs::remove_file(&tmp_path); + return Err(Diagnostic::error( + "state_write_error", + CLUSTER_STATE_FILE, + format!("could not replace state.json atomically: {err}"), + )); + } + + let written = fs::read_to_string(&self.state_path).map_err(|err| { + Diagnostic::error( + "state_write_error", + CLUSTER_STATE_FILE, + format!("could not read state.json after write: {err}"), + ) + })?; + observations.state_found = true; + observations.applied_config_digest = state.applied_revision.config_digest.clone(); + observations.state_revision = state.state_revision; + observations.state_cas = Some(format!("sha256:{}", sha256_hex(written.as_bytes()))); + observations.resource_count = state.applied_revision.resources.len(); + + Ok(()) + } + + fn current_state_cas(&self) -> Result, Diagnostic> { + match fs::read(&self.state_path) { + Ok(bytes) => Ok(Some(format!("sha256:{}", sha256_hex(&bytes)))), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(Diagnostic::error( + "state_read_error", + CLUSTER_STATE_FILE, + format!("could not read state file for CAS check: {err}"), + )), + } + } + + fn acquire_lock( + &self, + operation: &str, + observations: &mut StateObservations, + ) -> Result { + fs::create_dir_all(&self.state_dir).map_err(|err| { + Diagnostic::error( + "state_lock_error", + CLUSTER_STATE_DIR, + format!("could not create cluster state directory: {err}"), + ) + })?; + + let lock_id = Ulid::new().to_string(); + let lock = StateLockFile { + version: 1, + lock_id: lock_id.clone(), + operation: operation.to_string(), + created_at: OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string()), + pid: process::id(), + }; + let payload = serde_json::to_string_pretty(&lock).map_err(|err| { + Diagnostic::error( + "state_lock_error", + CLUSTER_LOCK_FILE, + format!("could not encode state lock: {err}"), + ) + })?; + + match OpenOptions::new() + .write(true) + .create_new(true) + .open(&self.lock_path) + { + Ok(mut file) => { + if let Err(err) = file.write_all(payload.as_bytes()) { + // No guard exists yet, so clean up the create-new file here + // instead of leaving a stale partial lock for the next run. + drop(file); + let _ = fs::remove_file(&self.lock_path); + return Err(Diagnostic::error( + "state_lock_error", + CLUSTER_LOCK_FILE, + format!("could not write state lock: {err}"), + )); + } + observations.lock_acquired = true; + observations.acquired_lock_id = Some(lock_id.clone()); + Ok(StateLockGuard { + path: self.lock_path.clone(), + }) + } + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + self.observe_lock_id(observations); + Err(Diagnostic::error( + "state_lock_held", + CLUSTER_LOCK_FILE, + "cluster state lock already exists; remove it only after confirming no cluster operation is active", + )) + } + Err(err) => Err(Diagnostic::error( + "state_lock_error", + CLUSTER_LOCK_FILE, + format!("could not acquire state lock: {err}"), + )), + } + } + + fn observe_lock( + &self, + observations: &mut StateObservations, + diagnostics: &mut Vec, + ) { + if self.lock_path.exists() { + observations.locked = true; + match fs::read_to_string(&self.lock_path) { + Ok(text) => match serde_json::from_str::(&text) { + Ok(lock) if lock.version == 1 => { + observations.lock_id = Some(lock.lock_id); + } + Ok(lock) => diagnostics.push(Diagnostic::warning( + "unsupported_state_lock_version", + CLUSTER_LOCK_FILE, + format!("unsupported cluster state lock version {}", lock.version), + )), + Err(err) => diagnostics.push(Diagnostic::warning( + "invalid_state_lock", + CLUSTER_LOCK_FILE, + format!("could not parse state lock: {err}"), + )), + }, + Err(err) => diagnostics.push(Diagnostic::warning( + "state_lock_read_error", + CLUSTER_LOCK_FILE, + format!("could not read state lock: {err}"), + )), + } + } + } + + fn observe_lock_id(&self, observations: &mut StateObservations) { + observations.locked = true; + if let Ok(text) = fs::read_to_string(&self.lock_path) { + if let Ok(lock) = serde_json::from_str::(&text) { + if lock.version == 1 { + observations.lock_id = Some(lock.lock_id); + } + } + } + } +} + +impl Drop for StateLockGuard { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +fn state_resource_digests(state: &ClusterState) -> BTreeMap { + state + .applied_revision + .resources + .iter() + .map(|(address, resource)| (address.clone(), resource.digest.clone())) + .collect() +} + +fn initial_import_state(desired: &DesiredCluster) -> ClusterState { + ClusterState { + version: 1, + state_revision: 0, + applied_revision: AppliedRevisionState { + config_digest: Some(desired.config_digest.clone()), + resources: BTreeMap::new(), + }, + resource_statuses: BTreeMap::new(), + approval_records: BTreeMap::new(), + recovery_records: BTreeMap::new(), + observations: BTreeMap::new(), + } +} + +async fn observe_declared_graphs(desired: &DesiredCluster, state: &mut ClusterState) -> usize { + let mut graph_error_count = 0; + for graph in &desired.graphs { + let graph_address = graph_address(&graph.id); + let schema_address = schema_address(&graph.id); + let graph_path = desired + .config_dir + .join(CLUSTER_GRAPHS_DIR) + .join(format!("{}.omni", graph.id)); + let graph_uri = display_path(&graph_path); + let observed_at = now_rfc3339(); + + if !graph_path.exists() { + state.applied_revision.resources.remove(&graph_address); + state.applied_revision.resources.remove(&schema_address); + state.observations.insert( + graph_address.clone(), + graph_observation_json(GraphObservationJson { + address: &graph_address, + graph_uri: &graph_uri, + observed_at: &observed_at, + exists: false, + manifest_version: None, + schema_digest: None, + desired_schema_digest: &graph.schema_digest, + schema_matches_desired: Some(false), + error: Some("derived graph root is missing"), + }), + ); + set_resource_status( + state, + &graph_address, + ResourceLifecycleStatus::Drifted, + "graph_missing", + "derived graph root is missing", + ); + set_resource_status( + state, + &schema_address, + ResourceLifecycleStatus::Drifted, + "graph_missing", + "derived graph root is missing", + ); + continue; + } + + match observe_live_graph(&graph_uri).await { + Ok(observation) => { + let schema_matches = observation.schema_digest == graph.schema_digest; + state.applied_revision.resources.insert( + schema_address.clone(), + StateResource { + digest: observation.schema_digest.clone(), + }, + ); + let query_digests = state_query_digests_for_graph(state, &graph.id); + let graph_digest_value = graph_digest( + &graph.id, + Some(&observation.schema_digest), + Some(&query_digests), + ); + state.applied_revision.resources.insert( + graph_address.clone(), + StateResource { + digest: graph_digest_value, + }, + ); + state.observations.insert( + graph_address.clone(), + graph_observation_json(GraphObservationJson { + address: &graph_address, + graph_uri: &graph_uri, + observed_at: &observed_at, + exists: true, + manifest_version: Some(observation.manifest_version), + schema_digest: Some(observation.schema_digest.as_str()), + desired_schema_digest: &graph.schema_digest, + schema_matches_desired: Some(schema_matches), + error: None, + }), + ); + if schema_matches { + set_resource_status_applied(state, &graph_address); + set_resource_status_applied(state, &schema_address); + } else { + set_resource_status( + state, + &graph_address, + ResourceLifecycleStatus::Drifted, + "schema_mismatch", + "live schema digest differs from desired schema digest", + ); + set_resource_status( + state, + &schema_address, + ResourceLifecycleStatus::Drifted, + "schema_mismatch", + "live schema digest differs from desired schema digest", + ); + } + } + Err(error) => { + graph_error_count += 1; + state.observations.insert( + graph_address.clone(), + graph_observation_json(GraphObservationJson { + address: &graph_address, + graph_uri: &graph_uri, + observed_at: &observed_at, + exists: true, + manifest_version: None, + schema_digest: None, + desired_schema_digest: &graph.schema_digest, + schema_matches_desired: None, + error: Some(error.as_str()), + }), + ); + set_resource_status( + state, + &graph_address, + ResourceLifecycleStatus::Error, + "graph_observation_error", + error.as_str(), + ); + set_resource_status( + state, + &schema_address, + ResourceLifecycleStatus::Error, + "graph_observation_error", + error.as_str(), + ); + } + } + } + graph_error_count +} + +struct LiveGraphObservation { + manifest_version: u64, + schema_digest: String, +} + +async fn observe_live_graph(graph_uri: &str) -> Result { + let db = Omnigraph::open_read_only(graph_uri) + .await + .map_err(|err| err.to_string())?; + let snapshot = db + .snapshot_of(ReadTarget::branch("main")) + .await + .map_err(|err| err.to_string())?; + let schema_source = db.schema_source(); + Ok(LiveGraphObservation { + manifest_version: snapshot.version(), + schema_digest: sha256_hex(schema_source.as_bytes()), + }) +} + +struct GraphObservationJson<'a> { + address: &'a str, + graph_uri: &'a str, + observed_at: &'a str, + exists: bool, + manifest_version: Option, + schema_digest: Option<&'a str>, + desired_schema_digest: &'a str, + schema_matches_desired: Option, + error: Option<&'a str>, +} + +fn graph_observation_json(observation: GraphObservationJson<'_>) -> serde_json::Value { + json!({ + "kind": "graph", + "address": observation.address, + "graph_uri": observation.graph_uri, + "observed_at": observation.observed_at, + "exists": observation.exists, + "manifest_version": observation.manifest_version, + "schema_digest": observation.schema_digest, + "desired_schema_digest": observation.desired_schema_digest, + "schema_matches_desired": observation.schema_matches_desired, + "error": observation.error, + }) +} + +fn state_query_digests_for_graph(state: &ClusterState, graph_id: &str) -> BTreeMap { + let prefix = format!("query.{graph_id}."); + state + .applied_revision + .resources + .iter() + .filter_map(|(address, resource)| { + address + .strip_prefix(&prefix) + .map(|name| (name.to_string(), resource.digest.clone())) + }) + .collect() +} + +fn set_resource_status_applied(state: &mut ClusterState, address: &str) { + state.resource_statuses.insert( + address.to_string(), + ResourceStatusRecord { + status: ResourceLifecycleStatus::Applied, + conditions: Vec::new(), + message: None, + }, + ); +} + +fn set_resource_status( + state: &mut ClusterState, + address: &str, + status: ResourceLifecycleStatus, + condition: &str, + message: &str, +) { + state.resource_statuses.insert( + address.to_string(), + ResourceStatusRecord { + status, + conditions: vec![condition.to_string()], + message: Some(message.to_string()), + }, + ); +} + +fn load_desired(config_dir: &Path) -> LoadOutcome { + let parsed = parse_cluster_config(config_dir); + let config_dir = parsed.config_dir; + let config_file = parsed.config_file; + let mut diagnostics = parsed.diagnostics; + let Some(raw) = parsed.raw else { + return LoadOutcome { + desired: None, + diagnostics, + config_dir, + config_file, + }; + }; + let settings = validate_cluster_header(&raw, &mut diagnostics); + + let mut resources = BTreeMap::new(); + let mut dependencies = BTreeSet::new(); + let mut graph_query_digests: BTreeMap> = BTreeMap::new(); + let mut graph_schema_digests: BTreeMap = BTreeMap::new(); + + for (graph_id, graph) in &raw.graphs { + validate_id( + "graph id", + &format!("graphs.{graph_id}"), + graph_id, + &mut diagnostics, + ); + let graph_address = graph_address(graph_id); + let schema_address = schema_address(graph_id); + dependencies.insert(Dependency { + from: schema_address.clone(), + to: graph_address.clone(), + }); + + let schema_path = resolve_config_path(&config_dir, &graph.schema); + let schema_source = match fs::read_to_string(&schema_path) { + Ok(source) => { + let digest = sha256_hex(source.as_bytes()); + graph_schema_digests.insert(graph_id.clone(), digest.clone()); + resources.insert( + schema_address.clone(), + ResourceSummary { + address: schema_address.clone(), + kind: "schema".to_string(), + digest, + path: Some(display_path(&schema_path)), + }, + ); + Some(source) + } + Err(err) => { + diagnostics.push(Diagnostic::error( + "schema_file_missing", + format!("graphs.{graph_id}.schema"), + format!( + "could not read schema file '{}': {err}", + schema_path.display() + ), + )); + None + } + }; + + let catalog = schema_source.and_then(|source| match parse_schema(&source) { + Ok(schema) => match build_catalog(&schema) { + Ok(catalog) => Some(catalog), + Err(err) => { + diagnostics.push(Diagnostic::error( + "schema_catalog_error", + format!("graphs.{graph_id}.schema"), + err.to_string(), + )); + None + } + }, + Err(err) => { + diagnostics.push(Diagnostic::error( + "schema_parse_error", + format!("graphs.{graph_id}.schema"), + err.to_string(), + )); + None + } + }); + + for (query_name, query) in &graph.queries { + validate_id( + "query name", + &format!("graphs.{graph_id}.queries.{query_name}"), + query_name, + &mut diagnostics, + ); + let query_address = query_address(graph_id, query_name); + dependencies.insert(Dependency { + from: query_address.clone(), + to: graph_address.clone(), + }); + dependencies.insert(Dependency { + from: query_address.clone(), + to: schema_address.clone(), + }); + + let query_path = resolve_config_path(&config_dir, &query.file); + match fs::read_to_string(&query_path) { + Ok(source) => { + let digest = sha256_hex(source.as_bytes()); + graph_query_digests + .entry(graph_id.clone()) + .or_default() + .insert(query_name.clone(), digest.clone()); + resources.insert( + query_address.clone(), + ResourceSummary { + address: query_address, + kind: "query".to_string(), + digest, + path: Some(display_path(&query_path)), + }, + ); + validate_query_source( + graph_id, + query_name, + &source, + catalog.as_ref(), + &mut diagnostics, + ); + } + Err(err) => diagnostics.push(Diagnostic::error( + "query_file_missing", + format!("graphs.{graph_id}.queries.{query_name}.file"), + format!( + "could not read query file '{}': {err}", + query_path.display() + ), + )), + } + } + } + + for graph_id in raw.graphs.keys() { + let digest = graph_digest( + graph_id, + graph_schema_digests.get(graph_id), + graph_query_digests.get(graph_id), + ); + resources.insert( + graph_address(graph_id), + ResourceSummary { + address: graph_address(graph_id), + kind: "graph".to_string(), + digest, + path: None, + }, + ); + } + + for (policy_name, policy) in &raw.policies { + validate_id( + "policy name", + &format!("policies.{policy_name}"), + policy_name, + &mut diagnostics, + ); + if policy.applies_to.is_empty() { + diagnostics.push(Diagnostic::error( + "policy_missing_applies_to", + format!("policies.{policy_name}.applies_to"), + "policy.applies_to must name `cluster` or at least one graph", + )); + } + + let policy_address = policy_address(policy_name); + for (idx, target) in policy.applies_to.iter().enumerate() { + match normalize_policy_target(target) { + PolicyTarget::Cluster => {} + PolicyTarget::Graph(graph_id) => { + if raw.graphs.contains_key(&graph_id) { + dependencies.insert(Dependency { + from: policy_address.clone(), + to: graph_address(&graph_id), + }); + } else { + diagnostics.push(Diagnostic::error( + "dangling_graph_reference", + format!("policies.{policy_name}.applies_to[{idx}]"), + format!( + "policy references graph `{graph_id}`, but no graph with that id is declared" + ), + )); + } + } + PolicyTarget::WrongKind(kind) => diagnostics.push(Diagnostic::error( + "wrong_kind_reference", + format!("policies.{policy_name}.applies_to[{idx}]"), + format!("policy applies_to expects graph refs or `cluster`, got `{kind}`"), + )), + } + } + + let policy_path = resolve_config_path(&config_dir, &policy.file); + match fs::read(&policy_path) { + Ok(bytes) => { + resources.insert( + policy_address.clone(), + ResourceSummary { + address: policy_address, + kind: "policy".to_string(), + digest: sha256_hex(&bytes), + path: Some(display_path(&policy_path)), + }, + ); + } + Err(err) => diagnostics.push(Diagnostic::error( + "policy_file_missing", + format!("policies.{policy_name}.file"), + format!( + "could not read policy file '{}': {err}", + policy_path.display() + ), + )), + } + } + + let mut resource_digests = BTreeMap::new(); + let mut resource_list = Vec::new(); + for (address, resource) in resources { + resource_digests.insert(address, resource.digest.clone()); + resource_list.push(resource); + } + let dependencies: Vec<_> = dependencies.into_iter().collect(); + let graphs = raw + .graphs + .keys() + .map(|graph_id| DesiredGraph { + id: graph_id.clone(), + schema_digest: graph_schema_digests + .get(graph_id) + .cloned() + .unwrap_or_default(), + }) + .collect(); + let config_digest = desired_config_digest(&raw, &resource_digests); + + LoadOutcome { + desired: Some(DesiredCluster { + config_dir: config_dir.clone(), + config_digest, + state_lock: settings.state_lock, + graphs, + resource_digests, + resources: resource_list, + dependencies, + }), + diagnostics, + config_dir, + config_file, + } +} + +fn validate_query_source( + graph_id: &str, + query_name: &str, + source: &str, + catalog: Option<&omnigraph_compiler::catalog::Catalog>, + diagnostics: &mut Vec, +) { + let path = format!("graphs.{graph_id}.queries.{query_name}"); + match parse_query(source) { + Ok(query_file) => { + let Some(query_decl) = query_file.queries.iter().find(|q| q.name == query_name) else { + diagnostics.push(Diagnostic::error( + "query_key_mismatch", + path, + format!("no `query {query_name}` declaration found in the referenced .gq file"), + )); + return; + }; + if let Some(catalog) = catalog { + if let Err(err) = typecheck_query_decl(catalog, query_decl) { + diagnostics.push(Diagnostic::error( + "query_typecheck_error", + format!("graphs.{graph_id}.queries.{query_name}"), + err.to_string(), + )); + } + } else { + diagnostics.push(Diagnostic::warning( + "query_typecheck_skipped", + format!("graphs.{graph_id}.queries.{query_name}"), + "query parsed, but type-check was skipped because the graph schema is invalid", + )); + } + } + Err(err) => diagnostics.push(Diagnostic::error( + "query_parse_error", + path, + err.to_string(), + )), + } +} + +fn diff_resources( + prior: &BTreeMap, + desired: &BTreeMap, +) -> Vec { + let mut changes = Vec::new(); + for (address, after) in desired { + match prior.get(address) { + None => changes.push(PlanChange { + resource: address.clone(), + operation: PlanOperation::Create, + before_digest: None, + after_digest: Some(after.clone()), + }), + Some(before) if before != after => changes.push(PlanChange { + resource: address.clone(), + operation: PlanOperation::Update, + before_digest: Some(before.clone()), + after_digest: Some(after.clone()), + }), + Some(_) => {} + } + } + for (address, before) in prior { + if !desired.contains_key(address) { + changes.push(PlanChange { + resource: address.clone(), + operation: PlanOperation::Delete, + before_digest: Some(before.clone()), + after_digest: None, + }); + } + } + changes.sort_by(|a, b| a.resource.cmp(&b.resource)); + changes +} + +fn compute_blast_radius(changes: &[PlanChange], dependencies: &[Dependency]) -> Vec { + changes + .iter() + .filter_map(|change| { + let affected: Vec<_> = dependencies + .iter() + .filter_map(|dep| (dep.to == change.resource).then_some(dep.from.clone())) + .collect(); + (!affected.is_empty()).then(|| BlastRadius { + resource: change.resource.clone(), + affected, + }) + }) + .collect() +} + +fn compute_approvals(changes: &[PlanChange]) -> Vec { + changes + .iter() + .filter_map(|change| { + if change.operation == PlanOperation::Delete + && (change.resource.starts_with("graph.") || change.resource.starts_with("schema.")) + { + Some(ApprovalRequirement { + resource: change.resource.clone(), + reason: "delete may remove deployed graph or schema definition".to_string(), + }) + } else { + None + } + }) + .collect() +} + +fn duplicate_key_diagnostics(text: &str) -> Vec { + #[derive(Debug)] + struct Frame { + indent: isize, + path: String, + keys: BTreeSet, + } + + let mut diagnostics = Vec::new(); + let mut stack = vec![Frame { + indent: -1, + path: String::new(), + keys: BTreeSet::new(), + }]; + + for (line_idx, line) in text.lines().enumerate() { + let line_without_comment = strip_comment(line); + if line_without_comment.trim().is_empty() { + continue; + } + let indent = line_without_comment + .chars() + .take_while(|ch| *ch == ' ') + .count() as isize; + let trimmed = line_without_comment.trim_start(); + if trimmed.starts_with('-') { + continue; + } + let Some((raw_key, raw_value)) = trimmed.split_once(':') else { + continue; + }; + let key = raw_key.trim(); + if key.is_empty() || key.starts_with('{') || key.starts_with('[') { + continue; + } + + while stack.last().is_some_and(|frame| indent <= frame.indent) { + stack.pop(); + } + let parent = stack.last_mut().expect("root frame is always present"); + let full_path = if parent.path.is_empty() { + key.to_string() + } else { + format!("{}.{}", parent.path, key) + }; + if !parent.keys.insert(key.to_string()) { + diagnostics.push(Diagnostic::error( + "duplicate_yaml_key", + full_path.clone(), + format!("duplicate YAML key `{key}` on line {}", line_idx + 1), + )); + } + if raw_value.trim().is_empty() { + stack.push(Frame { + indent, + path: full_path, + keys: BTreeSet::new(), + }); + } + } + + diagnostics +} + +fn future_field_diagnostics(text: &str) -> Vec { + let Ok(value) = serde_yaml::from_str::(text) else { + return Vec::new(); + }; + let Some(mapping) = value.as_mapping() else { + return Vec::new(); + }; + let future_fields = [ + "apply", + "env_file", + "providers", + "pipelines", + "embeddings", + "ui", + "aliases", + "bindings", + ]; + mapping + .keys() + .filter_map(|key| key.as_str()) + .filter(|key| future_fields.contains(key)) + .map(|key| { + Diagnostic::error( + "future_phase_field", + key, + format!("`{key}` is reserved for a later cluster-control phase"), + ) + }) + .collect() +} + +fn strip_comment(line: &str) -> String { + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut escaped = false; + + for (idx, ch) in line.char_indices() { + if escaped { + escaped = false; + continue; + } + match ch { + '\\' if in_double_quote => escaped = true, + '\'' if !in_double_quote => in_single_quote = !in_single_quote, + '"' if !in_single_quote => in_double_quote = !in_double_quote, + '#' if !in_single_quote && !in_double_quote => return line[..idx].to_string(), + _ => {} + } + } + + line.to_string() +} + +fn validate_id(kind: &str, path: &str, value: &str, diagnostics: &mut Vec) { + let mut chars = value.chars(); + let valid = chars + .next() + .is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_') + && chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if !valid { + diagnostics.push(Diagnostic::error( + "invalid_resource_id", + path, + format!("{kind} `{value}` must start with a letter or `_` and contain only ASCII letters, digits, `_`, or `-`"), + )); + } +} + +enum PolicyTarget { + Cluster, + Graph(String), + WrongKind(String), +} + +fn normalize_policy_target(value: &str) -> PolicyTarget { + if value == "cluster" { + PolicyTarget::Cluster + } else if let Some(graph_id) = value.strip_prefix("graph.") { + PolicyTarget::Graph(graph_id.to_string()) + } else if value.contains('.') { + PolicyTarget::WrongKind(value.to_string()) + } else { + PolicyTarget::Graph(value.to_string()) + } +} + +fn graph_address(graph_id: &str) -> String { + format!("graph.{graph_id}") +} + +fn schema_address(graph_id: &str) -> String { + format!("schema.{graph_id}") +} + +fn query_address(graph_id: &str, query_name: &str) -> String { + format!("query.{graph_id}.{query_name}") +} + +fn policy_address(policy_name: &str) -> String { + format!("policy.{policy_name}") +} + +fn resolve_config_path(config_dir: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + config_dir.join(path) + } +} + +fn graph_digest( + graph_id: &str, + schema_digest: Option<&String>, + query_digests: Option<&BTreeMap>, +) -> String { + let mut input = format!( + "graph\0{graph_id}\0schema\0{}\0", + schema_digest.map_or("", String::as_str) + ); + if let Some(query_digests) = query_digests { + for (name, digest) in query_digests { + input.push_str("query\0"); + input.push_str(name); + input.push('\0'); + input.push_str(digest); + input.push('\0'); + } + } + sha256_hex(input.as_bytes()) +} + +fn desired_config_digest( + raw: &RawClusterConfig, + resource_digests: &BTreeMap, +) -> String { + let mut input = String::from("cluster-config\0"); + // Hash parsed semantics, not raw YAML bytes, so comments and formatting do + // not create a new desired revision and the digest cannot drift from parse. + let config_semantics = + serde_json::to_string(raw).expect("raw cluster config must serialize deterministically"); + input.push_str(&config_semantics); + input.push('\0'); + for (address, digest) in resource_digests { + input.push_str(address); + input.push('\0'); + input.push_str(digest); + input.push('\0'); + } + sha256_hex(input.as_bytes()) +} + +fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(digest.len() * 2); + for byte in digest { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} + +fn now_rfc3339() -> String { + OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string()) +} + +fn state_sync_operation_label(operation: StateSyncOperation) -> &'static str { + match operation { + StateSyncOperation::Refresh => "refresh", + StateSyncOperation::Import => "import", + } +} + +fn has_errors(diagnostics: &[Diagnostic]) -> bool { + diagnostics + .iter() + .any(|diagnostic| diagnostic.severity == DiagnosticSeverity::Error) +} + +fn display_path(path: &Path) -> String { + path.display().to_string() +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::Path; + + use omnigraph::db::Omnigraph; + use serde_json::json; + use tempfile::tempdir; + + use super::*; + + const SCHEMA: &str = r#" +node Person { + name: String @key + age: I32? +} +"#; + + const QUERY: &str = r#" +query find_person($name: String) { + match { $p: Person { name: $name } } + return { $p.name, $p.age } +} +"#; + + fn fixture() -> tempfile::TempDir { + let dir = tempdir().unwrap(); + fs::write(dir.path().join("people.pg"), SCHEMA).unwrap(); + fs::write(dir.path().join("people.gq"), QUERY).unwrap(); + fs::write(dir.path().join("base.policy.yaml"), "rules: []\n").unwrap(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +metadata: + name: test +state: + backend: cluster + lock: true +graphs: + knowledge: + schema: ./people.pg + queries: + find_person: + file: ./people.gq +policies: + base: + file: ./base.policy.yaml + applies_to: [knowledge] +"#, + ) + .unwrap(); + dir + } + + async fn init_derived_graph(root: &Path) { + let graph_dir = root.join(CLUSTER_GRAPHS_DIR); + fs::create_dir_all(&graph_dir).unwrap(); + let graph = graph_dir.join("knowledge.omni"); + Omnigraph::init(graph.to_string_lossy().as_ref(), SCHEMA) + .await + .unwrap(); + } + + #[test] + fn valid_minimal_config() { + let dir = fixture(); + let out = validate_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.resource_digests.contains_key("graph.knowledge")); + assert!(out.resource_digests.contains_key("schema.knowledge")); + assert!( + out.dependencies + .iter() + .any(|dep| dep.from == "policy.base" && dep.to == "graph.knowledge") + ); + } + + #[test] + fn unknown_field_rejection() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\ngraphs: {}\nwat: true\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert!(out.diagnostics[0].message.contains("unknown field")); + } + + #[test] + fn future_phase_field_rejection() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\ngraphs: {}\npipelines: {}\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "future_phase_field"); + } + + #[test] + fn duplicate_yaml_key_rejection() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\ngraphs: {}\ngraphs: {}\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "duplicate_yaml_key"); + } + + #[test] + fn duplicate_yaml_key_rejection_keeps_quoted_hashes() { + let diagnostics = + duplicate_key_diagnostics("\"name#display\": one\n\"name#display\": two\n"); + assert_eq!(diagnostics.len(), 1); + assert_eq!(diagnostics[0].code, "duplicate_yaml_key"); + } + + #[test] + fn missing_schema_query_and_policy_files() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +graphs: + knowledge: + schema: ./missing.pg + queries: + find_person: { file: ./missing.gq } +policies: + base: + file: ./missing.policy.yaml + applies_to: [knowledge] +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); + assert!(codes.contains("schema_file_missing")); + assert!(codes.contains("query_file_missing")); + assert!(codes.contains("policy_file_missing")); + } + + #[test] + fn wrong_kind_and_dangling_refs_fail() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +graphs: + knowledge: + schema: ./people.pg +policies: + base: + file: ./base.policy.yaml + applies_to: [query.knowledge.find_person, missing] +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); + assert!(codes.contains("wrong_kind_reference")); + assert!(codes.contains("dangling_graph_reference")); + } + + #[test] + fn query_key_mismatch_fails() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +graphs: + knowledge: + schema: ./people.pg + queries: + different: { file: ./people.gq } +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "query_key_mismatch"); + } + + #[test] + fn query_typecheck_failure_fails() { + let dir = fixture(); + fs::write( + dir.path().join("people.gq"), + "query find_person() { match { $d: DoesNotExist } return { $d.name } }\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "query_typecheck_error") + ); + } + + #[test] + fn missing_state_plans_creates() { + let dir = fixture(); + let out = plan_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.state_found); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_acquired); + assert!( + out.changes + .iter() + .all(|c| c.operation == PlanOperation::Create) + ); + assert!(out.changes.iter().any(|c| c.resource == "graph.knowledge")); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn config_digest_ignores_yaml_comments_and_formatting() { + let dir = fixture(); + let first = plan_config_dir(dir.path()); + assert!(first.ok, "{:?}", first.diagnostics); + + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +# Same semantic config as the fixture, intentionally rendered differently. +version: 1 +metadata: { name: test } +state: { backend: cluster, lock: true } +graphs: + knowledge: + schema: ./people.pg + queries: { find_person: { file: ./people.gq } } +policies: + base: + file: ./base.policy.yaml + applies_to: + - knowledge +"#, + ) + .unwrap(); + + let second = plan_config_dir(dir.path()); + assert!(second.ok, "{:?}", second.diagnostics); + assert_eq!( + first.desired_revision.config_digest, + second.desired_revision.config_digest + ); + } + + #[test] + fn existing_state_plans_update_and_delete_deterministically() { + let dir = fixture(); + let first = plan_config_dir(dir.path()); + let state_dir = dir.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + serde_json::to_string_pretty(&json!({ + "version": 1, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": first.resource_digests["graph.knowledge"] }, + "policy.old": { "digest": "abc" }, + "schema.knowledge": { "digest": "old-schema" } + } + } + })) + .unwrap(), + ) + .unwrap(); + + let out = plan_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + let rendered: Vec<_> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), &change.operation)) + .collect(); + assert_eq!( + rendered, + vec![ + ("policy.base", &PlanOperation::Create), + ("policy.old", &PlanOperation::Delete), + ("query.knowledge.find_person", &PlanOperation::Create), + ("schema.knowledge", &PlanOperation::Update), + ] + ); + } + + #[test] + fn old_minimal_state_json_still_plans_with_default_revision() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{ + "version": 1, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": "old-graph" } + } + } +}"#, + ) + .unwrap(); + + let out = plan_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 0); + assert!(out.state_observations.state_cas.is_some()); + assert!(out.changes.iter().any(|change| { + change.resource == "graph.knowledge" && change.operation == PlanOperation::Update + })); + } + + #[test] + fn extended_state_json_status_surfaces_statuses() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + let state = r#"{ + "version": 1, + "state_revision": 42, + "applied_revision": { + "config_digest": "applied-config", + "resources": { + "graph.knowledge": { "digest": "graph-digest" } + } + }, + "resource_statuses": { + "graph.knowledge": { + "status": "applied", + "conditions": ["healthy"], + "message": "ready" + } + }, + "approval_records": {}, + "recovery_records": {}, + "observations": { + "graph.knowledge": { "manifest_version": 12 } + } +}"#; + fs::write(state_dir.join("state.json"), state).unwrap(); + + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.state_observations.state_found); + assert_eq!(out.state_observations.state_revision, 42); + assert_eq!( + out.state_observations.state_cas.as_deref(), + Some(format!("sha256:{}", sha256_hex(state.as_bytes())).as_str()) + ); + assert_eq!( + out.resource_digests + .get("graph.knowledge") + .map(String::as_str), + Some("graph-digest") + ); + assert_eq!( + out.resource_statuses["graph.knowledge"].status, + ResourceLifecycleStatus::Applied + ); + } + + #[test] + fn missing_state_status_succeeds_with_warning() { + let dir = fixture(); + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.state_found); + assert_eq!(out.state_observations.state_revision, 0); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_missing") + ); + } + + #[test] + fn invalid_state_status_fails() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write(state_dir.join("state.json"), "{").unwrap(); + + let out = status_config_dir(dir.path()); + assert!(!out.ok); + assert!(out.state_observations.state_found); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "invalid_state_json") + ); + } + + #[test] + fn plan_reports_state_cas_revision_and_removes_lock() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + let state = r#"{ + "version": 1, + "state_revision": 7, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": "old-graph" } + } + } +}"#; + fs::write(state_dir.join("state.json"), state).unwrap(); + + let out = plan_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 7); + assert_eq!( + out.state_observations.state_cas.as_deref(), + Some(format!("sha256:{}", sha256_hex(state.as_bytes())).as_str()) + ); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_id.is_none()); + assert!(out.state_observations.lock_acquired); + assert!(out.state_observations.acquired_lock_id.is_some()); + assert!( + !dir.path().join(CLUSTER_LOCK_FILE).exists(), + "plan must release lock before returning" + ); + } + + #[test] + fn existing_lock_makes_plan_fail() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{ + "version": 1, + "lock_id": "held-lock", + "operation": "plan", + "created_at": "2026-06-08T00:00:00Z", + "pid": 123 +}"#, + ) + .unwrap(); + + let out = plan_config_dir(dir.path()); + assert!(!out.ok); + assert!(out.state_observations.locked); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert!(!out.state_observations.lock_acquired); + assert!(out.state_observations.acquired_lock_id.is_none()); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_held") + ); + } + + #[test] + fn state_lock_false_bypasses_lock_with_warning() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: cluster + lock: false +graphs: + knowledge: + schema: ./people.pg +"#, + ) + .unwrap(); + + let out = plan_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.locked); + assert!(!out.state_observations.lock_acquired); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_disabled") + ); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn external_state_backend_rejected() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "unsupported_state_backend"); + } + + #[test] + fn external_state_backend_plan_rejected() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", + ) + .unwrap(); + let out = plan_config_dir(dir.path()); + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "unsupported_state_backend") + ); + } + + #[tokio::test] + async fn import_missing_state_creates_state_with_graph_observation() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + + let out = import_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 1); + assert!(out.state_observations.state_cas.is_some()); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_acquired); + assert!(out.state_observations.acquired_lock_id.is_some()); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + assert_eq!( + out.resource_digests + .get("schema.knowledge") + .map(String::as_str), + Some(sha256_hex(SCHEMA.as_bytes()).as_str()) + ); + assert!(out.observations["graph.knowledge"]["manifest_version"].is_number()); + assert_eq!( + out.observations["graph.knowledge"]["schema_matches_desired"], + true + ); + + let state: serde_json::Value = + serde_json::from_str(&fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap()) + .unwrap(); + assert_eq!(state["state_revision"], 1); + assert_eq!( + state["resource_statuses"]["graph.knowledge"]["status"], + "applied" + ); + } + + #[tokio::test] + async fn import_existing_state_fails() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let out = import_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_already_exists") + ); + } + + #[tokio::test] + async fn refresh_missing_state_fails() { + let dir = fixture(); + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_missing") + ); + } + + #[tokio::test] + async fn refresh_existing_minimal_state_increments_revision_and_updates_cas() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"config_digest":"old","resources":{"graph.knowledge":{"digest":"old"}}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 1); + assert!(out.state_observations.state_cas.is_some()); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_acquired); + assert_eq!( + out.resource_statuses["graph.knowledge"].status, + ResourceLifecycleStatus::Applied + ); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[tokio::test] + async fn refresh_records_live_schema_digest_and_manifest_version() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"state_revision":4,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 5); + assert_eq!( + out.observations["graph.knowledge"]["schema_digest"], + sha256_hex(SCHEMA.as_bytes()) + ); + assert!(out.observations["graph.knowledge"]["manifest_version"].is_u64()); + } + + #[tokio::test] + async fn missing_derived_graph_root_marks_drifted_and_plans_creates() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{"graph.knowledge":{"digest":"old-graph"},"schema.knowledge":{"digest":"old-schema"}}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!( + out.resource_statuses["graph.knowledge"].status, + ResourceLifecycleStatus::Drifted + ); + assert!(!out.resource_digests.contains_key("graph.knowledge")); + assert_eq!(out.observations["graph.knowledge"]["exists"], false); + + let plan = plan_config_dir(dir.path()); + assert!(plan.ok, "{:?}", plan.diagnostics); + assert!(plan.changes.iter().any(|change| { + change.resource == "graph.knowledge" && change.operation == PlanOperation::Create + })); + assert!(plan.changes.iter().any(|change| { + change.resource == "schema.knowledge" && change.operation == PlanOperation::Create + })); + } + + #[tokio::test] + async fn live_schema_mismatch_marks_drifted_and_causes_plan_update() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + fs::write( + dir.path().join("people.pg"), + SCHEMA.replace("age: I32?", "age: I32?\n nickname: String?"), + ) + .unwrap(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{"graph.knowledge":{"digest":"old-graph"},"schema.knowledge":{"digest":"old-schema"}}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!( + out.resource_statuses["schema.knowledge"].status, + ResourceLifecycleStatus::Drifted + ); + assert_eq!( + out.observations["graph.knowledge"]["schema_matches_desired"], + false + ); + + let plan = plan_config_dir(dir.path()); + assert!(plan.ok, "{:?}", plan.diagnostics); + assert!(plan.changes.iter().any(|change| { + change.resource == "schema.knowledge" && change.operation == PlanOperation::Update + })); + } + + #[tokio::test] + async fn existing_lock_makes_refresh_fail() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{"version":1,"lock_id":"held-lock","operation":"refresh","created_at":"2026-06-08T00:00:00Z","pid":123}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!(out.state_observations.locked); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert!(!out.state_observations.lock_acquired); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_held") + ); + } + + #[tokio::test] + async fn state_lock_false_bypasses_refresh_lock_with_warning() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: cluster + lock: false +graphs: + knowledge: + schema: ./people.pg +"#, + ) + .unwrap(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.locked); + assert!(!out.state_observations.lock_acquired); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_disabled") + ); + } + + #[tokio::test] + async fn external_state_backend_refresh_rejected() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "unsupported_state_backend") + ); + } + + #[tokio::test] + async fn import_graph_open_error_does_not_create_state() { + let dir = fixture(); + fs::create_dir_all(dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni")).unwrap(); + + let out = import_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "graph_observation_error") + ); + assert!(!dir.path().join(CLUSTER_STATE_FILE).exists()); + } +} diff --git a/crates/omnigraph-compiler/Cargo.toml b/crates/omnigraph-compiler/Cargo.toml index 545db83..8db46e6 100644 --- a/crates/omnigraph-compiler/Cargo.toml +++ b/crates/omnigraph-compiler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-compiler" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "Schema/query compiler for Omnigraph. Zero Lance dependency." license = "MIT" diff --git a/crates/omnigraph-config/Cargo.toml b/crates/omnigraph-config/Cargo.toml index ccabcfa..39bcc77 100644 --- a/crates/omnigraph-config/Cargo.toml +++ b/crates/omnigraph-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-config" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "Configuration schema and loader for the Omnigraph graph database." license = "MIT" diff --git a/crates/omnigraph-policy/Cargo.toml b/crates/omnigraph-policy/Cargo.toml index 3d14fc5..0df2a12 100644 --- a/crates/omnigraph-policy/Cargo.toml +++ b/crates/omnigraph-policy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-policy" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "Policy / authorization layer for Omnigraph β€” Cedar-backed PolicyEngine, PolicyChecker trait, ResourceScope enum." license = "MIT" diff --git a/crates/omnigraph-queries/Cargo.toml b/crates/omnigraph-queries/Cargo.toml index 6a7a720..cca8411 100644 --- a/crates/omnigraph-queries/Cargo.toml +++ b/crates/omnigraph-queries/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-queries" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "Stored-query registry and validation for the Omnigraph graph database." license = "MIT" @@ -9,8 +9,8 @@ homepage = "https://github.com/ModernRelay/omnigraph" documentation = "https://docs.rs/omnigraph-queries" [dependencies] -omnigraph-config = { path = "../omnigraph-config", version = "0.6.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.1" } +omnigraph-config = { path = "../omnigraph-config", version = "0.6.2" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/omnigraph-server/Cargo.toml b/crates/omnigraph-server/Cargo.toml index 5a66552..08b5067 100644 --- a/crates/omnigraph-server/Cargo.toml +++ b/crates/omnigraph-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-server" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "HTTP server for the Omnigraph graph database." license = "MIT" @@ -19,12 +19,12 @@ default = [] aws = ["dep:aws-config", "dep:aws-sdk-secretsmanager"] [dependencies] -omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.1" } -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.1" } -omnigraph-policy = { path = "../omnigraph-policy", version = "0.6.1" } -omnigraph-config = { path = "../omnigraph-config", version = "0.6.1" } -omnigraph-queries = { path = "../omnigraph-queries", version = "0.6.1" } -omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.6.1" } +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.6.2" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } +omnigraph-policy = { path = "../omnigraph-policy", version = "0.6.2" } +omnigraph-config = { path = "../omnigraph-config", version = "0.6.2" } +omnigraph-queries = { path = "../omnigraph-queries", version = "0.6.2" } +omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.6.2" } axum = { workspace = true } clap = { workspace = true } color-eyre = { workspace = true } diff --git a/crates/omnigraph/Cargo.toml b/crates/omnigraph/Cargo.toml index 70f51d8..9cc2148 100644 --- a/crates/omnigraph/Cargo.toml +++ b/crates/omnigraph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omnigraph-engine" -version = "0.6.1" +version = "0.6.2" edition = "2024" description = "Runtime engine for the Omnigraph graph database." license = "MIT" @@ -16,8 +16,8 @@ default = [] failpoints = ["dep:fail", "fail/failpoints"] [dependencies] -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.1" } -omnigraph-policy = { path = "../omnigraph-policy", version = "0.6.1" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } +omnigraph-policy = { path = "../omnigraph-policy", version = "0.6.2" } lance = { workspace = true } lance-datafusion = { workspace = true } datafusion = { workspace = true } @@ -51,7 +51,8 @@ chrono = { workspace = true } arc-swap = { workspace = true } [dev-dependencies] -omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.1" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.6.2" } tokio = { workspace = true } lance-namespace-impls = { workspace = true } serial_test = "3" +proptest = "1" diff --git a/crates/omnigraph/examples/bench_expand.rs b/crates/omnigraph/examples/bench_expand.rs index c723b24..bb904a0 100644 --- a/crates/omnigraph/examples/bench_expand.rs +++ b/crates/omnigraph/examples/bench_expand.rs @@ -221,6 +221,65 @@ fn microbench_dedup() { ); } +/// Selective single-source traversal, timed cold in CSR vs indexed mode across +/// growing |E|. The win of the indexed path: a small fixed frontier should be +/// ~flat in |E| (one BTREE scan per hop), whereas CSR pays an O(|E|) adjacency +/// build on the first (cold) query. Also asserts both modes return the same +/// rows β€” a guard against the scalar-index `physical_rows` silent fallback +/// dropping unindexed-fragment rows. +async fn bench_selective_modes() { + println!("\n── Selective traversal: indexed vs CSR (cold, single-source knows{{1,2}}) ──"); + let sel = r#" +query sel($name: String) { + match { + $a: Person { name: $name } + $a knows{1,2} $b + } + return { $b.name } +} +"#; + for &(n, avg_deg) in &[(1_000usize, 8usize), (10_000, 8), (30_000, 8)] { + let jsonl = generate_jsonl(n, avg_deg, 42); + let mut params = ParamMap::new(); + params.insert( + "name".to_string(), + omnigraph_compiler::query::ast::Literal::String("p0".to_string()), + ); + + let mut rows_by_mode: Vec<(&str, usize)> = Vec::new(); + for mode in ["csr", "indexed"] { + // Fresh db per measurement so the query is cold (CSR pays its build). + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, SCHEMA).await.unwrap(); + load_jsonl(&mut db, &jsonl, LoadMode::Overwrite).await.unwrap(); + // SAFE: example main drives queries sequentially; no concurrent env reader. + unsafe { std::env::set_var("OMNIGRAPH_TRAVERSAL_MODE", mode) }; + + let t = Instant::now(); + let r = db + .query(ReadTarget::branch("main"), sel, "sel", ¶ms) + .await + .expect("sel query"); + let elapsed = t.elapsed(); + let rows = r.num_rows(); + rows_by_mode.push((mode, rows)); + println!( + " |E|β‰ˆ{:>7} {:<8} cold={:>9.2?} rows={}", + n * avg_deg, + mode, + elapsed, + rows + ); + } + unsafe { std::env::remove_var("OMNIGRAPH_TRAVERSAL_MODE") }; + assert_eq!( + rows_by_mode[0].1, rows_by_mode[1].1, + "indexed and CSR must return identical rows (no silent drop under partial index coverage)" + ); + } +} + #[tokio::main(flavor = "multi_thread")] async fn main() { println!("── End-to-end query latency ──"); @@ -262,5 +321,7 @@ async fn main() { } } + bench_selective_modes().await; + microbench_dedup(); } diff --git a/crates/omnigraph/src/db/manifest.rs b/crates/omnigraph/src/db/manifest.rs index 7fcf7de..5bf1f87 100644 --- a/crates/omnigraph/src/db/manifest.rs +++ b/crates/omnigraph/src/db/manifest.rs @@ -36,7 +36,7 @@ use publisher::{GraphNamespacePublisher, ManifestBatchPublisher}; pub(crate) use recovery::{ RecoveryMode, RecoverySidecar, RecoverySidecarHandle, SidecarKind, SidecarTablePin, SidecarTableRegistration, SidecarTombstone, delete_sidecar, has_schema_apply_sidecar, - new_sidecar, recover_manifest_drift, write_sidecar, + list_sidecars, new_sidecar, recover_manifest_drift, write_sidecar, }; pub use state::SubTableEntry; #[cfg(test)] @@ -48,6 +48,22 @@ const OBJECT_TYPE_TABLE_VERSION: &str = "table_version"; const OBJECT_TYPE_TABLE_TOMBSTONE: &str = "table_tombstone"; const TABLE_VERSION_MANAGEMENT_KEY: &str = "table_version_management"; +/// Apply pending internal-schema migrations against `__manifest` on the +/// open-for-write path, independent of a publish. +/// +/// `Omnigraph::open(ReadWrite)` calls this before the coordinator reads branch +/// state, so branch-observing code (`branch_list`, the schema-apply +/// blocking-branch checks) sees the post-migration graph. In particular the +/// v2β†’v3 step sweeps legacy `__run__*` staging branches off `__manifest` +/// (MR-770); running it here closes the window where those branches would +/// otherwise block schema apply before the first publish runs the migration. +/// +/// Idempotent: a no-op stamp read when the on-disk version already matches. +pub(crate) async fn migrate_on_open(root_uri: &str) -> Result<()> { + let mut dataset = open_manifest_dataset(root_uri, None).await?; + migrations::migrate_internal_schema(&mut dataset).await +} + /// Immutable point-in-time view of the database. /// /// Cheap to create (no storage I/O). All reads within a query go through one diff --git a/crates/omnigraph/src/db/manifest/migrations.rs b/crates/omnigraph/src/db/manifest/migrations.rs index bbb7995..e2801fe 100644 --- a/crates/omnigraph/src/db/manifest/migrations.rs +++ b/crates/omnigraph/src/db/manifest/migrations.rs @@ -46,7 +46,11 @@ use crate::error::{OmniError, Result}; /// - v2 β€” `__manifest.object_id` carries the unenforced-PK annotation, /// engaging Lance's bloom-filter conflict resolver at commit time. Added /// alongside `expected_table_versions` OCC on `ManifestBatchPublisher::publish`. -pub(super) const INTERNAL_MANIFEST_SCHEMA_VERSION: u32 = 2; +/// - v3 β€” one-time sweep of legacy `__run__` staging branches left on the +/// `__manifest` dataset by the pre-v0.4.0 Run state machine (removed in +/// MR-771). Once swept, the `is_internal_run_branch` defense-in-depth guard +/// is no longer needed (MR-770). +pub(super) const INTERNAL_MANIFEST_SCHEMA_VERSION: u32 = 3; const INTERNAL_SCHEMA_VERSION_KEY: &str = "omnigraph:internal_schema_version"; const OBJECT_ID_PK_KEY: &str = "lance-schema:unenforced-primary-key"; @@ -89,6 +93,10 @@ pub(super) async fn migrate_internal_schema(dataset: &mut Dataset) -> Result<()> migrate_v1_to_v2(dataset).await?; current = 2; } + 2 => { + migrate_v2_to_v3(dataset).await?; + current = 3; + } other => { return Err(OmniError::manifest_internal(format!( "no internal-schema migration registered for v{} β†’ v{}", @@ -122,6 +130,51 @@ async fn migrate_v1_to_v2(dataset: &mut Dataset) -> Result<()> { set_stamp(dataset, 2).await } +/// v2 β†’ v3: sweep legacy `__run__` staging branches off the `__manifest` +/// dataset, then bump the stamp. +/// +/// The pre-v0.4.0 Run state machine (removed in MR-771) created graph-level +/// staging branches named `__run__` on `__manifest`. MR-771 stopped +/// creating them but left any pre-existing ones in place; Lance's +/// `list_branches` still enumerates them, so they leak into `branch_list()` +/// and count as blocking branches at schema-apply time. This one-time sweep +/// removes them so the `is_internal_run_branch` guard can retire (MR-770). +/// +/// The `"__run__"` prefix is inlined here on purpose: this migration must keep +/// working after the `run_registry` module (the guard) is deleted, so it does +/// not depend on it. +/// +/// Idempotent under both sequential retry and concurrent runners: each run +/// re-enumerates `list_branches` fresh, and `force_delete_branch` tolerates a +/// branch that is already gone β€” so a crash before the stamp bump, or a second +/// process opening the same legacy graph at the same time, never errors out. +async fn migrate_v2_to_v3(dataset: &mut Dataset) -> Result<()> { + const LEGACY_RUN_BRANCH_PREFIX: &str = "__run__"; + let branches = dataset + .list_branches() + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + let run_branches: Vec = branches + .into_keys() + .filter(|name| { + name.trim_start_matches('/') + .starts_with(LEGACY_RUN_BRANCH_PREFIX) + }) + .collect(); + for name in run_branches { + // `force_delete_branch` deletes even when the `BranchContents` is + // already gone. Plain `delete_branch` errors "BranchContents not + // found", which would fail a second concurrent open (or a retry that + // raced another runner) after the first one swept the branch. Force is + // exactly Lance's documented path for cleaning up zombie branches. + dataset + .force_delete_branch(&name) + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + } + set_stamp(dataset, 3).await +} + async fn set_stamp(dataset: &mut Dataset, version: u32) -> Result<()> { dataset .update_schema_metadata([(INTERNAL_SCHEMA_VERSION_KEY.to_string(), version.to_string())]) diff --git a/crates/omnigraph/src/db/manifest/recovery.rs b/crates/omnigraph/src/db/manifest/recovery.rs index 4c1b987..3119531 100644 --- a/crates/omnigraph/src/db/manifest/recovery.rs +++ b/crates/omnigraph/src/db/manifest/recovery.rs @@ -106,6 +106,12 @@ pub(crate) enum SidecarKind { BranchMerge, /// `ensure_indices_for_branch` β€” index lifecycle commits. EnsureIndices, + /// `optimize_all_tables` β€” Lance `compact_files` (reserve-fragments + + /// rewrite commits) followed by a manifest publish of the compacted + /// version. Loose-match like the other multi-commit writers; roll-forward + /// is always safe because compaction is content-preserving (Lance + /// `Operation::Rewrite` "reorganizes data without semantic modification"). + Optimize, } /// One table's contribution to a sidecar's intended commit. The classifier @@ -412,11 +418,13 @@ pub(crate) fn parse_sidecar(sidecar_uri: &str, body: &str) -> Result manifest_pinned` as `RolledPastExpected` when /// `pin.expected_version == manifest_pinned` (the writer's CAS /// target matches what the manifest currently shows). The risk this @@ -494,9 +502,12 @@ pub(crate) fn decide(classifications: &[TableClassification]) -> SidecarDecision /// Skipping the restore in those cases would leave Lance HEAD ahead of /// the manifest with no recovery artifact left. /// -/// Cost: under repeated mid-rollback crashes (rare), Lance HEAD -/// accumulates extra restore commits that `omnigraph cleanup` reclaims. -/// Bounded by the number of recovery iterations β€” typically 1. +/// Cost: a successful roll-back appends one restore commit and then publishes +/// the manifest to match (`roll_back_sidecar`), so the table converges +/// (`manifest == HEAD`) in one pass. Only repeated crashes *between* the restore +/// and that publish (rare) accumulate extra restore commits; each re-classified +/// roll-back restores again and `omnigraph cleanup` reclaims the surplus. +/// Bounded by the number of interrupted recovery iterations β€” typically 0. pub(crate) async fn restore_table_to_version( table_path: &str, branch: Option<&str>, @@ -801,13 +812,24 @@ async fn roll_back_sidecar( sidecar: &RecoverySidecar, states: &[ClassifiedTable], ) -> Result<()> { - // Restore every table whose Lance HEAD has drifted from the - // manifest pin (RolledPastExpected, UnexpectedAtP1, - // UnexpectedMultistep). NoMovement tables are already at the - // manifest pin β€” no action. Restore is unconditional; repeated - // mid-rollback crashes accumulate a few extra Lance commits that - // `omnigraph cleanup` reclaims. + // Restore every drifted table (RolledPastExpected / UnexpectedAtP1 / + // UnexpectedMultistep) to its manifest-pinned content, then PUBLISH so + // `manifest == Lance HEAD` for each β€” symmetric with roll-forward. The + // restore commit's content equals the manifest-pinned version, so re-pinning + // the manifest to the new (restored) HEAD is content-correct and closes the + // orphaned-drift class (`HEAD > manifest` with no covering sidecar). This is + // what makes a failed-then-retried schema_apply converge: after one + // roll-back `manifest == HEAD`, so the retry's precondition passes instead of + // failing one version higher each iteration. + // + // NoMovement tables are already at the pin β€” excluded from both the restore + // and the publish. The audit `to_version` stays the *logical* rolled-back-to + // version (`manifest_pinned`), while the manifest is published at + // `manifest_pinned + 1` (the restore commit, same content) β€” keep that + // asymmetry so the audit records the drift (`from_version > to_version`). let mut outcomes = Vec::with_capacity(sidecar.tables.len()); + let mut updates: Vec = Vec::with_capacity(sidecar.tables.len()); + let mut expected: HashMap = HashMap::with_capacity(sidecar.tables.len()); for (pin, state) in sidecar.tables.iter().zip(states.iter()) { if matches!( state.classification, @@ -821,10 +843,20 @@ async fn roll_back_sidecar( state.manifest_pinned, ) .await?; - // `from_version` records the Lance HEAD observed BEFORE the - // restore (the actual drift), not the manifest pin. Operators - // reading `_graph_commit_recoveries.lance` see "rolled back - // from v7 to v5" rather than "v5 β†’ v5". + // Publish the post-restore HEAD, CAS against the current (unmoved) + // manifest pin β€” the same helper roll-forward uses. + push_table_update_at_head( + root_uri, + &pin.table_key, + &pin.table_path, + pin.table_branch.as_deref(), + state.manifest_pinned, + &mut updates, + &mut expected, + ) + .await?; + // `from_version` records the Lance HEAD observed BEFORE the restore + // (the actual drift); `to_version` the logical pin we rolled back to. outcomes.push(TableOutcome { table_key: pin.table_key.clone(), from_version: state.lance_head, @@ -832,13 +864,23 @@ async fn roll_back_sidecar( }); } } - // Manifest pin doesn't move on rollback; record an audit-only - // commit at the existing version so operators can correlate via - // `omnigraph commit list --filter actor=omnigraph:recovery`. + // Publish the restored HEADs so manifest == HEAD. A degenerate all-NoMovement + // roll-back restores nothing β€” there's nothing to publish, and the audit + // records the unchanged snapshot version. + let manifest_version = if updates.is_empty() { + snapshot.version() + } else { + let publisher = GraphNamespacePublisher::new(root_uri, sidecar.branch.as_deref()); + publisher + .publish(&updates, &expected) + .await? + .version() + .version + }; record_audit( root_uri, sidecar, - snapshot.version(), + manifest_version, RecoveryKind::RolledBack, outcomes, ) @@ -919,44 +961,20 @@ async fn roll_forward_all( HashMap::with_capacity(sidecar.tables.len() + sidecar.additional_registrations.len()); for pin in &sidecar.tables { - // Open the dataset at its CURRENT Lance HEAD on the pin's branch - // (not at the sidecar's post_commit_pin). For strict-match writers - // (Mutation/Load) HEAD == post_commit_pin by construction. For - // loose-match writers (SchemaApply/EnsureIndices/BranchMerge) HEAD - // may be higher than post_commit_pin (multiple commit_staged - // calls per table); we want to publish to the actual current HEAD. - let head_ds = Dataset::open(&pin.table_path) - .await - .map_err(|e| OmniError::Lance(e.to_string()))?; - let head_ds = match pin.table_branch.as_deref() { - Some(b) if b != "main" => head_ds - .checkout_branch(b) - .await - .map_err(|e| OmniError::Lance(e.to_string()))?, - _ => head_ds, - }; - let head_version = head_ds.version().version; - - let row_count = head_ds - .count_rows(None) - .await - .map_err(|e| OmniError::Lance(e.to_string()))? as u64; - - let table_relative_path = super::table_path_for_table_key(&pin.table_key)?; - let version_metadata = super::metadata::TableVersionMetadata::from_dataset( + // Publish to the table's CURRENT Lance HEAD on the pin's branch (not the + // sidecar's `post_commit_pin`, a lower bound for loose-match writers that + // run multiple commit_staged calls per table). CAS against the pin's + // pre-write `expected_version`. + let head_version = push_table_update_at_head( root_uri, - &table_relative_path, - &head_ds, - )?; - - updates.push(ManifestChange::Update(SubTableUpdate { - table_key: pin.table_key.clone(), - table_version: head_version, - table_branch: pin.table_branch.clone(), - row_count, - version_metadata, - })); - expected.insert(pin.table_key.clone(), pin.expected_version); + &pin.table_key, + &pin.table_path, + pin.table_branch.as_deref(), + pin.expected_version, + &mut updates, + &mut expected, + ) + .await?; published_versions.insert(pin.table_key.clone(), head_version); } @@ -1047,6 +1065,57 @@ async fn roll_forward_all( Ok((new_dataset.version().version, published_versions)) } +/// Open `table_path` at its branch HEAD, read the current Lance HEAD version, +/// row count, and version metadata, and push a `ManifestChange::Update` (plus +/// its CAS `expected` entry) that re-pins the manifest to that HEAD. Returns the +/// published HEAD version. +/// +/// Shared by `roll_forward_all` (where `expected_version` is the sidecar's +/// pre-write pin) and `roll_back_sidecar` (where it is the manifest-pinned +/// version the table was just restored to). The HEAD is read AFTER any restore +/// in the same single-threaded sweep, so no concurrent writer can have advanced +/// it. +async fn push_table_update_at_head( + root_uri: &str, + table_key: &str, + table_path: &str, + branch: Option<&str>, + expected_version: u64, + updates: &mut Vec, + expected: &mut HashMap, +) -> Result { + let head_ds = Dataset::open(table_path) + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + let head_ds = match branch { + Some(b) if b != "main" => head_ds + .checkout_branch(b) + .await + .map_err(|e| OmniError::Lance(e.to_string()))?, + _ => head_ds, + }; + let head_version = head_ds.version().version; + let row_count = head_ds + .count_rows(None) + .await + .map_err(|e| OmniError::Lance(e.to_string()))? as u64; + let table_relative_path = super::table_path_for_table_key(table_key)?; + let version_metadata = super::metadata::TableVersionMetadata::from_dataset( + root_uri, + &table_relative_path, + &head_ds, + )?; + updates.push(ManifestChange::Update(SubTableUpdate { + table_key: table_key.to_string(), + table_version: head_version, + table_branch: branch.map(str::to_string), + row_count, + version_metadata, + })); + expected.insert(table_key.to_string(), expected_version); + Ok(head_version) +} + /// Append the audit row describing this recovery action. /// /// Two-part write: (a) `_graph_commits.lance` row anchored on the recovery diff --git a/crates/omnigraph/src/db/manifest/tests.rs b/crates/omnigraph/src/db/manifest/tests.rs index effa0b5..885a2a8 100644 --- a/crates/omnigraph/src/db/manifest/tests.rs +++ b/crates/omnigraph/src/db/manifest/tests.rs @@ -1461,6 +1461,80 @@ async fn test_publish_migrates_pre_stamp_manifest_to_current_version() { assert!(reopened.snapshot().entry("node:Person").is_some()); } +#[tokio::test] +async fn test_v2_to_v3_sweeps_legacy_run_branches_on_write_open() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let catalog = build_test_catalog(); + let mut mc = ManifestCoordinator::init(uri, &catalog).await.unwrap(); + + // Synthesize a pre-MR-770 graph: several stale `__run__` staging branches + // left on `__manifest` (a real legacy graph accumulates one per run), plus + // a real user branch that must survive the sweep. Multiple run branches + // exercise the migration's delete loop on a single reused dataset handle. + mc.create_branch("__run__01J9LEGACY").await.unwrap(); + mc.create_branch("__run__01J9SECOND").await.unwrap(); + mc.create_branch("__run__01J9THIRD").await.unwrap(); + mc.create_branch("feature").await.unwrap(); + let before = mc.list_branches().await.unwrap(); + assert_eq!( + before.iter().filter(|b| b.starts_with("__run__")).count(), + 3, + "precondition: three legacy run branches exist on __manifest; got {before:?}", + ); + + // Rewind the internal-schema stamp to v2 so the next write-open runs the + // v2 β†’ v3 sweep arm (init stamps at the current version, which is past it). + { + let mut ds = open_manifest_dataset(uri, None).await.unwrap(); + ds.update_schema_metadata([( + "omnigraph:internal_schema_version".to_string(), + Some("2".to_string()), + )]) + .await + .unwrap(); + let post = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!(super::migrations::read_stamp(&post), 2, "stamp rewound to v2"); + } + + // A no-op publish forces the open-for-write path, which runs the migration. + let mut expected = HashMap::new(); + expected.insert("node:Person".to_string(), 1); + GraphNamespacePublisher::new(uri, None) + .publish(&[], &expected) + .await + .unwrap(); + + // Stamp advanced to current; the legacy run branch is physically gone from + // `__manifest` (checked via the raw, unfiltered manifest list β€” not the + // guard-filtered `branch_list`), and the real branch + `main` survive. + let post = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!( + super::migrations::read_stamp(&post), + super::migrations::INTERNAL_MANIFEST_SCHEMA_VERSION, + ); + let reopened = ManifestCoordinator::open(uri).await.unwrap(); + let after = reopened.list_branches().await.unwrap(); + assert!( + !after.iter().any(|b| b.starts_with("__run__")), + "legacy run branch must be swept; got {after:?}", + ); + assert!(after.iter().any(|b| b == "feature"), "user branch must survive"); + assert!(after.iter().any(|b| b == "main"), "main must survive"); + + // Idempotent: a second write-open finds the stamp at current and does not + // re-run the sweep or error. + GraphNamespacePublisher::new(uri, None) + .publish(&[], &expected) + .await + .unwrap(); + let final_ds = open_manifest_dataset(uri, None).await.unwrap(); + assert_eq!( + super::migrations::read_stamp(&final_ds), + super::migrations::INTERNAL_MANIFEST_SCHEMA_VERSION, + ); +} + #[tokio::test] async fn test_publish_rejects_manifest_stamped_at_future_version() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index 8702f88..000602a 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -3,7 +3,6 @@ pub mod graph_coordinator; pub mod manifest; mod omnigraph; mod recovery_audit; -mod run_registry; mod schema_state; pub(crate) mod write_queue; @@ -12,10 +11,10 @@ pub use graph_coordinator::{GraphCoordinator, ReadTarget, ResolvedTarget, Snapsh pub use manifest::{Snapshot, SubTableEntry, SubTableUpdate}; pub(crate) use omnigraph::ensure_public_branch_ref; pub use omnigraph::{ - CleanupPolicyOptions, InitOptions, MergeOutcome, Omnigraph, OpenMode, SchemaApplyOptions, - SchemaApplyResult, SkipReason, TableCleanupStats, TableOptimizeStats, + CleanupPolicyOptions, InitOptions, MergeOutcome, Omnigraph, OpenMode, RepairAction, + RepairClassification, RepairOptions, RepairStats, SchemaApplyOptions, SchemaApplyResult, + SkipReason, TableCleanupStats, TableOptimizeStats, TableRepairStats, }; -pub(crate) use run_registry::is_internal_run_branch; pub(crate) const SCHEMA_APPLY_LOCK_BRANCH: &str = "__schema_apply_lock__"; @@ -69,5 +68,8 @@ pub(crate) fn is_schema_apply_lock_branch(name: &str) -> bool { } pub(crate) fn is_internal_system_branch(name: &str) -> bool { - is_internal_run_branch(name) || is_schema_apply_lock_branch(name) + // Legacy `__run__*` staging branches (Run state machine, removed MR-771) + // are swept off `__manifest` by the v2β†’v3 internal-schema migration, so the + // only internal branch the engine still creates is the schema-apply lock. + is_schema_apply_lock_branch(name) } diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 6e49826..0f39120 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -30,10 +30,14 @@ use crate::table_store::TableStore; mod export; mod optimize; +mod repair; mod schema_apply; mod table_ops; pub use optimize::{CleanupPolicyOptions, SkipReason, TableCleanupStats, TableOptimizeStats}; +pub use repair::{ + RepairAction, RepairClassification, RepairOptions, RepairStats, TableRepairStats, +}; pub use schema_apply::SchemaApplyOptions; use super::commit_graph::GraphCommit; @@ -346,6 +350,16 @@ impl Omnigraph { mode: OpenMode, ) -> Result { let root = normalize_root_uri(uri)?; + // Apply pending internal-schema migrations before the coordinator reads + // branch state, so `branch_list` and the schema-apply blocking-branch + // checks observe the post-migration graph β€” notably the v2β†’v3 sweep of + // legacy `__run__*` staging branches (MR-770). ReadWrite only: a + // read-only open must not trigger object-store writes, so a read-only + // open of an unmigrated legacy graph still lists `__run__*` until its + // first read-write open (an accepted, documented limitation). + if matches!(mode, OpenMode::ReadWrite) { + crate::db::manifest::migrate_on_open(&root).await?; + } // Open the coordinator first so the schema-staging recovery sweep can // compare its snapshot against any leftover staging files. let mut coordinator = GraphCoordinator::open(&root, Arc::clone(&storage)).await?; @@ -672,6 +686,16 @@ impl Omnigraph { .map(|resolved| resolved.snapshot) } + pub(crate) async fn fresh_snapshot_for_branch(&self, branch: Option<&str>) -> Result { + self.ensure_schema_state_valid().await?; + let requested = ReadTarget::Branch(branch.unwrap_or("main").to_string()); + let coord = self.coordinator.read().await; + coord + .resolve_target(&requested) + .await + .map(|resolved| resolved.snapshot) + } + pub(crate) async fn version(&self) -> u64 { self.coordinator.read().await.version() } @@ -989,6 +1013,13 @@ impl Omnigraph { optimize::optimize_all_tables(self).await } + /// Classify and explicitly repair uncovered manifest/head drift. See + /// [`repair`] for the distinction between safe maintenance drift and + /// suspicious/unverifiable drift. + pub async fn repair(&self, options: repair::RepairOptions) -> Result { + repair::repair_all_tables(self, options).await + } + /// Remove Lance manifests (and the fragments they uniquely own) per the /// given [`optimize::CleanupPolicyOptions`]. Destructive to version /// history. See [`optimize`] for details. @@ -1495,12 +1526,6 @@ pub(crate) fn normalize_branch_name(branch: &str) -> Result> { } pub(crate) fn ensure_public_branch_ref(branch: &str, operation: &str) -> Result<()> { - if super::is_internal_run_branch(branch) { - return Err(OmniError::manifest(format!( - "{} does not allow internal run ref '{}'", - operation, branch - ))); - } if is_internal_system_branch(branch) { return Err(OmniError::manifest(format!( "{} does not allow internal system ref '{}'", @@ -1904,7 +1929,6 @@ fn json_value_from_array(array: &dyn Array, row: usize) -> Result Company #[tokio::test] async fn test_apply_schema_succeeds_after_load() { // Historical: schema apply used to be blocked by leftover - // `__run__` branches. A defense-in-depth filter now skips - // internal system branches, and run branches were made - // ephemeral on every terminal state β€” so in practice no - // `__run__` branch survives publish. The filter still guards - // the invariant. + // `__run__` branches. The Run state machine was removed in + // MR-771, so a fresh graph never creates a `__run__` branch; + // legacy ones are swept by the v2β†’v3 manifest migration. This + // asserts the invariant a current graph upholds: publish leaves + // no `__run__` branch behind, so schema apply proceeds. let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); @@ -2261,8 +2285,8 @@ edge WorksAt: Person -> Company let all_branches = db.coordinator.read().await.all_branches().await.unwrap(); assert!( - !all_branches.iter().any(|b| is_internal_run_branch(b)), - "run branch should be deleted after publish, got: {:?}", + !all_branches.iter().any(|b| b.starts_with("__run__")), + "no __run__ branch should exist after publish, got: {:?}", all_branches ); @@ -2274,6 +2298,56 @@ edge WorksAt: Person -> Company assert!(result.applied, "schema apply should have applied"); } + /// Regression (MR-770): a pre-v0.4.0 graph that still carries a stale + /// `__run__*` branch on `__manifest` must not block schema apply. The + /// v2β†’v3 sweep runs in `Omnigraph::open(ReadWrite)` β€” before the + /// schema-apply blocking-branch check β€” so apply succeeds with no + /// intervening publish. + /// + /// Confirmed to fail before the open-time migration landed: the reopened + /// graph still listed `__run__legacy`, and `apply_schema` returned + /// "found non-main branches: __run__legacy". + #[tokio::test] + async fn legacy_run_branch_is_swept_on_open_and_does_not_block_schema_apply() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + + // Synthesize a legacy graph: a stale `__run__` branch on `__manifest` + // plus the manifest stamp rewound to v2 (pre-sweep). + db.branch_create("__run__legacy").await.unwrap(); + drop(db); + { + let mut ds = lance::Dataset::open(&format!("{}/__manifest", uri)) + .await + .unwrap(); + ds.update_schema_metadata([( + "omnigraph:internal_schema_version".to_string(), + Some("2".to_string()), + )]) + .await + .unwrap(); + } + + // Reopen (ReadWrite): the open-time migration must sweep `__run__legacy` + // before any branch-observing code runs. + let db = Omnigraph::open(uri).await.unwrap(); + let branches = db.branch_list().await.unwrap(); + assert!( + !branches.iter().any(|b| b.starts_with("__run__")), + "open-time migration must sweep legacy __run__ branches; got {branches:?}", + ); + + // Schema apply must proceed with no intervening publish β€” the + // blocking-branch check no longer sees `__run__legacy`. + let desired = TEST_SCHEMA.replace( + " age: I32?\n}", + " age: I32?\n nickname: String?\n}", + ); + let result = db.apply_schema(&desired).await.unwrap(); + assert!(result.applied, "schema apply should have applied"); + } + #[tokio::test] async fn test_apply_schema_adds_index_for_existing_property() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph/src/db/omnigraph/optimize.rs b/crates/omnigraph/src/db/omnigraph/optimize.rs index 0986bfb..434485a 100644 --- a/crates/omnigraph/src/db/omnigraph/optimize.rs +++ b/crates/omnigraph/src/db/omnigraph/optimize.rs @@ -8,8 +8,14 @@ //! Two dials: //! //! * `optimize_all_tables` β€” Lance `compact_files` on every table. Rewrites -//! small fragments into fewer large ones. Non-destructive (creates a new -//! version; old fragments remain reachable via older manifest versions). +//! small fragments into fewer large ones, then **publishes the compacted +//! version to the `__manifest`** so the manifest's `table_version` tracks the +//! compacted Lance HEAD (reads pin the manifest version, so without the +//! publish compaction would be invisible to readers and would break the +//! HEAD-vs-manifest precondition of schema apply / strict writes). Compaction +//! is content-preserving (Lance `Operation::Rewrite` "reorganizes data +//! without semantic modification"), so old fragments remain reachable via +//! older manifest versions until `cleanup` runs. //! * `cleanup_all_tables` β€” Lance `cleanup_old_versions` on every table. //! Removes manifests (and their unique fragments) older than the configured //! retention. Destructive to version history β€” callers should gate this @@ -23,7 +29,9 @@ use std::time::Duration; use chrono::Utc; use futures::stream::StreamExt; use lance::dataset::cleanup::{CleanupPolicy, RemovalStats}; -use lance::dataset::optimize::{CompactionMetrics, CompactionOptions, compact_files}; +use lance::dataset::optimize::{ + CompactionMetrics, CompactionOptions, compact_files, plan_compaction, +}; use super::*; @@ -67,8 +75,7 @@ pub struct CleanupPolicyOptions { } /// Why `optimize` did not compact a table. Typed so callers branch on the -/// reason rather than sniffing a string. One variant today, gated by -/// [`LANCE_SUPPORTS_BLOB_COMPACTION`]. +/// reason rather than sniffing a string. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum SkipReason { @@ -76,6 +83,12 @@ pub enum SkipReason { /// `BlobHandling::AllBinary`, which mis-decodes blob-v2 columns; see /// [`LANCE_SUPPORTS_BLOB_COMPACTION`] and `docs/dev/lance.md`. BlobColumnsUnsupportedByLance, + /// The Lance dataset HEAD is ahead of the version recorded in + /// `__manifest`, and no recovery sidecar covers that movement. `optimize` + /// cannot infer whether the drift is benign maintenance or an external + /// semantic write, so it leaves the table untouched and points operators at + /// explicit `repair`. + DriftNeedsRepair, } impl SkipReason { @@ -84,6 +97,7 @@ impl SkipReason { pub fn as_str(&self) -> &'static str { match self { SkipReason::BlobColumnsUnsupportedByLance => "blob_columns_unsupported_by_lance", + SkipReason::DriftNeedsRepair => "drift_needs_repair", } } } @@ -95,6 +109,7 @@ impl std::fmt::Display for SkipReason { SkipReason::BlobColumnsUnsupportedByLance => { "blob columns β€” Lance compaction unsupported" } + SkipReason::DriftNeedsRepair => "manifest/head drift β€” run omnigraph repair", }; f.write_str(msg) } @@ -111,11 +126,18 @@ pub struct TableOptimizeStats { pub fragments_removed: usize, /// Number of new, larger fragments Lance produced. pub fragments_added: usize, - /// Did this table get a new Lance manifest version from the compaction? + /// Did this table get a new manifest version from the compaction? True when + /// compaction ran and its compacted version was published to `__manifest`. pub committed: bool, /// `Some(reason)` if this table was deliberately not compacted. When set, /// `fragments_removed == 0`, `fragments_added == 0`, and `!committed`. pub skipped: Option, + /// Manifest table version observed by optimize for drift skips. `None` for + /// normal compaction/no-op/blob skips. + pub manifest_version: Option, + /// Lance HEAD version observed by optimize for drift skips. `None` for + /// normal compaction/no-op/blob skips. + pub lance_head_version: Option, } impl TableOptimizeStats { @@ -127,6 +149,8 @@ impl TableOptimizeStats { fragments_added: metrics.fragments_added, committed, skipped: None, + manifest_version: None, + lance_head_version: None, } } @@ -138,6 +162,25 @@ impl TableOptimizeStats { fragments_added: 0, committed: false, skipped: Some(reason), + manifest_version: None, + lance_head_version: None, + } + } + + /// Stat for a table skipped because the manifest and Lance HEAD disagree. + fn skipped_for_drift( + table_key: String, + manifest_version: u64, + lance_head_version: u64, + ) -> Self { + Self { + table_key, + fragments_removed: 0, + fragments_added: 0, + committed: false, + skipped: Some(SkipReason::DriftNeedsRepair), + manifest_version: Some(manifest_version), + lance_head_version: Some(lance_head_version), } } } @@ -153,14 +196,30 @@ pub struct TableCleanupStats { pub error: Option, } -/// Run Lance `compact_files` on every node + edge table on `main`. -/// Tables run in parallel (bounded concurrency). +/// Run Lance `compact_files` on every node + edge table on `main`, publishing +/// each compacted table's new version to the `__manifest`. Tables run in +/// parallel (bounded concurrency); each is fault-isolated only at the Lance +/// level β€” a publish error is propagated (the recovery sidecar covers it). pub async fn optimize_all_tables(db: &Omnigraph) -> Result> { db.ensure_schema_state_valid().await?; db.ensure_schema_apply_idle("optimize").await?; - let resolved = db.resolved_branch_target(None).await?; - let snapshot = resolved.snapshot; + // Refuse on an unrecovered graph. A pending recovery sidecar means a failed + // write left partial state that the open-time sweep must resolve (roll + // forward/back) first; compacting + publishing a table covered by such a + // sidecar could commit a partial write the sweep would roll back. Reopen the + // graph to run recovery, then re-run optimize. + if !crate::db::manifest::list_sidecars(db.root_uri(), db.storage_adapter()) + .await? + .is_empty() + { + return Err(OmniError::manifest_conflict( + "optimize requires a clean recovery state; reopen the graph to run the \ + recovery sweep before optimizing", + )); + } + + let snapshot = db.fresh_snapshot_for_branch(None).await?; // Compute per-table state (path + whether it has blob columns) up front, in // a scope that drops the catalog handle before the async stream starts. @@ -183,49 +242,201 @@ pub async fn optimize_all_tables(db: &Omnigraph) -> Result> = futures::stream::iter(table_tasks.into_iter()) - .map(|(table_key, full_path, has_blob)| async move { - // Lance `compact_files` mis-decodes blob-v2 columns under the forced - // `BlobHandling::AllBinary` read (see LANCE_SUPPORTS_BLOB_COMPACTION). - // Skip blob-bearing tables and report it rather than aborting the - // whole sweep β€” the other tables still compact. - if has_blob && !LANCE_SUPPORTS_BLOB_COMPACTION { - tracing::warn!( - target: "omnigraph::optimize", - table = %table_key, - "skipping compaction: table has blob columns the current Lance \ - cannot rewrite (blob-v2 AllBinary decode bug); other tables \ - unaffected β€” rerun after the Lance fix", - ); - return Ok(TableOptimizeStats::skipped( - table_key, - SkipReason::BlobColumnsUnsupportedByLance, - )); - } - let mut ds = table_store - .open_dataset_head_for_write(&table_key, &full_path, None) - .await?; - let version_before = ds.version().version; - let metrics: CompactionMetrics = - compact_files(&mut ds, CompactionOptions::default(), None) - .await - .map_err(|e| OmniError::Lance(e.to_string()))?; - let version_after = ds.version().version; - Ok(TableOptimizeStats::compacted( - table_key, - &metrics, - version_after != version_before, - )) + .map(move |(table_key, full_path, has_blob)| async move { + optimize_one_table(db, table_key, full_path, has_blob).await }) .buffer_unordered(concurrency) .collect() .await; + // Invalidate caches for any table that published a compaction β€” done BEFORE + // propagating a sibling table's error, since the published versions are + // durable and reads must observe the new fragment layout (Lance invalidates + // the original row addresses on rewrite). The CSR/CSC graph topology index + // is rebuilt only when an edge table moved. Mirrors schema_apply's + // post-publish invalidation. + let any_committed = stats + .iter() + .any(|s| matches!(s, Ok(st) if st.committed)); + let edge_committed = stats + .iter() + .any(|s| matches!(s, Ok(st) if st.committed && st.table_key.starts_with("edge:"))); + if any_committed { + db.runtime_cache.invalidate_all().await; + if edge_committed { + db.invalidate_graph_index().await; + } + } + stats.into_iter().collect() } +/// Compact one table and publish the compacted version to the `__manifest`. +/// +/// Compaction (`compact_files`) advances the *dataset's* Lance HEAD via a +/// reserve-fragments + rewrite commit, but Lance knows nothing about the +/// `__manifest`. To keep the manifest the single authority for each table's +/// visible version (invariant 2), optimize must publish the compacted version. +/// The Lance-HEAD-before-manifest-publish gap is unavoidable (Lance has no +/// staged/uncommitted compaction), so it is covered by a recovery sidecar like +/// the other multi-commit writers; roll-forward is always safe because +/// compaction is content-preserving. +async fn optimize_one_table( + db: &Omnigraph, + table_key: String, + full_path: String, + has_blob: bool, +) -> Result { + // Lance `compact_files` mis-decodes blob-v2 columns under the forced + // `BlobHandling::AllBinary` read (see LANCE_SUPPORTS_BLOB_COMPACTION). Skip + // blob-bearing tables before acquiring the write queue; `repair` is the + // operator tool for full manifest/head drift classification. + if has_blob && !LANCE_SUPPORTS_BLOB_COMPACTION { + tracing::warn!( + target: "omnigraph::optimize", + table = %table_key, + "skipping compaction: table has blob columns the current Lance \ + cannot rewrite (blob-v2 AllBinary decode bug); other tables \ + unaffected β€” rerun after the Lance fix", + ); + return Ok(TableOptimizeStats::skipped( + table_key, + SkipReason::BlobColumnsUnsupportedByLance, + )); + } + + // Serialize the whole compactβ†’publish against concurrent mutations on this + // (table, main): compaction is a Rewrite op that retryable-conflicts with a + // concurrent Merge/Update/Delete on overlapping fragments, and an + // interleaved write would also move the manifest version out from under the + // CAS below. Holding the queue makes the CAS baseline read under it exact. + let _guard = db + .write_queue() + .acquire_many(&[(table_key.clone(), None)]) + .await; + + let mut ds = db + .table_store + .open_dataset_head_for_write(&table_key, &full_path, None) + .await?; + + // CAS baseline: the table's current manifest version, read under the queue + // (in-memory coordinator snapshot, no storage I/O β€” stable for this section). + let expected_version = db + .fresh_snapshot_for_branch(None) + .await? + .entry(&table_key) + .map(|e| e.table_version) + .ok_or_else(|| OmniError::manifest(format!("no manifest entry for {}", table_key)))?; + + let lance_head_version = ds.version().version; + if lance_head_version < expected_version { + return Err(OmniError::manifest_internal(format!( + "table '{}' Lance HEAD version {} is behind manifest version {}", + table_key, lance_head_version, expected_version + ))); + } + if lance_head_version > expected_version { + tracing::warn!( + target: "omnigraph::optimize", + table = %table_key, + manifest_version = expected_version, + lance_head_version, + "skipping compaction: Lance HEAD is ahead of the manifest; run `omnigraph repair` \ + to classify and publish covered maintenance drift explicitly", + ); + return Ok(TableOptimizeStats::skipped_for_drift( + table_key, + expected_version, + lance_head_version, + )); + } + + // Precise "will it compact?" check β€” `plan_compaction` also accounts for + // deletion materialization (which can rewrite even a single fragment). A + // steady-state already-compacted table yields an empty plan and is never + // pinned in a sidecar (a zero-commit pin would classify NoMovement on + // recovery and force an all-or-nothing rollback). Uncovered pre-existing + // drift is skipped above and must go through explicit repair. + let options = CompactionOptions::default(); + let plan = plan_compaction(&ds, &options) + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + if plan.num_tasks() == 0 { + return Ok(TableOptimizeStats::compacted( + table_key, + &CompactionMetrics::default(), + false, + )); + } + + // Phase A: recovery sidecar BEFORE compaction advances the Lance HEAD, so a + // crash before the manifest publish rolls forward on next open. + let sidecar = crate::db::manifest::new_sidecar( + crate::db::manifest::SidecarKind::Optimize, + None, + // optimize is system-attributed (no `optimize_as` actor API today). + None, + vec![crate::db::manifest::SidecarTablePin { + table_key: table_key.clone(), + table_path: full_path.clone(), + expected_version, + // Lower bound β€” compaction commits Nβ‰₯1 versions (reserve + rewrite); + // the classifier loose-matches SidecarKind::Optimize. + post_commit_pin: expected_version + 1, + table_branch: None, + }], + ); + let handle = + crate::db::manifest::write_sidecar(db.root_uri(), db.storage_adapter(), &sidecar).await?; + + // Phase B: compaction (reserve-fragments + rewrite commits advance HEAD). + let version_before = ds.version().version; + let metrics: CompactionMetrics = compact_files(&mut ds, options, None) + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + let version_after = ds.version().version; + let committed = version_after != version_before; + + // Pin the per-writer Phase B β†’ Phase C residual for optimize: Lance HEAD has + // advanced but the manifest publish below hasn't run. + crate::failpoints::maybe_fail("optimize.post_phase_b_pre_manifest_commit")?; + + // Phase C: publish the compacted version to the manifest (one CAS commit, + // expected = the version observed under the queue). On failure the sidecar + // is intentionally left for the open-time recovery sweep to roll forward. + if committed { + let state = db.table_store.table_state(&full_path, &ds).await?; + let update = crate::db::SubTableUpdate { + table_key: table_key.clone(), + table_version: state.version, + table_branch: None, + row_count: state.row_count, + version_metadata: state.version_metadata, + }; + let mut expected = std::collections::HashMap::new(); + expected.insert(table_key.clone(), expected_version); + db.coordinator + .write() + .await + .commit_updates_with_actor_with_expected(&[update], &expected, None) + .await?; + } + + // Phase D: delete the sidecar (best-effort; recovery resolves a leftover). + if let Err(err) = crate::db::manifest::delete_sidecar(&handle, db.storage_adapter()).await { + tracing::warn!( + error = %err, + operation_id = handle.operation_id.as_str(), + "optimize recovery sidecar cleanup failed; next open's recovery sweep will resolve it" + ); + } + + Ok(TableOptimizeStats::compacted(table_key, &metrics, committed)) +} + /// Run Lance `cleanup_old_versions` on every node + edge table on `main`, /// using [`CleanupPolicyOptions`]. The latest manifest is always preserved /// regardless (Lance invariant). @@ -493,7 +704,7 @@ fn orphan_branches(present: Vec, keep: &std::collections::HashSet Vec { +pub(super) fn all_table_keys(catalog: &omnigraph_compiler::catalog::Catalog) -> Vec { let mut keys: Vec = catalog .node_types .keys() diff --git a/crates/omnigraph/src/db/omnigraph/repair.rs b/crates/omnigraph/src/db/omnigraph/repair.rs new file mode 100644 index 0000000..aaef2ba --- /dev/null +++ b/crates/omnigraph/src/db/omnigraph/repair.rs @@ -0,0 +1,332 @@ +//! Explicit repair for uncovered manifest/head drift. +//! +//! Recovery sidecars handle deterministic crash residuals automatically. This +//! module is for the different case: a table's Lance HEAD is ahead of the +//! version recorded in `__manifest` and there is no sidecar encoding writer +//! intent. `repair` classifies that uncovered drift from Lance transactions and +//! only auto-publishes maintenance-only drift when the operator confirms. + +use std::collections::HashMap; + +use lance::Dataset; +use lance::dataset::transaction::Operation; + +use super::*; + +/// Options for [`Omnigraph::repair`]. +#[derive(Debug, Clone, Copy, Default)] +pub struct RepairOptions { + /// Preview by default. With `confirm`, verified maintenance drift is + /// published to `__manifest`. + pub confirm: bool, + /// Also publish suspicious/unverifiable drift. Requires `confirm`. + pub force: bool, +} + +/// Classification of a table's manifest/head state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RepairClassification { + /// Lance HEAD equals the manifest pin. + NoDrift, + /// Every uncovered Lance transaction is maintenance-only (`Rewrite` or + /// `ReserveFragments`), so publishing the HEAD is content-preserving. + VerifiedMaintenance, + /// At least one uncovered transaction is semantic (`Append`, `Delete`, + /// `Update`, etc.). + Suspicious, + /// A needed transaction could not be read, so the drift cannot be judged. + Unverifiable, +} + +impl RepairClassification { + /// Stable machine-readable token for serialized output. + pub fn as_str(&self) -> &'static str { + match self { + Self::NoDrift => "no_drift", + Self::VerifiedMaintenance => "verified_maintenance", + Self::Suspicious => "suspicious", + Self::Unverifiable => "unverifiable", + } + } +} + +impl std::fmt::Display for RepairClassification { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// What repair did for a table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RepairAction { + /// Nothing to do. + NoOp, + /// Drift was reported but not published because this was a preview. + Preview, + /// Verified maintenance drift was published to `__manifest`. + Healed, + /// Suspicious/unverifiable drift was published because `force` was set. + Forced, + /// Drift was left untouched because it was not safe to publish without + /// `force`. + Refused, +} + +impl RepairAction { + /// Stable machine-readable token for serialized output. + pub fn as_str(&self) -> &'static str { + match self { + Self::NoOp => "no_op", + Self::Preview => "preview", + Self::Healed => "healed", + Self::Forced => "forced", + Self::Refused => "refused", + } + } +} + +impl std::fmt::Display for RepairAction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Per-table repair outcome. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct TableRepairStats { + pub table_key: String, + pub manifest_version: u64, + pub lance_head_version: u64, + pub classification: RepairClassification, + pub action: RepairAction, + pub operations: Vec, + pub error: Option, +} + +/// Whole-graph repair outcome. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct RepairStats { + pub tables: Vec, + /// New graph manifest version if repair published any table pins. + pub manifest_version: Option, +} + +struct ClassificationResult { + classification: RepairClassification, + operations: Vec, + error: Option, +} + +pub async fn repair_all_tables(db: &Omnigraph, options: RepairOptions) -> Result { + if options.force && !options.confirm { + return Err(OmniError::manifest("repair --force requires --confirm")); + } + + db.ensure_schema_state_valid().await?; + db.ensure_schema_apply_idle("repair").await?; + ensure_no_pending_recovery_sidecars(db, "repair").await?; + + let snapshot = db.fresh_snapshot_for_branch(None).await?; + let table_tasks: Vec<(String, String)> = { + let catalog = db.catalog(); + let mut tasks = Vec::new(); + for table_key in optimize::all_table_keys(&catalog) { + let Some(entry) = snapshot.entry(&table_key) else { + continue; + }; + let full_path = format!("{}/{}", db.root_uri, entry.table_path); + tasks.push((table_key, full_path)); + } + tasks + }; + + if table_tasks.is_empty() { + return Ok(RepairStats { + tables: Vec::new(), + manifest_version: None, + }); + } + + let queue_keys: Vec<(String, Option)> = table_tasks + .iter() + .map(|(table_key, _)| (table_key.clone(), None)) + .collect(); + let _guards = db.write_queue().acquire_many(&queue_keys).await; + ensure_no_pending_recovery_sidecars(db, "repair").await?; + + let snapshot = db.fresh_snapshot_for_branch(None).await?; + let mut tables = Vec::with_capacity(table_tasks.len()); + let mut updates = Vec::new(); + let mut expected = HashMap::new(); + let mut any_forced = false; + + for (table_key, full_path) in table_tasks { + let ds = db + .table_store + .open_dataset_head_for_write(&table_key, &full_path, None) + .await?; + let manifest_version = snapshot + .entry(&table_key) + .map(|e| e.table_version) + .ok_or_else(|| OmniError::manifest(format!("no manifest entry for {}", table_key)))?; + let lance_head_version = ds.version().version; + + if lance_head_version < manifest_version { + return Err(OmniError::manifest_internal(format!( + "table '{}' Lance HEAD version {} is behind manifest version {}", + table_key, lance_head_version, manifest_version + ))); + } + + if lance_head_version == manifest_version { + tables.push(TableRepairStats { + table_key, + manifest_version, + lance_head_version, + classification: RepairClassification::NoDrift, + action: RepairAction::NoOp, + operations: Vec::new(), + error: None, + }); + continue; + } + + let classification = classify_drift(&ds, manifest_version, lance_head_version).await; + let action = match ( + options.confirm, + options.force, + classification.classification, + ) { + (false, _, _) => RepairAction::Preview, + (true, _, RepairClassification::VerifiedMaintenance) => RepairAction::Healed, + (true, true, RepairClassification::Suspicious | RepairClassification::Unverifiable) => { + any_forced = true; + RepairAction::Forced + } + (true, _, RepairClassification::Suspicious | RepairClassification::Unverifiable) => { + RepairAction::Refused + } + (true, _, RepairClassification::NoDrift) => RepairAction::NoOp, + }; + + if matches!(action, RepairAction::Healed | RepairAction::Forced) { + let state = db.table_store.table_state(&full_path, &ds).await?; + updates.push(crate::db::SubTableUpdate { + table_key: table_key.clone(), + table_version: state.version, + table_branch: None, + row_count: state.row_count, + version_metadata: state.version_metadata, + }); + expected.insert(table_key.clone(), manifest_version); + } + + tables.push(TableRepairStats { + table_key, + manifest_version, + lance_head_version, + classification: classification.classification, + action, + operations: classification.operations, + error: classification.error, + }); + } + + let manifest_version = if updates.is_empty() { + None + } else { + let actor = if any_forced { + Some("omnigraph:repair:force") + } else { + Some("omnigraph:repair") + }; + let PublishedSnapshot { + manifest_version, + _snapshot_id: _, + } = db + .coordinator + .write() + .await + .commit_updates_with_actor_with_expected(&updates, &expected, actor) + .await?; + db.runtime_cache.invalidate_all().await; + if updates + .iter() + .any(|update| update.table_key.starts_with("edge:")) + { + db.invalidate_graph_index().await; + } + Some(manifest_version) + }; + + Ok(RepairStats { + tables, + manifest_version, + }) +} + +async fn ensure_no_pending_recovery_sidecars(db: &Omnigraph, operation: &str) -> Result<()> { + if !crate::db::manifest::list_sidecars(db.root_uri(), db.storage_adapter()) + .await? + .is_empty() + { + return Err(OmniError::manifest_conflict(format!( + "{operation} requires a clean recovery state; reopen the graph to run the \ + recovery sweep before repairing" + ))); + } + Ok(()) +} + +async fn classify_drift( + ds: &Dataset, + manifest_version: u64, + lance_head_version: u64, +) -> ClassificationResult { + let mut operations = Vec::new(); + let mut saw_suspicious = false; + let mut error = None; + + for version in manifest_version.saturating_add(1)..=lance_head_version { + match ds.read_transaction_by_version(version).await { + Ok(Some(transaction)) => { + let operation = transaction.operation; + operations.push(operation.name().to_string()); + if !matches!( + operation, + Operation::Rewrite { .. } | Operation::ReserveFragments { .. } + ) { + saw_suspicious = true; + } + } + Ok(None) => { + error = Some(format!("missing Lance transaction for version {version}")); + break; + } + Err(err) => { + error = Some(format!( + "failed to read Lance transaction for version {version}: {err}" + )); + break; + } + } + } + + let classification = if error.is_some() { + RepairClassification::Unverifiable + } else if saw_suspicious { + RepairClassification::Suspicious + } else { + RepairClassification::VerifiedMaintenance + }; + + ClassificationResult { + classification, + operations, + error, + } +} diff --git a/crates/omnigraph/src/db/omnigraph/schema_apply.rs b/crates/omnigraph/src/db/omnigraph/schema_apply.rs index 35fe161..7cb3193 100644 --- a/crates/omnigraph/src/db/omnigraph/schema_apply.rs +++ b/crates/omnigraph/src/db/omnigraph/schema_apply.rs @@ -61,11 +61,11 @@ async fn plan_schema_for_apply( ) -> Result { db.ensure_schema_state_valid().await?; let branches = db.coordinator.read().await.all_branches().await?; - // Skip `main` and internal system branches. The schema-apply lock branch - // is excluded because it is the cluster-wide schema-apply serializer. - // `__run__*` branches are no longer created; the filter remains as - // defense-in-depth for legacy graphs with leftover staging branches. - // A future production sweep will let this guard go. + // Skip `main` and internal system branches (the schema-apply lock branch, + // the cluster-wide schema-apply serializer). Legacy `__run__*` staging + // branches were swept off `__manifest` by the v2β†’v3 migration that runs in + // `Omnigraph::open(ReadWrite)` before this check (MR-770), so they no + // longer appear here. let blocking_branches = branches .into_iter() .filter(|branch| branch != "main" && !is_internal_system_branch(branch)) diff --git a/crates/omnigraph/src/db/run_registry.rs b/crates/omnigraph/src/db/run_registry.rs deleted file mode 100644 index ee3d336..0000000 --- a/crates/omnigraph/src/db/run_registry.rs +++ /dev/null @@ -1,16 +0,0 @@ -// The Run state machine has been removed. Mutations now write directly -// to target tables and use the publisher's `expected_table_versions` -// CAS for cross-table OCC; `__run__` staging branches and the -// `_graph_runs.lance` state machine no longer exist. -// -// What remains is the branch-name predicate, kept as a defense-in-depth -// guard against users naming a public branch `__run__*`. A future -// production sweep of legacy `_graph_runs.lance` rows and stale -// `__run__*` branches will let this predicate (and this file) go too. - -pub(crate) const INTERNAL_RUN_BRANCH_PREFIX: &str = "__run__"; - -pub(crate) fn is_internal_run_branch(name: &str) -> bool { - name.trim_start_matches('/') - .starts_with(INTERNAL_RUN_BRANCH_PREFIX) -} diff --git a/crates/omnigraph/src/exec/merge.rs b/crates/omnigraph/src/exec/merge.rs index 2e5f32e..1068f90 100644 --- a/crates/omnigraph/src/exec/merge.rs +++ b/crates/omnigraph/src/exec/merge.rs @@ -670,36 +670,34 @@ fn update_unique_constraints( table_key: &str, batch: &RecordBatch, constraints: &[Vec], - seen: &mut [HashMap], + seen: &mut [HashMap, String>], conflicts: &mut Vec, ) -> Result<()> { for (constraint_idx, columns) in constraints.iter().enumerate() { let seen = &mut seen[constraint_idx]; - for row in 0..batch.num_rows() { - let mut parts = Vec::with_capacity(columns.len()); - let mut any_null = false; - for column_name in columns { - let column = batch.column_by_name(column_name).ok_or_else(|| { + // Resolve the group's columns once. The candidate dataset always + // carries the full table schema, so a missing column is an internal + // error rather than a skip. + let group_columns = columns + .iter() + .map(|column_name| { + batch.column_by_name(column_name).cloned().ok_or_else(|| { OmniError::manifest(format!( "table {} missing unique column '{}'", table_key, column_name )) - })?; - if column.is_null(row) { - any_null = true; - break; - } - parts.push( - array_value_to_string(column.as_ref(), row) - .map_err(|e| OmniError::Lance(e.to_string()))?, - ); - } - if any_null { + }) + }) + .collect::>>()?; + for row in 0..batch.num_rows() { + // Same tuple key as the intake path β€” one shared derivation in + // `crate::loader::composite_unique_key`, so the two cannot drift on + // separator or scalar conversion. Null rows are exempt. + let Some(key) = crate::loader::composite_unique_key(&group_columns, row)? else { continue; - } - let value = parts.join("|"); + }; let row_id = row_id_at(batch, row)?; - if let Some(first_row_id) = seen.insert(value.clone(), row_id.clone()) { + if let Some(first_row_id) = seen.insert(key, row_id.clone()) { conflicts.push(MergeConflict { table_key: table_key.to_string(), row_id: Some(row_id.clone()), @@ -1087,9 +1085,9 @@ impl Omnigraph { target: &str, actor_id: Option<&str>, ) -> Result { - if is_internal_run_branch(source) || is_internal_run_branch(target) { + if is_internal_system_branch(source) || is_internal_system_branch(target) { return Err(OmniError::manifest(format!( - "branch_merge does not allow internal run refs ('{}' -> '{}')", + "branch_merge does not allow internal system refs ('{}' -> '{}')", source, target ))); } diff --git a/crates/omnigraph/src/exec/mod.rs b/crates/omnigraph/src/exec/mod.rs index 33a7e41..ce72d42 100644 --- a/crates/omnigraph/src/exec/mod.rs +++ b/crates/omnigraph/src/exec/mod.rs @@ -35,7 +35,7 @@ use time::format_description::well_known::Rfc3339; use crate::db::commit_graph::CommitGraph; use crate::db::manifest::ManifestCoordinator; -use crate::db::{MergeOutcome, Omnigraph, is_internal_run_branch}; +use crate::db::{MergeOutcome, Omnigraph, is_internal_system_branch}; use crate::db::{ReadTarget, Snapshot}; use crate::embedding::EmbeddingClient; use crate::error::{MergeConflict, MergeConflictKind, OmniError, Result}; diff --git a/crates/omnigraph/src/exec/mutation.rs b/crates/omnigraph/src/exec/mutation.rs index 02b2a21..0e7ded7 100644 --- a/crates/omnigraph/src/exec/mutation.rs +++ b/crates/omnigraph/src/exec/mutation.rs @@ -569,7 +569,8 @@ use super::staging::{MutationStaging, PendingMode}; /// via `open_for_mutation_on_branch`, which compares Lance HEAD against /// the manifest's pinned version β€” that fence is the engine's /// publisher-style OCC catching cross-writer drift before we make any -/// changes. +/// changes. For delete-only queries, this strict open is also the uncovered +/// drift guard that runs before `delete_where` can inline-commit. /// /// On subsequent touches *within the same query*, behavior depends on /// whether the table has already been inline-committed by a delete op: @@ -904,12 +905,12 @@ impl Omnigraph { let batch = build_insert_batch(&schema, &id, &resolved, &blob_props)?; crate::loader::validate_value_constraints(&batch, node_type)?; crate::loader::validate_enum_constraints(&batch, &node_type.properties, type_name)?; - let unique_props = crate::loader::unique_property_names_for_node(node_type); - if !unique_props.is_empty() { + let unique_groups = crate::loader::unique_constraint_groups_for_node(node_type); + if !unique_groups.is_empty() { crate::loader::enforce_unique_constraints_intra_batch( &batch, type_name, - &unique_props, + &unique_groups, )?; } let has_key = node_type.key_property().is_some(); @@ -945,12 +946,12 @@ impl Omnigraph { let batch = build_insert_batch(&schema, &id, &resolved, &blob_props)?; validate_edge_insert_endpoints(self, staging, branch, type_name, &resolved).await?; crate::loader::validate_enum_constraints(&batch, &edge_type.properties, type_name)?; - let unique_props = crate::loader::unique_property_names_for_edge(edge_type); - if !unique_props.is_empty() { + let unique_groups = crate::loader::unique_constraint_groups_for_edge(edge_type); + if !unique_groups.is_empty() { crate::loader::enforce_unique_constraints_intra_batch( &batch, type_name, - &unique_props, + &unique_groups, )?; } let table_key = format!("edge:{}", type_name); @@ -1093,12 +1094,12 @@ impl Omnigraph { let node_type = &self.catalog().node_types[type_name]; crate::loader::validate_value_constraints(&updated, node_type)?; crate::loader::validate_enum_constraints(&updated, &node_type.properties, type_name)?; - let unique_props = crate::loader::unique_property_names_for_node(node_type); - if !unique_props.is_empty() { + let unique_groups = crate::loader::unique_constraint_groups_for_node(node_type); + if !unique_groups.is_empty() { crate::loader::enforce_unique_constraints_intra_batch( &updated, type_name, - &unique_props, + &unique_groups, )?; } diff --git a/crates/omnigraph/src/exec/projection.rs b/crates/omnigraph/src/exec/projection.rs index dec13a8..7280ec5 100644 --- a/crates/omnigraph/src/exec/projection.rs +++ b/crates/omnigraph/src/exec/projection.rs @@ -422,6 +422,35 @@ pub(super) fn apply_ordering( }); } + // Deterministic tie-break for a TOTAL order. `lexsort_to_indices` is unstable + // and the input row order is not guaranteed (scan parallelism, upstream + // hashing), so equal user-sort keys would otherwise come out run-dependent β€” + // making `ORDER ... LIMIT` non-deterministic. Append the bound entities' key + // columns (`.id`, unique per row) in canonical (name-sorted) order as + // ascending tie-breaks. The combination of all bound keys uniquely identifies + // a result row, so the order is total and reproducible. (Aggregate results + // have no `.id` columns; their group rows are already distinct on the + // projected group keys.) + let mut tiebreak_cols: Vec = source + .schema() + .fields() + .iter() + .map(|f| f.name().to_string()) + .filter(|name| name.ends_with(".id")) + .collect(); + tiebreak_cols.sort(); + for name in &tiebreak_cols { + if let Some(col) = source.column_by_name(name) { + sort_columns.push(SortColumn { + values: col.clone(), + options: Some(arrow_schema::SortOptions { + descending: false, + nulls_first: true, + }), + }); + } + } + let indices = lexsort_to_indices(&sort_columns, None).map_err(|e| OmniError::Lance(e.to_string()))?; diff --git a/crates/omnigraph/src/exec/query.rs b/crates/omnigraph/src/exec/query.rs index 7590512..5bc18f2 100644 --- a/crates/omnigraph/src/exec/query.rs +++ b/crates/omnigraph/src/exec/query.rs @@ -24,20 +24,14 @@ impl Omnigraph { .pipeline .iter() .any(|op| matches!(op, IROp::Expand { .. } | IROp::AntiJoin { .. })); + // Lazy: an index-served query with no AntiJoin never builds the CSR. let graph_index = if needs_graph { - Some(self.graph_index_for_resolved(&resolved).await?) + GraphIndexHandle::cached(self, &resolved) } else { - None + GraphIndexHandle::none() }; - execute_query( - &ir, - params, - &resolved.snapshot, - graph_index.as_deref(), - &catalog, - ) - .await + execute_query(&ir, params, &resolved.snapshot, &graph_index, &catalog).await } /// Run a named query against the graph as it existed at a prior manifest version. @@ -64,18 +58,21 @@ impl Omnigraph { .pipeline .iter() .any(|op| matches!(op, IROp::Expand { .. } | IROp::AntiJoin { .. })); + // Lazy build against this historical snapshot (not the RuntimeCache, + // which is keyed to live branch targets); only a CSR-path Expand or an + // AntiJoin triggers it. let graph_index = if needs_graph { let edge_types = catalog .edge_types .iter() .map(|(name, et)| (name.clone(), (et.from_type.clone(), et.to_type.clone()))) .collect(); - Some(Arc::new(GraphIndex::build(&snapshot, &edge_types).await?)) + GraphIndexHandle::direct(&snapshot, edge_types) } else { - None + GraphIndexHandle::none() }; - execute_query(&ir, params, &snapshot, graph_index.as_deref(), &catalog).await + execute_query(&ir, params, &snapshot, &graph_index, &catalog).await } } @@ -342,7 +339,7 @@ pub async fn execute_query( ir: &QueryIR, params: &ParamMap, snapshot: &Snapshot, - graph_index: Option<&GraphIndex>, + graph_index: &GraphIndexHandle<'_>, catalog: &Catalog, ) -> Result { let search_mode = extract_search_mode(ir, params, catalog).await?; @@ -400,7 +397,7 @@ async fn execute_rrf_query( ir: &QueryIR, params: &ParamMap, snapshot: &Snapshot, - graph_index: Option<&GraphIndex>, + graph_index: &GraphIndexHandle<'_>, catalog: &Catalog, rrf: &RrfMode, ) -> Result { @@ -583,7 +580,7 @@ fn execute_pipeline<'a>( pipeline: &'a [IROp], params: &'a ParamMap, snapshot: &'a Snapshot, - graph_index: Option<&'a GraphIndex>, + graph_index: &'a GraphIndexHandle<'a>, catalog: &'a Catalog, wide: &'a mut Option, search_mode: &'a SearchMode, @@ -653,13 +650,10 @@ fn execute_pipeline<'a>( max_hops, dst_filters, } => { - let gi = graph_index.ok_or_else(|| { - OmniError::manifest("graph index required for traversal".to_string()) - })?; if let Some(batch) = wide.as_mut() { execute_expand( batch, - gi, + graph_index, snapshot, catalog, src_var, @@ -688,8 +682,671 @@ fn execute_pipeline<'a>( }) } -/// Execute a graph traversal (Expand). +/// Lazily provides the in-memory CSR graph index, building it on first use and +/// memoizing for the rest of the query. Indexed-mode Expand never asks for it, +/// so a query that is entirely index-served and has no AntiJoin never pays the +/// O(|E|) CSR build (the whole point of the indexed path). The `Cached` builder +/// also reuses the cross-query `RuntimeCache` entry; `Direct` builds against an +/// arbitrary snapshot (time-travel reads); `None` is for queries with no +/// traversal at all. +pub struct GraphIndexHandle<'a> { + cell: tokio::sync::OnceCell>>, + builder: GraphIndexBuilder<'a>, +} + +enum GraphIndexBuilder<'a> { + None, + Cached(&'a Omnigraph, &'a crate::db::ResolvedTarget), + Direct(&'a Snapshot, HashMap), +} + +impl<'a> GraphIndexHandle<'a> { + fn none() -> Self { + Self { + cell: tokio::sync::OnceCell::new(), + builder: GraphIndexBuilder::None, + } + } + + fn cached(db: &'a Omnigraph, resolved: &'a crate::db::ResolvedTarget) -> Self { + Self { + cell: tokio::sync::OnceCell::new(), + builder: GraphIndexBuilder::Cached(db, resolved), + } + } + + fn direct(snapshot: &'a Snapshot, edge_types: HashMap) -> Self { + Self { + cell: tokio::sync::OnceCell::new(), + builder: GraphIndexBuilder::Direct(snapshot, edge_types), + } + } + + /// The CSR index, built on first call. `None` only when the query needs no + /// traversal (the `None` builder). + async fn get(&self) -> Result> { + let built = self + .cell + .get_or_try_init(|| async { + match &self.builder { + GraphIndexBuilder::None => Ok::>, OmniError>(None), + GraphIndexBuilder::Cached(db, resolved) => { + Ok(Some(db.graph_index_for_resolved(resolved).await?)) + } + GraphIndexBuilder::Direct(snapshot, edge_types) => { + Ok(Some(Arc::new(GraphIndex::build(snapshot, edge_types).await?))) + } + } + }) + .await?; + Ok(built.as_deref()) + } + + /// Whether the in-memory CSR is already materialized for this query (a prior + /// Expand or bulk AntiJoin realized it), so reusing it is ~free. Lets the + /// cost chooser prefer the warm CSR over per-hop indexed scans. + fn is_built(&self) -> bool { + matches!(self.cell.get(), Some(Some(_))) + } +} + +/// Explicit traversal-mode override. `OMNIGRAPH_TRAVERSAL_MODE=indexed|csr` +/// forces the path (ops escape hatch + test hook). Both modes are semantically +/// identical, so the override only changes which path runs, never the result. +fn traversal_indexed_override() -> Option { + match std::env::var("OMNIGRAPH_TRAVERSAL_MODE").ok().as_deref() { + Some("indexed") => Some(true), + Some("csr") => Some(false), + _ => None, + } +} + +/// Max source-row frontier for which Expand uses the BTREE-indexed path. +/// Larger frontiers fall back to the in-memory CSR (dense / whole-graph). See +/// `docs/user/constants.md`. +const DEFAULT_EXPAND_INDEXED_MAX_FRONTIER: usize = 1024; +/// Max hop count for the indexed path (each hop is one indexed scan; very deep +/// traversals fan out toward whole-graph and are better served by CSR). +const DEFAULT_EXPAND_INDEXED_MAX_HOPS: u32 = 6; + +fn expand_indexed_max_frontier() -> usize { + std::env::var("OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_EXPAND_INDEXED_MAX_FRONTIER) +} + +fn expand_indexed_max_hops() -> u32 { + std::env::var("OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v > 0) + .unwrap_or(DEFAULT_EXPAND_INDEXED_MAX_HOPS) +} + +/// The two Expand execution paths the chooser dispatches between. Extensible: +/// a future persisted-adjacency artifact would become a third variant here, and +/// `choose_expand_mode` would learn to prefer it when covered. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExpandMode { + /// Per-hop neighbor lookup via the persisted src/dst BTREE. Work scales + /// with the frontier, not |E| β€” best for selective traversals. + IndexedScan, + /// Whole-graph in-memory CSR (built once, reused). Best for dense / deep / + /// large-frontier traversals, or when the index is degraded and a full + /// scan would be paid per hop anyway. + Csr, +} + +/// Building the in-memory CSR costs more than a bare edge scan: it scans every +/// edge AND allocates + groups the adjacency. This factor expresses that +/// overhead so a one-off degraded single-hop scan can still edge out a full CSR +/// build. The crossover is insensitive to its exact value. +const CSR_BUILD_FACTOR: f64 = 1.5; + +/// Cardinality inputs for the (pure, IO-free) traversal-mode cost model. Every +/// field is a cheap manifest-resident count or an already-in-hand value β€” the +/// chooser performs no scans. +#[derive(Debug, Clone)] +struct ExpandCostInputs { + /// Current frontier size (`wide.num_rows()`). + frontier_rows: usize, + /// |E| for the edge type (manifest `row_count`). + edge_count: u64, + /// |V_src| β€” node count of the keyed endpoint type (manifest `row_count`). + src_node_count: u64, + /// Effective max hop count for this Expand. + effective_max_hops: u32, + /// Hard ceiling above which the indexed path is never used (resolved + /// `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS`). + max_hops_cap: u32, + /// Hard ceiling above which the indexed path is never used (resolved + /// `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`). + max_frontier_cap: usize, + /// Whether `scan_edges_by_endpoint`'s `key_col IN (...)` is served by the + /// BTREE (`Indexed`) or silently falls back to a full scan (`Degraded`). + coverage: crate::table_store::IndexCoverage, + /// Whether the cross-query CSR for this snapshot+edge-version is already + /// built (making the CSR path β‰ˆ free). Conservatively `false` until the + /// cache-peek is wired (the plan's optional refinement). + csr_cached: bool, +} + +/// Pure cost-based traversal-mode chooser. Compares an estimate of the indexed +/// path's frontier-relative work against the cost of building (or reusing) the +/// whole-graph CSR, and picks the cheaper. Deterministic and IO-free so it is +/// unit-tested at the crossover; the caller supplies the manifest counts and the +/// (optionally degraded) index coverage. +/// +/// Under `Indexed` coverage and a cold CSR the decision reduces to a clean +/// selectivity ratio β€” indexed wins when `hops * frontier < BUILD_FACTOR * +/// |V_src|`, i.e. when the frontier is a small fraction of the source vertex +/// set β€” which is independent of |E| (the flat-in-|E| property PR #149 shipped). +fn choose_expand_mode(i: &ExpandCostInputs) -> ExpandMode { + // Hard ceilings: very deep or very large frontiers fan out toward + // whole-graph and are always better served by CSR, regardless of the cost + // estimate. These preserve the documented semantics of the two cap flags. + if i.effective_max_hops > i.max_hops_cap || i.frontier_rows > i.max_frontier_cap { + return ExpandMode::Csr; + } + + let hops = i.effective_max_hops.max(1) as f64; + let frontier = i.frontier_rows as f64; + let edges = i.edge_count as f64; + let src = i.src_node_count.max(1) as f64; + let fanout = edges / src; + + // Indexed work scales with the frontier when the BTREE serves the IN-list; + // a degraded scan is a full edge scan per hop instead (the C6 perf cliff). + let indexed_cost = match i.coverage { + crate::table_store::IndexCoverage::Indexed => hops * frontier * fanout, + crate::table_store::IndexCoverage::Degraded { .. } => hops * edges, + }; + // A warm CSR is ~free to reuse; a cold one costs a build over all edges. + let csr_cost = if i.csr_cached { + 0.0 + } else { + CSR_BUILD_FACTOR * edges + }; + + if indexed_cost < csr_cost { + ExpandMode::IndexedScan + } else { + ExpandMode::Csr + } +} + +/// Hops the indexed path will actually run, for cost-model purposes. A cross-type +/// edge cannot chain, so `execute_expand_indexed` caps it at one hop regardless of +/// the requested range; the cost model must use that, or it over-estimates the +/// indexed cost of a cross-type variable-length expand and skews toward CSR. +fn cost_effective_hops(requested_max_hops: u32, same_type: bool) -> u32 { + if same_type { + requested_max_hops + } else { + requested_max_hops.min(1) + } +} + +/// Gather the cost-model inputs from cheap manifest counts. `None` when the +/// edge type, its source node type, or their manifest entries are absent (e.g. +/// a not-yet-materialized table) β€” the caller then falls back to the legacy +/// frontier/hop ceiling so the decision is always defined. +fn gather_cost_inputs( + snapshot: &Snapshot, + catalog: &Catalog, + edge_type: &str, + direction: Direction, + frontier_rows: usize, + effective_max_hops: u32, + coverage: crate::table_store::IndexCoverage, + csr_cached: bool, +) -> Option { + let edge_entry = snapshot.entry(&format!("edge:{}", edge_type))?; + let edge_def = catalog.edge_types.get(edge_type)?; + // Match the indexed path's cross-type one-hop cap so the cost estimate + // reflects what actually runs (see `cost_effective_hops`). + let effective_max_hops = + cost_effective_hops(effective_max_hops, edge_def.from_type == edge_def.to_type); + // The frontier source vertices are the keyed endpoint's type: `from` for an + // Out traversal (keyed on `src`), `to` for In (keyed on `dst`). + let src_type = match direction { + Direction::Out => &edge_def.from_type, + Direction::In => &edge_def.to_type, + }; + let src_entry = snapshot.entry(&format!("node:{}", src_type))?; + Some(ExpandCostInputs { + frontier_rows, + edge_count: edge_entry.row_count, + src_node_count: src_entry.row_count, + effective_max_hops, + max_hops_cap: expand_indexed_max_hops(), + max_frontier_cap: expand_indexed_max_frontier(), + coverage, + csr_cached, + }) +} + +/// Coverage value to feed the cost decision. A failed coverage probe is treated +/// as `Degraded` (conservative: don't over-favor the indexed path when we can't +/// confirm the BTREE will serve the scan). +fn coverage_for_decision( + coverage: &Result, +) -> crate::table_store::IndexCoverage { + match coverage { + Ok(c) => c.clone(), + Err(_) => crate::table_store::IndexCoverage::Degraded { + reason: "coverage check failed".to_string(), + }, + } +} + +/// Surface the C6 silent scalar-index fallback (commit `5a7ab6d`): warn when the +/// per-hop `key_col IN (...)` won't route through the BTREE. Detection-only; +/// never fails the query. Behavior-identical to the inline check it replaced. +fn warn_on_degraded_coverage( + coverage: &Result, + key_col: &str, + edge_type: &str, +) { + match coverage { + Ok(crate::table_store::IndexCoverage::Degraded { reason }) => tracing::warn!( + target: "omnigraph::traverse", + edge = %edge_type, + key_col = key_col, + reason = %reason, + "indexed traversal falls back to a full edge scan (results correct, perf degraded)" + ), + Ok(crate::table_store::IndexCoverage::Indexed) => {} + Err(e) => tracing::debug!( + target: "omnigraph::traverse", + error = %e, + "index-coverage check failed; proceeding with traversal" + ), + } +} + +/// The (key, opposite) endpoint columns for a traversal direction. Out follows +/// src -> dst (key on src); In follows the reverse. The persisted BTREE exists +/// on both columns. +fn endpoint_columns(direction: Direction) -> (&'static str, &'static str) { + match direction { + Direction::Out => ("src", "dst"), + Direction::In => ("dst", "src"), + } +} + +/// Execute a graph traversal (Expand). Dispatches to the BTREE-indexed path +/// (selective traversals β€” neighbor lookups via the persisted src/dst index) or +/// the in-memory CSR path (dense / whole-graph traversals). The CSR index is +/// built lazily and only the CSR path requests it. async fn execute_expand( + wide: &mut RecordBatch, + graph_index: &GraphIndexHandle<'_>, + snapshot: &Snapshot, + catalog: &Catalog, + src_var: &str, + dst_var: &str, + edge_type: &str, + direction: Direction, + dst_type: &str, + min_hops: u32, + max_hops: Option, + dst_filters: &[IRFilter], + params: &ParamMap, +) -> Result<()> { + let frontier_rows = wide.num_rows(); + let effective_max_hops = max_hops.unwrap_or(min_hops.max(1)); + let (key_col, _) = endpoint_columns(direction); + let edge_table_key = format!("edge:{}", edge_type); + + // Cardinality-first preliminary decision (no IO). The override wins; else the + // cost model decides under *optimistic* coverage. Optimistic is what lets us + // skip the dataset open on a clearly-CSR traversal: real coverage can only + // make the indexed path costlier, so if even a perfectly-indexed scan loses + // to CSR here, it loses for real. + let forced = traversal_indexed_override(); + let lean_indexed = match forced { + Some(v) => v, + None => match gather_cost_inputs( + snapshot, + catalog, + edge_type, + direction, + frontier_rows, + effective_max_hops, + crate::table_store::IndexCoverage::Indexed, + graph_index.is_built(), + ) { + Some(inputs) => choose_expand_mode(&inputs) == ExpandMode::IndexedScan, + // Manifest counts absent (e.g. not-yet-materialized table): fall back + // to the legacy frontier/hop ceiling so the decision is defined. + None => { + frontier_rows <= expand_indexed_max_frontier() + && effective_max_hops <= expand_indexed_max_hops() + } + }, + }; + + if !lean_indexed { + tracing::debug!( + target: "omnigraph::traverse", + edge = %edge_type, + frontier = frontier_rows, + hops = effective_max_hops, + mode = "csr", + "expand mode chosen", + ); + let gi = graph_index.get().await?.ok_or_else(|| { + OmniError::manifest("graph index required for CSR traversal".to_string()) + })?; + return execute_expand_csr( + wide, gi, snapshot, catalog, src_var, dst_var, edge_type, direction, dst_type, + min_hops, max_hops, dst_filters, params, + ) + .await; + } + + // Leaning indexed: open the edge dataset once, confirm real coverage, and + // (unless forced) re-decide with it. The opened dataset is threaded into the + // indexed path so it is never opened twice. + let edge_ds = snapshot.open(&edge_table_key).await?; + let coverage = + crate::table_store::TableStore::key_column_index_coverage(&edge_ds, key_col).await; + + if forced.is_none() { + if let Some(inputs) = gather_cost_inputs( + snapshot, + catalog, + edge_type, + direction, + frontier_rows, + effective_max_hops, + coverage_for_decision(&coverage), + graph_index.is_built(), + ) { + if choose_expand_mode(&inputs) == ExpandMode::Csr { + tracing::debug!( + target: "omnigraph::traverse", + edge = %edge_type, + frontier = frontier_rows, + hops = effective_max_hops, + mode = "csr", + reason = "index coverage degraded", + "expand mode chosen", + ); + let gi = graph_index.get().await?.ok_or_else(|| { + OmniError::manifest("graph index required for CSR traversal".to_string()) + })?; + return execute_expand_csr( + wide, gi, snapshot, catalog, src_var, dst_var, edge_type, direction, dst_type, + min_hops, max_hops, dst_filters, params, + ) + .await; + } + } + } + + tracing::debug!( + target: "omnigraph::traverse", + edge = %edge_type, + frontier = frontier_rows, + hops = effective_max_hops, + mode = "indexed", + "expand mode chosen", + ); + // Surface the C6 silent scalar-index fallback once, now that coverage is known. + warn_on_degraded_coverage(&coverage, key_col, edge_type); + execute_expand_indexed( + wide, snapshot, catalog, src_var, dst_var, edge_type, direction, dst_type, min_hops, + max_hops, dst_filters, params, edge_ds, + ) + .await +} + +/// BTREE-indexed graph traversal: per hop, batch the current frontier into one +/// `scan_edges_by_endpoint` call against the persisted src/dst index, then fan +/// out per source row. Cost scales with the frontier, not |E|. Produces the +/// same `(src_row, dst_id)` pairs as the CSR path and shares its hydrate+align +/// tail. Multi-hop only advances for same-type edges; cross-type frontiers go +/// empty after one hop (no edges key off the destination type), matching CSR. +async fn execute_expand_indexed( + wide: &mut RecordBatch, + snapshot: &Snapshot, + catalog: &Catalog, + src_var: &str, + dst_var: &str, + edge_type: &str, + direction: Direction, + dst_type: &str, + min_hops: u32, + max_hops: Option, + dst_filters: &[IRFilter], + params: &ParamMap, + edge_ds: Dataset, +) -> Result<()> { + let src_id_col_name = format!("{}.id", src_var); + let src_ids = wide + .column_by_name(&src_id_col_name) + .ok_or_else(|| { + OmniError::manifest(format!("wide batch missing '{}' column", src_id_col_name)) + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| OmniError::manifest(format!("'{}' column is not Utf8", src_id_col_name)))? + .clone(); + + let edge_def = catalog + .edge_types + .get(edge_type) + .ok_or_else(|| OmniError::manifest(format!("unknown edge type '{}'", edge_type)))?; + let same_type = edge_def.from_type == edge_def.to_type; + // The keyed/opposite endpoint columns for this direction. The edge dataset + // and the C6 coverage warn are owned by the caller (`execute_expand`), which + // opens the dataset once and threads it in. + let (key_col, opp_col) = endpoint_columns(direction); + + let max = max_hops.unwrap_or(min_hops.max(1)); + // Cross-type edges cannot chain (a Company is not a `WorksAt` source), so a + // variable-length traversal over one is structurally single-hop. Enforce it + // here instead of relying on the hop-2 scan returning empty: this BFS interns + // every endpoint string into ONE dense id space, so a cross-type id-string + // collision (a Person and a Company sharing an id) would otherwise let hop 2 + // de-intern a destination id back to the colliding source-type id and match + // its edges, emitting rows the CSR path never produces. + let max = if same_type { max } else { max.min(1) }; + + // Per-source BFS state in DENSE id space: intern node ids to u32 once via a + // per-traversal interner so visited/seen/frontier/neighbor-map avoid string + // hashing + cloning in the hot loop (mirrors the CSR path's TypeIndex). The + // GraphIndex/CSR is NOT built β€” only a local id↔u32 dictionary. Strings + // survive at the substrate edges only: the per-hop IN-list to Lance, and the + // emitted dst ids handed to the string-keyed hydrate+align tail. + let mut interner = crate::graph_index::TypeIndex::new(); + let n = src_ids.len(); + let mut frontiers: Vec> = Vec::with_capacity(n); + let mut visited: Vec> = Vec::with_capacity(n); + let mut seen_dst: Vec> = Vec::with_capacity(n); + for i in 0..n { + let sid = interner.get_or_insert(src_ids.value(i)); + let mut v = HashSet::new(); + if same_type { + v.insert(sid); + } + frontiers.push(vec![sid]); + visited.push(v); + seen_dst.push(HashSet::new()); + } + + let mut src_indices: Vec = Vec::new(); + let mut dst_dense: Vec = Vec::new(); + + for hop in 1..=max { + // Union of all live frontiers (dense), de-interned once for the IN-list. + let mut union_dense: Vec = Vec::new(); + { + let mut seen: HashSet = HashSet::new(); + for f in &frontiers { + for &node in f { + if seen.insert(node) { + union_dense.push(node); + } + } + } + } + if union_dense.is_empty() { + break; + } + let union_keys: Vec = union_dense + .iter() + .map(|&u| { + interner + .to_id(u) + .expect("interned frontier id must resolve") + .to_string() + }) + .collect(); + + let batches = crate::table_store::TableStore::scan_edges_by_endpoint( + &edge_ds, key_col, opp_col, &union_keys, + ) + .await?; + + // dense key -> dense neighbors (scan order; duplicates preserved, like CSR multi-edges). + let mut neighbor_map: HashMap> = HashMap::new(); + for batch in &batches { + let keys = batch + .column_by_name(key_col) + .ok_or_else(|| OmniError::manifest(format!("edge batch missing '{}'", key_col)))? + .as_any() + .downcast_ref::() + .ok_or_else(|| OmniError::manifest(format!("edge '{}' is not Utf8", key_col)))?; + let opps = batch + .column_by_name(opp_col) + .ok_or_else(|| OmniError::manifest(format!("edge batch missing '{}'", opp_col)))? + .as_any() + .downcast_ref::() + .ok_or_else(|| OmniError::manifest(format!("edge '{}' is not Utf8", opp_col)))?; + for r in 0..batch.num_rows() { + let k = interner.get_or_insert(keys.value(r)); + let o = interner.get_or_insert(opps.value(r)); + neighbor_map.entry(k).or_default().push(o); + } + } + + // Advance each source row's frontier independently (dense ids). + for i in 0..n { + let cur = std::mem::take(&mut frontiers[i]); + let mut next: Vec = Vec::new(); + for &node in &cur { + let Some(neighbors) = neighbor_map.get(&node) else { + continue; + }; + for &neighbor in neighbors { + if !same_type || visited[i].insert(neighbor) { + next.push(neighbor); + if hop >= min_hops && seen_dst[i].insert(neighbor) { + src_indices.push(i as u32); + dst_dense.push(neighbor); + } + } + } + } + frontiers[i] = next; + } + } + + // De-intern emitted destination ids (parallel to src_indices) for the + // string-keyed hydrate+align tail, exactly as the CSR path does. + let dst_ids: Vec = dst_dense + .iter() + .map(|&d| { + interner + .to_id(d) + .expect("interned dst id must resolve") + .to_string() + }) + .collect(); + + expand_hydrate_and_align( + wide, src_indices, dst_ids, snapshot, catalog, dst_type, dst_var, dst_filters, params, + ) + .await +} + +/// Shared tail for both Expand modes: hydrate the unique destination ids, align +/// the `(src_row, dst_id)` pairs back onto `wide`, hconcat, and apply +/// non-pushable destination filters in memory. +async fn expand_hydrate_and_align( + wide: &mut RecordBatch, + src_indices: Vec, + dst_ids: Vec, + snapshot: &Snapshot, + catalog: &Catalog, + dst_type: &str, + dst_var: &str, + dst_filters: &[IRFilter], + params: &ParamMap, +) -> Result<()> { + // Pushable destination filters are applied by `hydrate_nodes`; the rest + // (`ir_filter_to_expr` β†’ None) are applied in memory after hconcat. + let non_pushable: Vec<&IRFilter> = dst_filters + .iter() + .filter(|f| ir_filter_to_expr(f, params).is_none()) + .collect(); + + // Unique destination ids (first-seen order) for one batched hydration. + let mut unique_dst_list: Vec = Vec::new(); + { + let mut seen: HashSet<&str> = HashSet::with_capacity(dst_ids.len()); + for id in &dst_ids { + if seen.insert(id.as_str()) { + unique_dst_list.push(id.clone()); + } + } + } + let dst_batch = + hydrate_nodes(snapshot, catalog, dst_type, &unique_dst_list, dst_filters, params).await?; + + // id -> row index in the hydrated batch. + let dst_batch_id_col = dst_batch + .column_by_name("id") + .ok_or_else(|| OmniError::manifest("hydrated batch missing 'id' column".to_string()))? + .as_any() + .downcast_ref::() + .ok_or_else(|| OmniError::manifest("hydrated 'id' column is not Utf8".to_string()))?; + let mut id_to_row: HashMap<&str, u32> = HashMap::with_capacity(dst_batch_id_col.len()); + for row in 0..dst_batch_id_col.len() { + id_to_row.insert(dst_batch_id_col.value(row), row as u32); + } + + // Align pairs to (src_row, hydrated_dst_row), dropping ids hydration filtered out. + let mut final_src_indices: Vec = Vec::with_capacity(src_indices.len()); + let mut dst_indices: Vec = Vec::with_capacity(src_indices.len()); + for (&src_idx, dst_id) in src_indices.iter().zip(dst_ids.iter()) { + if let Some(&dst_row) = id_to_row.get(dst_id.as_str()) { + final_src_indices.push(src_idx); + dst_indices.push(dst_row); + } + } + + let src_take = UInt32Array::from(final_src_indices); + let dst_take = UInt32Array::from(dst_indices); + let expanded_wide = take_batch(wide, &src_take)?; + let dst_prefixed = prefix_batch(&dst_batch, dst_var)?; + let aligned_dst = take_batch(&dst_prefixed, &dst_take)?; + *wide = hconcat_batches(&expanded_wide, &aligned_dst)?; + + for f in &non_pushable { + apply_filter(wide, f, params)?; + } + Ok(()) +} + +/// CSR-backed graph traversal: BFS over the in-memory adjacency index. Used for +/// dense / whole-graph traversals; selective traversals use +/// `execute_expand_indexed`. Both share `expand_hydrate_and_align`. +async fn execute_expand_csr( wide: &mut RecordBatch, graph_index: &GraphIndex, snapshot: &Snapshot, @@ -742,6 +1399,9 @@ async fn execute_expand( let max = max_hops.unwrap_or(min_hops.max(1)); let same_type = src_type_name == dst_type_name; + // Cross-type edges cannot chain; a variable-length traversal over one is + // structurally single-hop (mirrors the indexed path's guarantee). + let max = if same_type { max } else { max.min(1) }; // BFS to collect (src_row_idx, dst_dense) pairs with per-source dedup. // Dense u32 ids stay in hand through BFS, dedup, and align β€” we only @@ -785,88 +1445,52 @@ async fn execute_expand( } } - // Split dst_filters: SQL-pushable go to Lance, the rest applied post-hconcat - let pushdown_sql = build_lance_filter(dst_filters, params); - let non_pushable: Vec<&IRFilter> = dst_filters - .iter() - .filter(|f| ir_filter_to_sql(f, params).is_none()) - .collect(); - - // Dedup dst dense ids globally across source rows, then stringify once - // for the Lance IN-list. The post-hydrate alignment fans rows back out to - // the original (src, dst) pairs via a dense-indexed lookup below. - let mut unique_dst_list: Vec = Vec::new(); - { - let mut seen: HashSet = HashSet::with_capacity(dst_dense_list.len()); - for &d in &dst_dense_list { - if seen.insert(d) { - if let Some(id) = dst_type_idx.to_id(d) { - unique_dst_list.push(id.to_string()); - } - } + // Map BFS-produced dense destination ids to string ids for the shared + // hydrate+align tail. Dense ids always resolve (they came from the index); + // drop any that don't, keeping the (src, dst) arrays parallel. + let mut tail_src_indices: Vec = Vec::with_capacity(src_indices.len()); + let mut dst_ids: Vec = Vec::with_capacity(dst_dense_list.len()); + for (&s, &d) in src_indices.iter().zip(dst_dense_list.iter()) { + if let Some(id) = dst_type_idx.to_id(d) { + tail_src_indices.push(s); + dst_ids.push(id.to_string()); } } - let dst_batch = hydrate_nodes( + + expand_hydrate_and_align( + wide, + tail_src_indices, + dst_ids, snapshot, catalog, dst_type, - &unique_dst_list, - pushdown_sql.as_deref(), + dst_var, + dst_filters, + params, ) - .await?; - - // Build dense β†’ row-in-hydrated-batch via a direct-indexed array. - let dst_batch_id_col = dst_batch - .column_by_name("id") - .ok_or_else(|| OmniError::manifest("hydrated batch missing 'id' column".to_string()))? - .as_any() - .downcast_ref::() - .ok_or_else(|| OmniError::manifest("hydrated 'id' column is not Utf8".to_string()))?; - let mut dense_to_row: Vec> = vec![None; dst_type_idx.len()]; - for row in 0..dst_batch_id_col.len() { - let id_str = dst_batch_id_col.value(row); - if let Some(dense) = dst_type_idx.to_dense(id_str) { - dense_to_row[dense as usize] = Some(row as u32); - } - } - - // Build aligned src/dst index arrays (only for ids that exist in hydrated batch) - let mut final_src_indices: Vec = Vec::new(); - let mut dst_indices: Vec = Vec::new(); - for (src_idx, dst_dense) in src_indices.iter().zip(dst_dense_list.iter()) { - if let Some(dst_row) = dense_to_row[*dst_dense as usize] { - final_src_indices.push(*src_idx); - dst_indices.push(dst_row); - } - } - - let src_take = UInt32Array::from(final_src_indices); - let dst_take = UInt32Array::from(dst_indices); - let expanded_wide = take_batch(wide, &src_take)?; - let dst_prefixed = prefix_batch(&dst_batch, dst_var)?; - let aligned_dst = take_batch(&dst_prefixed, &dst_take)?; - *wide = hconcat_batches(&expanded_wide, &aligned_dst)?; - - // Apply any non-pushable destination filters (e.g. list-contains) in memory - for f in &non_pushable { - apply_filter(wide, f, params)?; - } - - Ok(()) + .await } /// Load full node rows for a set of IDs from a snapshot. /// -/// When `extra_filter_sql` is provided (from deferred destination-binding -/// filters), it is ANDed with the `id IN (...)` clause so that Lance can -/// skip non-matching rows at the storage level. +/// The `id IN (...)` predicate is built as a structured DataFusion `Expr` and +/// AND'd with any pushable `dst_filters` (destination-binding filters), then +/// applied via `Scanner::filter_expr`. The structured form routes the id +/// IN-list through the `id` BTREE scalar index (index-search β†’ take) rather +/// than evaluating a string filter via DataFusion `InListEval`, which is +/// O(NΓ—M) and was measured at 72Γ— the indexed cost on a 100k-node hop +/// (MR-376). Non-pushable `dst_filters` (`ir_filter_to_expr` β†’ None) are +/// applied in memory by the caller after hydration. async fn hydrate_nodes( snapshot: &Snapshot, catalog: &Catalog, type_name: &str, ids: &[String], - extra_filter_sql: Option<&str>, + dst_filters: &[IRFilter], + params: &ParamMap, ) -> Result { + use datafusion::prelude::{col, lit}; + let node_type = catalog .node_types .get(type_name) @@ -879,15 +1503,13 @@ async fn hydrate_nodes( let table_key = format!("node:{}", type_name); let ds = snapshot.open(&table_key).await?; - // Build filter: id IN ('a', 'b', 'c') - let escaped: Vec = ids - .iter() - .map(|id| format!("'{}'", id.replace('\'', "''"))) - .collect(); - let mut filter_sql = format!("id IN ({})", escaped.join(", ")); - if let Some(extra) = extra_filter_sql { - filter_sql = format!("({}) AND ({})", filter_sql, extra); + // `id IN (ids)` AND any pushable destination filters, as a structured Expr. + let id_list: Vec = ids.iter().map(|id| lit(id.clone())).collect(); + let mut filter_expr = col("id").in_list(id_list, false); + if let Some(dst_expr) = build_lance_filter_expr(dst_filters, params) { + filter_expr = filter_expr.and(dst_expr); } + let has_blobs = !node_type.blob_properties.is_empty(); let non_blob_cols: Vec<&str> = node_type .arrow_schema @@ -897,12 +1519,16 @@ async fn hydrate_nodes( .map(|f| f.name().as_str()) .collect(); let projection = has_blobs.then_some(non_blob_cols.as_slice()); - let batches = crate::table_store::TableStore::scan_stream( + let batches = crate::table_store::TableStore::scan_stream_with( &ds, projection, - Some(&filter_sql), + None, None, false, + |scanner| { + scanner.filter_expr(filter_expr); + Ok(()) + }, ) .await? .try_collect::>() @@ -925,6 +1551,25 @@ async fn hydrate_nodes( Ok(scan_result) } +/// Whether the inner pipeline is the bulk-anti-join shape: a single Expand from +/// the outer var with no destination filters (the only shape the CSR +/// `has_neighbors` fast path can serve). Pure β€” it does not touch the CSR β€” so +/// the caller can decide whether to realize the O(|E|) graph index at all. +fn bulk_anti_join_applies(inner_pipeline: &[IROp], outer_var: &str) -> bool { + matches!( + inner_pipeline, + [IROp::Expand { src_var, dst_filters, min_hops, max_hops, .. }] + if src_var == outer_var + && dst_filters.is_empty() + // `has_neighbors` is a ONE-hop existence test, so the fast path + // is valid only for a single-hop expand. Multi-hop negations + // (e.g. `not { $p knows{2,2} $x }`) fall to the slow path, whose + // inner Expand runs the real bounded traversal. + && *min_hops == 1 + && (*max_hops).unwrap_or(1) == 1 + ) +} + /// Try bulk anti-join via CSR existence check. Returns Some(mask) if the inner /// pipeline is a single Expand from outer_var (the common negation pattern). fn try_bulk_anti_join_mask( @@ -934,27 +1579,17 @@ fn try_bulk_anti_join_mask( catalog: &Catalog, outer_var: &str, ) -> Option { - if inner_pipeline.len() != 1 { + if !bulk_anti_join_applies(inner_pipeline, outer_var) { return None; } let IROp::Expand { - src_var, edge_type, direction, - dst_filters, .. } = &inner_pipeline[0] else { return None; }; - if src_var != outer_var { - return None; - } - // Bulk CSR check only tests neighbor existence, not destination - // properties. Fall back to the slow path when dst_filters are present. - if !dst_filters.is_empty() { - return None; - } let gi = graph_index?; let edge_def = catalog.edge_types.get(edge_type.as_str())?; @@ -993,49 +1628,106 @@ async fn execute_anti_join( inner_pipeline: &[IROp], params: &ParamMap, snapshot: &Snapshot, - graph_index: Option<&GraphIndex>, + graph_index: &GraphIndexHandle<'_>, catalog: &Catalog, outer_var: &str, ) -> Result<()> { + // Only the bulk fast path consumes the CSR; the slow path's inner Expand + // chooses its own access path. Realize the O(|E|) graph index ONLY when the + // inner-pipeline shape qualifies for the bulk check β€” a filtered/nested + // anti-join over a large graph must not pay a whole-graph build it won't use. + let gi = if bulk_anti_join_applies(inner_pipeline, outer_var) { + graph_index.get().await? + } else { + None + }; // Fast path: bulk CSR existence check (O(N), zero Lance I/O) - if let Some(mask) = - try_bulk_anti_join_mask(wide, inner_pipeline, graph_index, catalog, outer_var) - { + if let Some(mask) = try_bulk_anti_join_mask(wide, inner_pipeline, gi, catalog, outer_var) { *wide = arrow_select::filter::filter_record_batch(wide, &mask) .map_err(|e| OmniError::Lance(e.to_string()))?; return Ok(()); } - // Slow path: per-row inner pipeline execution + // Slow path (filtered / non-bulk inner): run the inner pipeline ONCE over the + // whole frontier β€” a set-oriented anti-semi-join β€” instead of row-by-row. + // Each outer row is tagged with a synthetic index; an outer row matches iff + // it produced at least one surviving inner row. No per-row dispatch, so the + // inner Expand runs as a single set-at-a-time traversal over the full + // frontier (its own chooser picks indexed vs CSR) rather than one Lance scan + // per outer row. let num_rows = wide.num_rows(); - let mut keep_mask = vec![true; num_rows]; + if num_rows == 0 { + return Ok(()); + } - for i in 0..num_rows { - let single_row = wide.slice(i, 1); - let mut inner_wide: Option = Some(single_row); + // The tag rides through the inner pipeline: Expand's hconcat preserves + // existing columns and Filter only drops rows, so each surviving row carries + // its originating outer-row index. Correlating on the row index (not + // `outer_var.id`) stays correct even if a dst-filter references other outer + // bindings. Nested anti-joins reuse this slow path and an enclosing tag rides + // through too; Arrow allows duplicate field names and `column_by_name` + // returns the FIRST match, so choose a tag name not already present (each + // nesting level then reads its own) instead of a fixed one. + let tag_col: String = { + let mut n = 0usize; + loop { + let candidate = format!("__antijoin_outer_row_{n}"); + if wide.schema().column_with_name(&candidate).is_none() { + break candidate; + } + n += 1; + } + }; + let mut fields: Vec = wide + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + fields.push(Field::new(tag_col.as_str(), DataType::UInt32, false)); + let mut columns: Vec = wide.columns().to_vec(); + columns.push(Arc::new(UInt32Array::from_iter_values(0..num_rows as u32))); + let tagged = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) + .map_err(|e| OmniError::Lance(e.to_string()))?; - let no_search = SearchMode::default(); - execute_pipeline( - inner_pipeline, - params, - snapshot, - graph_index, - catalog, - &mut inner_wide, - &no_search, - ) - .await?; + let mut inner_wide: Option = Some(tagged); + let no_search = SearchMode::default(); + execute_pipeline( + inner_pipeline, + params, + snapshot, + graph_index, + catalog, + &mut inner_wide, + &no_search, + ) + .await?; - let has_match = inner_wide - .as_ref() - .map(|batch| batch.num_rows() > 0) - .unwrap_or(false); - - if has_match { - keep_mask[i] = false; + // Outer rows whose tag survived have >= 1 match. A produced-but-untagged + // batch means the inner pipeline dropped the correlation column β€” fail loudly + // rather than silently keeping every row (which would corrupt the anti-join). + let mut matched: HashSet = HashSet::new(); + if let Some(batch) = inner_wide { + if batch.num_rows() > 0 { + let tags = batch + .column_by_name(tag_col.as_str()) + .ok_or_else(|| { + OmniError::manifest( + "anti-join inner pipeline dropped the correlation column".to_string(), + ) + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| { + OmniError::manifest(format!("'{}' column is not UInt32", tag_col)) + })?; + for i in 0..tags.len() { + matched.insert(tags.value(i)); + } } } + let keep_mask: Vec = (0..num_rows as u32).map(|i| !matched.contains(&i)).collect(); let mask = BooleanArray::from(keep_mask); *wide = arrow_select::filter::filter_record_batch(wide, &mask) .map_err(|e| OmniError::Lance(e.to_string()))?; @@ -1186,45 +1878,6 @@ fn add_null_blob_columns( .map_err(|e| OmniError::Lance(e.to_string())) } -/// Convert IR filters to a Lance SQL filter string. -fn build_lance_filter(filters: &[IRFilter], params: &ParamMap) -> Option { - if filters.is_empty() { - return None; - } - - let parts: Vec = filters - .iter() - .filter_map(|f| ir_filter_to_sql(f, params)) - .collect(); - - if parts.is_empty() { - return None; - } - - Some(parts.join(" AND ")) -} - -fn ir_filter_to_sql(filter: &IRFilter, params: &ParamMap) -> Option { - // Search predicates (search/fuzzy/match_text = true) are NOT converted to SQL. - // They are handled via scanner.full_text_search() in execute_node_scan. - if is_search_filter(filter) { - return None; - } - - let left = ir_expr_to_sql(&filter.left, params)?; - let right = ir_expr_to_sql(&filter.right, params)?; - let op = match filter.op { - CompOp::Eq => "=", - CompOp::Ne => "!=", - CompOp::Gt => ">", - CompOp::Lt => "<", - CompOp::Ge => ">=", - CompOp::Le => "<=", - CompOp::Contains => return None, // Can't pushdown list contains - }; - Some(format!("{} {} {}", left, op, right)) -} - /// Build a FullTextSearchQuery from a search IR expression. fn build_fts_query( expr: &IRExpr, @@ -1297,15 +1950,6 @@ fn resolve_to_int(expr: &IRExpr, params: &ParamMap) -> Option { } } -fn ir_expr_to_sql(expr: &IRExpr, params: &ParamMap) -> Option { - match expr { - IRExpr::PropAccess { property, .. } => Some(property.clone()), - IRExpr::Literal(lit) => Some(literal_to_sql(lit)), - IRExpr::Param(name) => params.get(name).map(literal_to_sql), - _ => None, - } -} - pub(super) fn literal_to_sql(lit: &Literal) -> String { match lit { Literal::Null => "NULL".to_string(), @@ -1336,10 +1980,10 @@ pub(super) fn literal_to_sql(lit: &Literal) -> String { // // Search predicates (`is_search_filter`) are still handled separately via // `scanner.full_text_search(...)`, not via filter_expr β€” they stay None -// here just like in `ir_filter_to_sql`. The `literal_to_sql` path remains -// because the mutation/update layer (`exec/mutation.rs`) still produces -// SQL strings for `Dataset::delete(&str)`; that migration is MR-A's -// territory (Lance #6658 + delete two-phase). +// here (search predicates are never lowered to a scalar filter). The +// `literal_to_sql` path remains because the mutation/update layer +// (`exec/mutation.rs`) still produces SQL strings for `Dataset::delete(&str)`; +// that migration is MR-A's territory (Lance #6658 + delete two-phase). /// Convert IR filters to a single DataFusion `Expr` (AND-joined), or /// `None` if no filter is pushable. @@ -1381,8 +2025,8 @@ pub(super) fn ir_filter_to_expr( } // List-contains: `prop CONTAINS value` lowers to `array_has(prop, value)`. - // This is the case `ir_filter_to_sql` had to return None for ("Can't - // pushdown list contains"); with structured Expr it pushes down fine. + // This is the case the old SQL-string pushdown had to return None for + // ("Can't pushdown list contains"); with structured Expr it pushes down fine. if matches!(filter.op, CompOp::Contains) { let left = ir_expr_to_expr(&filter.left, params)?; let right = ir_expr_to_expr(&filter.right, params)?; @@ -1517,3 +2161,127 @@ fn take_batch(batch: &RecordBatch, indices: &UInt32Array) -> Result .map_err(|e| OmniError::Lance(e.to_string()))?; RecordBatch::try_new(batch.schema(), columns).map_err(|e| OmniError::Lance(e.to_string())) } + +#[cfg(test)] +mod expand_chooser_tests { + use super::*; + use crate::table_store::IndexCoverage; + + /// Build cost inputs with generous hard caps, so the cost comparison (not a + /// ceiling) is what the assertions exercise unless a test sets one on purpose. + fn inputs( + frontier_rows: usize, + edge_count: u64, + src_node_count: u64, + effective_max_hops: u32, + coverage: IndexCoverage, + ) -> ExpandCostInputs { + ExpandCostInputs { + frontier_rows, + edge_count, + src_node_count, + effective_max_hops, + max_hops_cap: 6, + max_frontier_cap: 1024, + coverage, + csr_cached: false, + } + } + + #[test] + fn selective_frontier_on_large_graph_picks_indexed() { + // 50 source rows against 1M source vertices, one hop: tiny selectivity β€” + // the PR #149 win the chooser must preserve. + let m = choose_expand_mode(&inputs(50, 10_000_000, 1_000_000, 1, IndexCoverage::Indexed)); + assert_eq!(m, ExpandMode::IndexedScan); + } + + #[test] + fn flat_in_edge_count_same_selectivity_same_choice() { + // Same selectivity (frontier/|V_src|), 1000Γ— difference in |E|. Indexed + // cost is independent of |E|, so the choice must not flip. + let small = choose_expand_mode(&inputs(50, 100_000, 1_000_000, 1, IndexCoverage::Indexed)); + let huge = + choose_expand_mode(&inputs(50, 100_000_000, 1_000_000, 1, IndexCoverage::Indexed)); + assert_eq!(small, ExpandMode::IndexedScan); + assert_eq!(huge, ExpandMode::IndexedScan); + } + + #[test] + fn frontier_large_fraction_of_source_picks_csr() { + // hops*frontier (200) exceeds BUILD_FACTOR*|V_src| (1.5*100=150) β†’ CSR, + // and 200 is below the frontier cap, so it is the cost model deciding. + let m = choose_expand_mode(&inputs(200, 1_000, 100, 1, IndexCoverage::Indexed)); + assert_eq!(m, ExpandMode::Csr); + } + + #[test] + fn frontier_over_hard_cap_picks_csr() { + // 2000 > 1024 ceiling, even though the selectivity is tiny. + let m = choose_expand_mode(&inputs(2000, 10_000_000, 1_000_000, 1, IndexCoverage::Indexed)); + assert_eq!(m, ExpandMode::Csr); + } + + #[test] + fn hops_over_hard_cap_picks_csr() { + let m = choose_expand_mode(&inputs(10, 10_000_000, 1_000_000, 8, IndexCoverage::Indexed)); + assert_eq!(m, ExpandMode::Csr); + } + + #[test] + fn degraded_single_hop_tiny_frontier_stays_indexed() { + // One full degraded scan (1*|E|) still edges out a full CSR build + // (1.5*|E|) for a one-off single hop. + let m = choose_expand_mode(&inputs( + 5, + 10_000, + 10_000, + 1, + IndexCoverage::Degraded { + reason: "no btree".into(), + }, + )); + assert_eq!(m, ExpandMode::IndexedScan); + } + + #[test] + fn degraded_multi_hop_picks_csr() { + // Two degraded scans (2*|E|) lose to one CSR build (1.5*|E|). + let m = choose_expand_mode(&inputs( + 5, + 10_000, + 10_000, + 2, + IndexCoverage::Degraded { + reason: "no btree".into(), + }, + )); + assert_eq!(m, ExpandMode::Csr); + } + + #[test] + fn warm_csr_is_always_reused() { + // A maximally selective traversal still prefers an already-built CSR + // (cost ~0) over re-scanning per hop. + let mut i = inputs(1, 10_000_000, 1_000_000, 1, IndexCoverage::Indexed); + i.csr_cached = true; + assert_eq!(choose_expand_mode(&i), ExpandMode::Csr); + } + + #[test] + fn cost_model_caps_cross_type_hops() { + // Same-type passes the requested range through; cross-type caps at 1, + // matching execute_expand_indexed. + assert_eq!(cost_effective_hops(5, true), 5); + assert_eq!(cost_effective_hops(5, false), 1); + assert_eq!(cost_effective_hops(1, false), 1); + + // Consequence: a selective frontier where the requested 5 hops would + // (wrongly) flip cross-type to CSR, but the capped 1 hop β€” what actually + // runs β€” keeps it indexed. + let mut i = inputs(50, 10_000, 100, cost_effective_hops(5, false), IndexCoverage::Indexed); + assert_eq!(choose_expand_mode(&i), ExpandMode::IndexedScan); + i.effective_max_hops = 5; // as if the cross-type cap were not applied + assert_eq!(choose_expand_mode(&i), ExpandMode::Csr); + } +} diff --git a/crates/omnigraph/src/exec/staging.rs b/crates/omnigraph/src/exec/staging.rs index 0d26fd3..264ab59 100644 --- a/crates/omnigraph/src/exec/staging.rs +++ b/crates/omnigraph/src/exec/staging.rs @@ -495,25 +495,21 @@ impl StagedMutation { // until `ensure_path` learns how to bump expected_version on // op-kind upgrade. // - // Why per-branch (and not the bound-branch `db.snapshot()`): - // when the caller mutates a branch other than the engine's - // bound branch (e.g., feature-branch ingest from a server - // handle bound to main), `db.snapshot()` returns the bound - // branch's view of each table β€” which is the wrong pin for - // the publisher's CAS on a different branch. Using - // `snapshot_for_branch(branch)` resolves the per-branch - // entries correctly. The cost is one fresh manifest read per - // mutation; PR 1b's regression came from this same read, but - // that read is now strictly necessary for cross-branch - // correctness. Single-table same-branch mutations could still - // skip this read (queue exclusivity makes the publisher CAS a - // no-op), but the conditional adds complexity for marginal - // gain β€” left as a follow-up perf optimization. + // Why a fresh per-branch snapshot (and not the bound-branch + // `db.snapshot()` / `snapshot_for_branch()` fast path): a stale + // engine handle may be bound to the same branch it is writing. For + // non-strict Insert/Merge, that stale local view is allowed to rebase + // to the live manifest pin under the queue; only uncovered Lance + // HEAD>manifest drift is refused. For writes targeting a branch other + // than the engine's bound branch (e.g., feature-branch ingest from a + // server handle bound to main), the same helper also resolves the + // correct branch pin. The cost is one fresh manifest read per mutation + // plus one Lance HEAD open per staged table for the drift guard below. // // Multi-coordinator deployments (Β§VI.27 aspirational) get // genuine cross-process drift detection from this read for // free. - let snapshot = db.snapshot_for_branch(branch).await?; + let snapshot = db.fresh_snapshot_for_branch(branch).await?; for entry in staged.iter_mut() { let current = snapshot .entry(&entry.table_key) @@ -541,6 +537,35 @@ impl StagedMutation { )); } + // Separate manifest-visible concurrency from uncovered Lance drift. + // Non-strict inserts/merges are allowed to rebase from their staged + // read version to the fresh manifest pin above, but only if the + // live Lance HEAD still equals that manifest pin. If an external + // raw Lance write or a pre-fix maintenance path moved HEAD without + // publishing `__manifest`, this write must not silently fold it. + let head = db + .table_store() + .open_dataset_head_for_write( + &entry.table_key, + &entry.path.full_path, + entry.path.table_branch.as_deref(), + ) + .await? + .version() + .version; + if head < current { + return Err(OmniError::manifest_internal(format!( + "table '{}' Lance HEAD version {} is behind manifest version {}", + entry.table_key, head, current + ))); + } + if head > current { + return Err(OmniError::manifest_conflict(format!( + "table '{}' has Lance HEAD version {} ahead of manifest version {}; run `omnigraph repair` before writing", + entry.table_key, head, current + ))); + } + entry.expected_version = current; expected_versions.insert(entry.table_key.clone(), current); } diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index 46a46e2..707c46a 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -288,21 +288,24 @@ async fn load_jsonl_reader( let mut node_rows: HashMap> = HashMap::new(); let mut edge_rows: HashMap> = HashMap::new(); - for (line_num, line) in reader.lines().enumerate() { - let line = line?; - let line = line.trim(); - if line.is_empty() { - continue; - } - let value: JsonValue = serde_json::from_str(line).map_err(|e| { - OmniError::manifest(format!("invalid JSON on line {}: {}", line_num + 1, e)) + // Parse a stream of JSON values. Accepts both compact JSONL (one object + // per line) and pretty-printed JSON where a single object spans multiple + // lines β€” serde's streaming deserializer treats any whitespace (including + // newlines) between top-level values as a separator. + for (idx, parsed) in serde_json::Deserializer::from_reader(reader) + .into_iter::() + .enumerate() + { + let record_num = idx + 1; + let value: JsonValue = parsed.map_err(|e| { + OmniError::manifest(format!("invalid JSON at record {}: {}", record_num, e)) })?; if let Some(type_name) = value.get("type").and_then(|v| v.as_str()) { if !catalog.node_types.contains_key(type_name) { return Err(OmniError::manifest(format!( - "line {}: unknown node type '{}'", - line_num + 1, + "record {}: unknown node type '{}'", + record_num, type_name ))); } @@ -317,8 +320,8 @@ async fn load_jsonl_reader( } else if let Some(edge_name) = value.get("edge").and_then(|v| v.as_str()) { if catalog.lookup_edge_by_name(edge_name).is_none() { return Err(OmniError::manifest(format!( - "line {}: unknown edge type '{}'", - line_num + 1, + "record {}: unknown edge type '{}'", + record_num, edge_name ))); } @@ -326,14 +329,14 @@ async fn load_jsonl_reader( .get("from") .and_then(|v| v.as_str()) .ok_or_else(|| { - OmniError::manifest(format!("line {}: edge missing 'from'", line_num + 1)) + OmniError::manifest(format!("record {}: edge missing 'from'", record_num)) })? .to_string(); let to = value .get("to") .and_then(|v| v.as_str()) .ok_or_else(|| { - OmniError::manifest(format!("line {}: edge missing 'to'", line_num + 1)) + OmniError::manifest(format!("record {}: edge missing 'to'", record_num)) })? .to_string(); let data = value @@ -347,8 +350,8 @@ async fn load_jsonl_reader( .push((from, to, data)); } else { return Err(OmniError::manifest(format!( - "line {}: expected 'type' or 'edge' field", - line_num + 1 + "record {}: expected 'type' or 'edge' field", + record_num ))); } } @@ -396,9 +399,9 @@ async fn load_jsonl_reader( let batch = build_node_batch(node_type, rows)?; validate_value_constraints(&batch, node_type)?; validate_enum_constraints(&batch, &node_type.properties, type_name)?; - let unique_props = unique_property_names_for_node(node_type); - if !unique_props.is_empty() { - enforce_unique_constraints_intra_batch(&batch, type_name, &unique_props)?; + let unique_groups = unique_constraint_groups_for_node(node_type); + if !unique_groups.is_empty() { + enforce_unique_constraints_intra_batch(&batch, type_name, &unique_groups)?; } let loaded_count = batch.num_rows(); let table_key = format!("node:{}", type_name); @@ -507,9 +510,9 @@ async fn load_jsonl_reader( let edge_type = &catalog.edge_types[edge_name]; let batch = build_edge_batch(edge_type, rows)?; validate_enum_constraints(&batch, &edge_type.properties, edge_name)?; - let unique_props = unique_property_names_for_edge(edge_type); - if !unique_props.is_empty() { - enforce_unique_constraints_intra_batch(&batch, edge_name, &unique_props)?; + let unique_groups = unique_constraint_groups_for_edge(edge_type); + if !unique_groups.is_empty() { + enforce_unique_constraints_intra_batch(&batch, edge_name, &unique_groups)?; } let loaded_count = batch.num_rows(); let table_key = format!("edge:{}", edge_name); @@ -1422,8 +1425,16 @@ pub(crate) fn validate_enum_constraints( Ok(()) } -/// Detect duplicate values within a single `RecordBatch` for any of the named -/// `unique_properties`. Returns an error on the first duplicate found. +/// Detect duplicate values within a single `RecordBatch` for any of the +/// `unique_constraints` groups. Each group is a list of one or more columns +/// that together form a uniqueness key: a violation occurs when two rows share +/// the same tuple of values across *all* columns in a group, so a composite +/// `@unique(a, b)` only conflicts when both `a` and `b` match. Returns an +/// error on the first duplicate found. +/// +/// Rows where any column in a group is null are exempt (standard SQL semantics +/// for uniqueness over nullable columns), as is any group whose columns are +/// not all present in the batch (e.g. a partial-schema load). /// /// Note: this only catches duplicates *within* the batch. Cross-batch /// uniqueness against already-committed rows is not enforced here β€” that @@ -1431,22 +1442,37 @@ pub(crate) fn validate_enum_constraints( pub(crate) fn enforce_unique_constraints_intra_batch( batch: &RecordBatch, type_name: &str, - unique_properties: &[String], + unique_constraints: &[Vec], ) -> Result<()> { - for property in unique_properties { - let Some(col_idx) = batch.schema().index_of(property).ok() else { + for columns in unique_constraints { + // Resolve the group's columns once. A group whose columns aren't all + // present in this batch is skipped (e.g. a partial-schema load). + let Some(group_columns) = columns + .iter() + .map(|name| { + batch + .schema() + .index_of(name) + .ok() + .map(|i| batch.column(i).clone()) + }) + .collect::>>() + else { continue; }; - let arr = batch.column(col_idx); - let mut seen: HashMap = HashMap::new(); + let mut seen: HashMap, usize> = HashMap::new(); for row in 0..batch.num_rows() { - let Some(value) = scalar_to_string(arr, row) else { + let Some(key) = composite_unique_key(&group_columns, row)? else { continue; }; - if let Some(prev_row) = seen.insert(value.clone(), row) { + if let Some(prev_row) = seen.insert(key.clone(), row) { return Err(OmniError::manifest(format!( "@unique violation on {}.{}: value '{}' appears in rows {} and {}", - type_name, property, value, prev_row, row + type_name, + format_tuple(columns), + format_tuple(&key), + prev_row, + row ))); } } @@ -1454,80 +1480,131 @@ pub(crate) fn enforce_unique_constraints_intra_batch( Ok(()) } -/// Reduce a single Arrow scalar at (`array`, `row`) to a `String` for -/// uniqueness comparison. Returns `None` for null values (nulls are exempt -/// from uniqueness in standard SQL semantics). -fn scalar_to_string(array: &ArrayRef, row: usize) -> Option { - use arrow_array::Array; +/// Build the composite uniqueness key for `row` over a constraint group's +/// already-resolved columns (in declaration order). +/// +/// The key is the *tuple* of per-column scalar strings (`Vec`), keyed +/// directly in the dedup map β€” there is no separator, so no data value can +/// forge a collision (an earlier version joined on `U+001F`, which a value +/// containing that control char could still defeat). +/// +/// - `Ok(None)` if any column is null: the row is exempt (a partial tuple +/// can't violate uniqueness under SQL null semantics). +/// - `Ok(Some(tuple))` otherwise. +/// - `Err(..)` propagated from [`unique_key_scalar`] on an un-keyable value. +/// +/// Shared by the intake path (`enforce_unique_constraints_intra_batch`) and the +/// branch-merge path (`exec/merge.rs::update_unique_constraints`) so the two +/// derive identical keys and cannot drift on separator or scalar conversion. +pub(crate) fn composite_unique_key( + group_columns: &[ArrayRef], + row: usize, +) -> Result>> { + let mut parts = Vec::with_capacity(group_columns.len()); + for column in group_columns { + match unique_key_scalar(column, row)? { + Some(value) => parts.push(value), + None => return Ok(None), + } + } + Ok(Some(parts)) +} + +/// Render a constraint's column tuple for error messages: a single item as +/// `col`, a composite as `(a, b)`. Used for both the column list and the +/// offending value tuple, which share the same shape. +fn format_tuple(items: &[String]) -> String { + match items { + [single] => single.clone(), + _ => format!("({})", items.join(", ")), + } +} + +/// Reduce a single Arrow scalar at (`array`, `row`) to its uniqueness-key +/// string. +/// +/// - `Ok(None)` for a null value: nulls are exempt from uniqueness (standard +/// SQL semantics over nullable columns). +/// - `Ok(Some(s))` for every scalar type a `@unique` / `@key` column can hold. +/// Strings are covered in all three physical Arrow encodings (`Utf8`, +/// `LargeUtf8`, `Utf8View`), so a legal string column is always keyable +/// regardless of how Lance materializes it on read-back. +/// - `Err(..)` for a non-null value whose Arrow type can't be reduced to a key +/// (a list, blob, or vector column). This fails loudly rather than silently +/// exempting the row, and because every legal scalar encoding is handled +/// above, the error fires only for a genuinely un-keyable column type β€” never +/// for a legal value that merely arrived in an unenumerated encoding. +fn unique_key_scalar(array: &ArrayRef, row: usize) -> Result> { + use arrow_array::{Array, LargeStringArray, StringViewArray}; if array.is_null(row) { - return None; + return Ok(None); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); + } + if let Some(a) = array.as_any().downcast_ref::() { + return Ok(Some(a.value(row).to_string())); + } + if let Some(a) = array.as_any().downcast_ref::() { + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } if let Some(a) = array.as_any().downcast_ref::() { - return Some(a.value(row).to_string()); + return Ok(Some(a.value(row).to_string())); } - None + Err(OmniError::manifest(format!( + "uniqueness key: unsupported column type {:?} for @unique/@key enforcement", + array.data_type() + ))) } -/// Build the flat list of property names that must be checked for uniqueness -/// on a node type. Includes both `@unique` properties (from -/// `NodeType.unique_constraints`) and the `@key` (which implies uniqueness). -pub(crate) fn unique_property_names_for_node( +/// Build the list of uniqueness constraint groups to enforce on a node type. +/// Each group is the column tuple of one constraint. Includes every +/// `@unique(...)` constraint (from `NodeType.unique_constraints`) and the +/// `@key` (which implies uniqueness over its column tuple). Grouping is +/// preserved so a composite `@unique(a, b)` is enforced as a composite key +/// rather than degraded into independent single-field checks. +pub(crate) fn unique_constraint_groups_for_node( node_type: &omnigraph_compiler::catalog::NodeType, -) -> Vec { - let mut props: Vec = node_type - .unique_constraints - .iter() - .flatten() - .cloned() - .collect(); - if let Some(key) = &node_type.key { - props.extend(key.iter().cloned()); +) -> Vec> { + let mut groups: Vec> = node_type.unique_constraints.clone(); + if let Some(key) = &node_type.key + && !groups.contains(key) + { + groups.push(key.clone()); } - props.sort(); - props.dedup(); - props + groups } -/// Same as [`unique_property_names_for_node`] but for an edge type. -pub(crate) fn unique_property_names_for_edge( +/// Same as [`unique_constraint_groups_for_node`] but for an edge type (edges +/// have no `@key`). +pub(crate) fn unique_constraint_groups_for_edge( edge_type: &omnigraph_compiler::catalog::EdgeType, -) -> Vec { - let mut props: Vec = edge_type - .unique_constraints - .iter() - .flatten() - .cloned() - .collect(); - props.sort(); - props.dedup(); - props +) -> Vec> { + edge_type.unique_constraints.clone() } fn extract_numeric_value(col: &ArrayRef, row: usize) -> Option { @@ -2169,4 +2246,66 @@ edge WorksAt: Person -> Company let err = result.unwrap_err().to_string(); assert!(err.contains("NaN"), "error should mention NaN: {}", err); } + + #[test] + fn composite_unique_key_builds_tuple_and_exempts_null() { + let a: ArrayRef = Arc::new(StringArray::from(vec![Some("x|y"), Some("x"), None])); + let b: ArrayRef = Arc::new(StringArray::from(vec![Some("z"), Some("y|z"), Some("q")])); + let cols = [a, b]; + + // Tuple key, so `("x|y", "z")` and `("x", "y|z")` stay distinct β€” + // a separator-joined key (the old `|` join) would collapse both to + // `x|y|z`. + assert_eq!( + composite_unique_key(&cols, 0).unwrap(), + Some(vec!["x|y".to_string(), "z".to_string()]) + ); + assert_eq!( + composite_unique_key(&cols, 1).unwrap(), + Some(vec!["x".to_string(), "y|z".to_string()]) + ); + assert_ne!( + composite_unique_key(&cols, 0).unwrap(), + composite_unique_key(&cols, 1).unwrap() + ); + + // Any null column β†’ the whole row is exempt (SQL null semantics). + assert_eq!(composite_unique_key(&cols, 2).unwrap(), None); + } + + #[test] + fn unique_key_scalar_errors_loudly_on_unkeyable_type() { + use arrow_array::LargeBinaryArray; + // A binary/blob column can't be reduced to a uniqueness key. Before the + // hardening this returned `None`, so a `@unique` on such a column was + // silently un-enforced; now it errors instead of weakening the + // constraint in silence. + let blob: ArrayRef = Arc::new(LargeBinaryArray::from(vec![Some(&b"abc"[..])])); + let err = unique_key_scalar(&blob, 0).unwrap_err(); + assert!( + err.to_string().contains("unsupported column type"), + "un-keyable type must fail loudly (got: {err})" + ); + } + + #[test] + fn unique_key_scalar_handles_all_string_encodings() { + use arrow_array::{LargeStringArray, StringViewArray}; + // A legal string column is keyable in every physical Arrow encoding + // Lance might hand back (Utf8 / LargeUtf8 / Utf8View). None of these may + // fall through to the loud `Err` path β€” that branch is reserved for + // genuinely un-keyable column types, not a legal value in an + // unenumerated encoding. + let utf8: ArrayRef = Arc::new(StringArray::from(vec![Some("v")])); + let large: ArrayRef = Arc::new(LargeStringArray::from(vec![Some("v")])); + let view: ArrayRef = Arc::new(StringViewArray::from(vec![Some("v")])); + for array in [&utf8, &large, &view] { + assert_eq!( + unique_key_scalar(array, 0).unwrap(), + Some("v".to_string()), + "string array {:?} must render, not error", + array.data_type() + ); + } + } } diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index 10123b0..d786fc4 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -43,6 +43,19 @@ pub struct DeleteState { pub(crate) version_metadata: TableVersionMetadata, } +/// Whether a `key_col IN (...)` scan on a dataset will be served by the +/// persisted scalar (BTREE) index, or silently fall back to a full filtered +/// scan. Detection-only (metadata, no IO); the scan returns the correct rows +/// either way. Surfaced by the indexed traversal path so the silent perf +/// fallback is observable, and available to a future cost-based planner. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IndexCoverage { + /// The column has a usable BTREE and every fragment records `physical_rows`. + Indexed, + /// Lance will not use the scalar index for this scan (correct, full scan). + Degraded { reason: String }, +} + /// A Lance write that has produced fragment files on object storage but is /// not yet committed to the dataset's manifest. The staged-write primitives /// are consumed by `MutationStaging` (`exec/staging.rs`, @@ -582,6 +595,117 @@ impl TableStore { .map_err(|e| OmniError::Lance(e.to_string())) } + /// Indexed neighbor lookup for graph traversal. Given an edge dataset and a + /// set of endpoint keys on `key_col` (`"src"` for out-traversal, `"dst"` for + /// in-traversal), return the matching edge rows projected to + /// `[key_col, opposite_col]`. + /// + /// The `key_col IN (keys)` predicate is built as a structured DataFusion + /// `Expr` and applied via `Scanner::filter_expr`, so Lance routes it through + /// the persisted BTREE on `key_col` (index-search β†’ take). Cost scales with + /// the frontier size, not |E| β€” the basis for serving selective traversals + /// without building the whole in-memory CSR. Empty `keys` returns empty + /// without scanning. + /// + /// Note: like any indexed scan, this observes only fragments the BTREE + /// covers plus an unindexed-fragment scan fallback; it reads the committed + /// snapshot `ds` was opened at. + pub async fn scan_edges_by_endpoint( + ds: &Dataset, + key_col: &str, + opposite_col: &str, + keys: &[String], + ) -> Result> { + use datafusion::prelude::{col, lit}; + + if keys.is_empty() { + return Ok(Vec::new()); + } + let key_list: Vec = + keys.iter().map(|k| lit(k.clone())).collect(); + let filter_expr = col(key_col).in_list(key_list, false); + Self::scan_stream_with( + ds, + Some(&[key_col, opposite_col]), + None, + None, + false, + |scanner| { + scanner.filter_expr(filter_expr); + Ok(()) + }, + ) + .await? + .try_collect() + .await + .map_err(|e| OmniError::Lance(e.to_string())) + } + + /// Metadata-only check (no IO) of whether `scan_edges_by_endpoint` β€” a + /// `key_col IN (...)` filter β€” on `ds` will be served by the persisted BTREE + /// on `column`, or silently fall back to a full filtered scan. Mirrors + /// Lance's own decision: scalar indices are disabled for the whole scan if + /// ANY fragment lacks `physical_rows` (lance `dataset/scanner.rs` + /// `create_filter_plan`), and are obviously unused if no BTREE on the + /// column exists. The scan is correct (returns all rows) either way β€” this + /// only surfaces the perf cliff so the indexed traversal can warn on it. + pub async fn key_column_index_coverage(ds: &Dataset, column: &str) -> Result { + let Some(field_id) = ds.schema().field(column).map(|field| field.id) else { + return Ok(IndexCoverage::Degraded { + reason: format!("column '{}' not in schema", column), + }); + }; + let indices = ds + .load_indices() + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + let btree = indices + .iter() + .filter(|index| !is_system_index(index)) + .filter(|index| index.fields.len() == 1 && index.fields[0] == field_id) + .find(|index| { + index + .index_details + .as_ref() + .map(|details| details.type_url.ends_with("BTreeIndexDetails")) + .unwrap_or(false) + }); + let Some(btree) = btree else { + return Ok(IndexCoverage::Degraded { + reason: format!("no BTREE index on '{}'", column), + }); + }; + // Same check Lance runs: a fragment missing physical_rows disables + // scalar indices for the entire scan (all-or-nothing). + if ds.fragments().iter().any(|f| f.physical_rows.is_none()) { + return Ok(IndexCoverage::Degraded { + reason: "a fragment is missing physical_rows".to_string(), + }); + } + // An index only covers the fragments it was built over; fragments + // appended afterward (edge-index creation is skipped once a BTREE exists) + // are scanned unindexed. If any CURRENT fragment is absent from the + // index's `fragment_bitmap`, the scan is partly a full scan β€” so the + // chooser must not price it as fully indexed. A `None` bitmap means Lance + // can't report coverage; don't over-degrade in that case. + if let Some(bitmap) = btree.fragment_bitmap.as_ref() { + let uncovered = ds + .fragments() + .iter() + .filter(|f| !bitmap.contains(f.id as u32)) + .count(); + if uncovered > 0 { + return Ok(IndexCoverage::Degraded { + reason: format!( + "{} fragment(s) not covered by the index on '{}'", + uncovered, column + ), + }); + } + } + Ok(IndexCoverage::Indexed) + } + pub async fn count_rows(&self, ds: &Dataset, filter: Option) -> Result { ds.count_rows(filter) .await @@ -732,7 +856,7 @@ impl TableStore { // before the FirstSeen setter has a chance to silently collapse // anything): // - Load path: `enforce_unique_constraints_intra_batch` - // (`loader/mod.rs:1453`) errors on intra-batch `@key` dups. + // (`loader/mod.rs:1442`) errors on intra-batch `@key` dups. // - Mutate path: `MutationStaging::finalize` (`exec/staging.rs`) // accumulates and dedupes by `id`. // - Branch-merge path: `compute_source_delta` / diff --git a/crates/omnigraph/tests/branching.rs b/crates/omnigraph/tests/branching.rs index 5a0c47d..108702c 100644 --- a/crates/omnigraph/tests/branching.rs +++ b/crates/omnigraph/tests/branching.rs @@ -39,6 +39,26 @@ query insert_user($name: String, $email: String) { } "#; +const EDGE_UNIQUE_SCHEMA: &str = r#" +node Person { + name: String @key +} + +edge Knows: Person -> Person { + @unique(src, dst) +} +"#; + +const EDGE_UNIQUE_DATA: &str = r#"{"type":"Person","data":{"name":"Alice"}} +{"type":"Person","data":{"name":"Bob"}} +{"type":"Person","data":{"name":"Carol"}}"#; + +const EDGE_UNIQUE_MUTATIONS: &str = r#" +query add_knows($from: String, $to: String) { + insert Knows { from: $from, to: $to } +} +"#; + const CARDINALITY_SCHEMA: &str = r#" node Person { name: String @key @@ -1119,6 +1139,87 @@ async fn branch_merge_reports_unique_violation_conflict() { } } +/// Regression for the MR-983 follow-up: the branch-merge path must enforce an +/// edge composite `@unique(src, dst)` as a true composite key, consistent with +/// the intake path. Two branches inserting the *same* (src, dst) pair must +/// conflict on merge. +#[tokio::test] +async fn branch_merge_reports_composite_unique_violation_conflict() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut main = init_db_from_schema_and_data(&dir, EDGE_UNIQUE_SCHEMA, EDGE_UNIQUE_DATA).await; + main.branch_create("feature").await.unwrap(); + + let mut feature = Omnigraph::open(uri).await.unwrap(); + + mutate_main( + &mut main, + EDGE_UNIQUE_MUTATIONS, + "add_knows", + ¶ms(&[("$from", "Alice"), ("$to", "Bob")]), + ) + .await + .unwrap(); + + mutate_branch( + &mut feature, + "feature", + EDGE_UNIQUE_MUTATIONS, + "add_knows", + ¶ms(&[("$from", "Alice"), ("$to", "Bob")]), + ) + .await + .unwrap(); + + let err = main.branch_merge("feature", "main").await.unwrap_err(); + match err { + OmniError::MergeConflicts(conflicts) => { + assert!(conflicts.iter().any(|conflict| { + conflict.table_key == "edge:Knows" + && conflict.kind == MergeConflictKind::UniqueViolation + })); + } + other => panic!("expected merge conflicts, got {other:?}"), + } +} + +/// Sibling to the above: pairs sharing `src` but differing on `dst` are unique +/// on the (src, dst) tuple and must merge cleanly. Guards against the composite +/// degrading back into a single-field `@unique(src)` on the merge path. +#[tokio::test] +async fn branch_merge_allows_distinct_composite_unique_pairs() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut main = init_db_from_schema_and_data(&dir, EDGE_UNIQUE_SCHEMA, EDGE_UNIQUE_DATA).await; + main.branch_create("feature").await.unwrap(); + + let mut feature = Omnigraph::open(uri).await.unwrap(); + + mutate_main( + &mut main, + EDGE_UNIQUE_MUTATIONS, + "add_knows", + ¶ms(&[("$from", "Alice"), ("$to", "Bob")]), + ) + .await + .unwrap(); + + mutate_branch( + &mut feature, + "feature", + EDGE_UNIQUE_MUTATIONS, + "add_knows", + ¶ms(&[("$from", "Alice"), ("$to", "Carol")]), + ) + .await + .unwrap(); + + main.branch_merge("feature", "main") + .await + .expect("distinct (src, dst) pairs are unique on the composite and must merge cleanly"); + assert_eq!(count_rows(&main, "edge:Knows").await, 2); +} + #[tokio::test] async fn branch_merge_reports_cardinality_violation_conflict() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph/tests/composite_flow.rs b/crates/omnigraph/tests/composite_flow.rs index 6c720da..dd41310 100644 --- a/crates/omnigraph/tests/composite_flow.rs +++ b/crates/omnigraph/tests/composite_flow.rs @@ -294,21 +294,19 @@ async fn composite_flow_canonical_lifecycle() { ); // ───────────────────────────────────────────────────────────────── - // Step 10: optimize the post-merge graph β€” verify indices stay - // valid and queryable. + // Step 10: optimize the post-merge graph β€” verify compaction is + // published to the manifest (so the manifest pin tracks the compacted + // Lance HEAD), indices stay valid and queryable, and a post-optimize + // strict write commits. // - // **Known limitation**: `optimize_all_tables` calls Lance - // `compact_files` directly β€” it advances per-table Lance HEAD - // without updating the omnigraph `__manifest` pin. After optimize, - // the next writer's expected_table_versions captures the - // pre-optimize manifest pin, but the publisher's pre-check reads - // a higher version from the manifest dataset (because some other - // path β€” possibly schema-state recovery on reopen β€” wrote a newer - // __manifest row). The `ExpectedVersionMismatch` is benign - // (re-issuing the mutation after a snapshot refresh succeeds), but - // a composite test cannot reliably exercise post-optimize mutations - // until that path is investigated. Coverage of post-optimize - // mutations is left to a focused optimize+cleanup integration test. + // This step used to carry a "Known limitation": `optimize_all_tables` + // ran Lance `compact_files` without publishing the new version to + // `__manifest`, so the manifest pin lagged the Lance HEAD and the next + // strict write / schema apply failed with `ExpectedVersionMismatch` + // ("stale view … refresh and retry") β€” so post-optimize mutations were + // deliberately omitted here. optimize now publishes the compacted + // version, and this flow exercises exactly that previously-failing + // write below. // ───────────────────────────────────────────────────────────────── let optimize_stats = db.optimize().await.unwrap(); assert!( @@ -331,6 +329,28 @@ async fn composite_flow_canonical_lifecycle() { "row counts unchanged by optimize" ); + // A strict update on a compacted table is exactly the write that + // failed with "stale view" before optimize published its compaction. + // It must now commit (Alice is one of the seed Persons; an update + // leaves the row count at 6). + let post_optimize_update = mutate_main( + &mut db, + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Alice")], &[("$age", 41)]), + ) + .await + .expect("post-optimize strict update must commit β€” optimize published the manifest"); + assert_eq!( + post_optimize_update.affected_nodes, 1, + "post-optimize update must affect exactly Alice" + ); + assert_eq!( + count_rows(&db, "node:Person").await, + 6, + "an update must not change the Person row count" + ); + // ───────────────────────────────────────────────────────────────── // Step 11: cleanup β€” keep last 10 versions, only purge versions // older than 1 hour. With this small test, we have well under 10 @@ -373,14 +393,27 @@ async fn composite_flow_canonical_lifecycle() { branches, ); - // Final query exercise β€” full read path works post-reopen, - // post-cleanup. Post-cleanup mutation is omitted here pending - // resolution of the optimize-vs-manifest-pin interaction documented - // in Step 10. + // Final exercise β€” full read AND write path works post-reopen, + // post-cleanup. (The post-cleanup mutation was previously omitted + // pending resolution of the optimize-vs-manifest-pin interaction in + // Step 10; that is now fixed, so a strict write here must commit.) let final_total = query_main(&mut db, TEST_QUERIES, "total_people", &ParamMap::default()) .await .unwrap(); assert!(!final_total.batches().is_empty()); + + let post_reopen_update = mutate_main( + &mut db, + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Alice")], &[("$age", 42)]), + ) + .await + .expect("post-reopen, post-cleanup strict update must commit"); + assert_eq!( + post_reopen_update.affected_nodes, 1, + "post-reopen update must affect exactly Alice" + ); } /// Cross-handle sequence that exercises operations after a schema_apply diff --git a/crates/omnigraph/tests/consistency.rs b/crates/omnigraph/tests/consistency.rs index 26517db..b16aff9 100644 --- a/crates/omnigraph/tests/consistency.rs +++ b/crates/omnigraph/tests/consistency.rs @@ -188,7 +188,7 @@ node Thing { /// /// Defense in depth: /// 1. The loader's `enforce_unique_constraints_intra_batch` -/// (`loader/mod.rs:1453`), invoked unconditionally on any node type +/// (`loader/mod.rs:1442`), invoked unconditionally on any node type /// with a `@key`, errors on intra-batch duplicate `@key` values at /// intake β€” pinned by this test across every `LoadMode`. /// 2. The `check_batch_unique_by_keys` precondition at the top of @@ -229,6 +229,122 @@ node Thing { } } +/// Regression for MR-983: a node-level composite `@unique(a, b)` must be +/// enforced as a true composite key, not degraded into independent +/// single-field checks. Pre-fix, `unique_property_names_for_node` flattened +/// every constraint group into one property list, so `@unique(source, +/// external_id)` was enforced as `@unique(source)` *and* `@unique(external_id)` +/// β€” rejecting rows that were unique on the composite key and naming only the +/// first field in the error. +#[tokio::test] +async fn loader_enforces_composite_unique_as_composite_key() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let schema = r#" +node ExternalID { + slug: String @key + source: String @index + external_id: String @index + @unique(source, external_id) +} +"#; + let mut db = Omnigraph::init(uri, schema).await.unwrap(); + + // Same `source`, different `external_id` β†’ unique on the composite key. + // This is the exact repro from MR-983 and must be accepted. + let composite_ok = r#"{"type":"ExternalID","data":{"slug":"a","source":"whatsapp","external_id":"+E.164"}} +{"type":"ExternalID","data":{"slug":"b","source":"whatsapp","external_id":"pn:12345"}} +"#; + load_jsonl(&mut db, composite_ok, LoadMode::Overwrite) + .await + .expect("rows unique on the composite (source, external_id) must be accepted"); + assert_eq!(count_rows(&db, "node:ExternalID").await, 2); + + // Both composite columns equal β†’ genuine violation. The error must name + // the whole composite, not just the first field. + let composite_dupe = r#"{"type":"ExternalID","data":{"slug":"c","source":"whatsapp","external_id":"dup"}} +{"type":"ExternalID","data":{"slug":"d","source":"whatsapp","external_id":"dup"}} +"#; + let err = load_jsonl(&mut db, composite_dupe, LoadMode::Overwrite) + .await + .unwrap_err(); + let msg = err.to_string(); + // Columns are canonicalized to sorted order in the catalog, so the + // message reads `(external_id, source)`; assert order-agnostically that + // both composite columns are named (not just the first, as pre-fix). + assert!( + msg.contains("@unique violation") + && msg.contains("source") + && msg.contains("external_id"), + "composite violation must name both columns (got: {msg})" + ); +} + +/// Guard: the intake path (load/insert/update) and the branch-merge path must +/// derive the same composite `@unique(a, b)` key, so a pair of rows unique on +/// the tuple is accepted by BOTH. Both paths now key on the tuple itself (no +/// separator), so a value containing any byte β€” including the `|` that an +/// earlier merge-path join used as its separator β€” can't forge a collision. +/// `("x|y", "z")` and `("x", "y|z")` are distinct tuples and must survive a +/// load-on-branch then merge without a phantom `UniqueViolation`. This pins the +/// cross-path consistency against any future drift in the shared keying. +#[tokio::test] +async fn composite_unique_key_is_consistent_across_intake_and_merge() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let schema = r#" +node Item { + slug: String @key + a: String @index + b: String @index + @unique(a, b) +} +"#; + let insert_item = r#" +query insert_item($slug: String, $a: String, $b: String) { + insert Item { slug: $slug, a: $a, b: $b } +} +"#; + let main = Omnigraph::init(uri, schema).await.unwrap(); + main.branch_create("feature").await.unwrap(); + + // Two rows unique on the composite (a, b), where `a`/`b` carry a literal + // `|`. Distinct under a tuple key; identical (`x|y|z`) under a `|`-join. + let feature = Omnigraph::open(uri).await.unwrap(); + feature + .mutate( + "feature", + insert_item, + "insert_item", + ¶ms(&[("$slug", "r1"), ("$a", "x|y"), ("$b", "z")]), + ) + .await + .expect("intake must accept the first composite-unique row"); + feature + .mutate( + "feature", + insert_item, + "insert_item", + ¶ms(&[("$slug", "r2"), ("$a", "x"), ("$b", "y|z")]), + ) + .await + .expect("intake must accept the second composite-unique row (distinct on the tuple)"); + + // The merge re-validates uniqueness over the adopted source rows. Both + // rows are unique on (a, b), so this must merge cleanly with no phantom + // conflict β€” intake and merge must key the tuple identically. + let merge_result = feature.branch_merge("feature", "main").await; + assert!( + merge_result.is_ok(), + "rows unique on the composite (a, b) must merge cleanly; \ + intake and merge must key the tuple the same way (got: {:?})", + merge_result.err() + ); + + let reopened = Omnigraph::open(uri).await.unwrap(); + assert_eq!(count_rows(&reopened, "node:Item").await, 2); +} + /// Canary for the upstream Lance gap that the `FirstSeen` workaround /// in `table_store.rs` masks. The bug class is "Window 2": load β†’ /// indices built explicitly β†’ merge β†’ merge. Even with the engine diff --git a/crates/omnigraph/tests/end_to_end.rs b/crates/omnigraph/tests/end_to_end.rs index a0fdb0e..ea11d0e 100644 --- a/crates/omnigraph/tests/end_to_end.rs +++ b/crates/omnigraph/tests/end_to_end.rs @@ -1933,3 +1933,87 @@ query docs_with_tag($tag: String) { "contains-pushdown should return exactly the rows whose tags list contains 'red'" ); } + +// ─── Maintenance in the full lifecycle: optimize (compaction) ──────────────── + +/// `optimize` (Lance compaction) is part of a realistic graph lifecycle: it +/// advances the Lance HEAD and publishes the compacted version to the manifest. +/// The rest of the flow must keep working across that boundary β€” reads observe +/// the compacted data, strict updates (which check Lance HEAD == manifest +/// version) still commit, inserts still commit, and the state survives a reopen +/// (the open-time recovery sweep finds no leftover drift). Before optimize +/// published its compaction, the manifest lagged the Lance HEAD here and the +/// post-optimize update below failed with "stale view ... refresh and retry". +#[tokio::test] +async fn full_flow_optimize_then_query_update_and_reopen() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let mut db = init_and_load(&dir).await; + + // Build several Person fragments so compaction has something to merge. + for (name, age) in [("Eve", 40), ("Frank", 41), ("Grace", 42)] { + mutate_main( + &mut db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", name)], &[("$age", age)]), + ) + .await + .unwrap(); + } + + let stats = db.optimize().await.unwrap(); + assert!( + stats.iter().any(|s| s.committed), + "a multi-fragment table should have compacted in this flow" + ); + + // Reads observe the compacted data. + let qr = query_main( + &mut db, + TEST_QUERIES, + "get_person", + ¶ms(&[("$name", "Alice")]), + ) + .await + .unwrap(); + assert_eq!(qr.num_rows(), 1); + + // Strict update after optimize commits (previously failed with "stale view" + // because the manifest lagged the compacted Lance HEAD). + let upd = mutate_main( + &mut db, + MUTATION_QUERIES, + "set_age", + &mixed_params(&[("$name", "Alice")], &[("$age", 31)]), + ) + .await + .unwrap(); + assert_eq!(upd.affected_nodes, 1); + + // Insert after optimize also commits. + mutate_main( + &mut db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "Ivan")], &[("$age", 50)]), + ) + .await + .unwrap(); + assert_eq!(count_rows(&db, "node:Person").await, 8); // 4 seed + Eve/Frank/Grace + Ivan + + // State survives a reopen β€” the recovery sweep runs and finds no drift. + drop(db); + let reopened = Omnigraph::open(&uri).await.unwrap(); + assert_eq!(count_rows(&reopened, "node:Person").await, 8); + let alice = reopened + .entity_at_target(ReadTarget::branch("main"), "node:Person", "Alice") + .await + .unwrap() + .unwrap(); + assert_eq!( + alice["age"], + serde_json::json!(31), + "Alice's post-optimize age update must persist across reopen" + ); +} diff --git a/crates/omnigraph/tests/failpoints.rs b/crates/omnigraph/tests/failpoints.rs index 149c63a..d240108 100644 --- a/crates/omnigraph/tests/failpoints.rs +++ b/crates/omnigraph/tests/failpoints.rs @@ -1245,7 +1245,7 @@ async fn refresh_defers_rollback_eligible_sidecar_to_next_open() { // the rollback (will use Dataset::restore safely; no concurrent // writers at open time). drop(db); - let _db = Omnigraph::open(&uri).await.unwrap(); + let db = Omnigraph::open(&uri).await.unwrap(); // After full-sweep recovery, the sidecar should be processed // (deleted). Sidecar's tables are eligible for rollback (UnexpectedAtP1): // restore happens on Person (HEAD advances by 1). @@ -1268,6 +1268,19 @@ async fn refresh_defers_rollback_eligible_sidecar_to_next_open() { "full sweep must run Dataset::restore (head advances); \ post_head={post_head}, final_head={final_head}", ); + // Convergence: roll-back published the restored HEAD, so the manifest pin + // tracks Lance HEAD afterward (no residual drift). + let entry_version = db + .snapshot_of(omnigraph::db::ReadTarget::branch("main")) + .await + .unwrap() + .entry("node:Person") + .unwrap() + .table_version; + assert_eq!( + entry_version, final_head, + "full-sweep roll-back must publish so manifest pin ({entry_version}) == Lance HEAD ({final_head})", + ); } /// Companion to the above β€” confirms that a finalizeβ†’publisher failure @@ -1461,10 +1474,15 @@ edge WorksAt: Person -> Company } let db = Omnigraph::open(&uri).await.unwrap(); - assert_eq!( - version_main(&db).await.unwrap(), - pre_failure_version, - "manifest must remain on the old schema when no schema staging files existed" + // Roll-back now publishes the restored version, so the manifest version + // advances β€” but to the OLD-schema content: the migration never applied + // (asserted by count_rows + the `_schema.pg` checks below), and the sweep + // converges (`manifest == Lance HEAD`, asserted by + // assert_post_recovery_invariants's RolledBack arm). + assert!( + version_main(&db).await.unwrap() > pre_failure_version, + "roll-back publishes the restored (old-schema) version, advancing the manifest; \ + pre={pre_failure_version}", ); assert_eq!( helpers::count_rows(&db, "node:Person").await, @@ -1637,6 +1655,100 @@ edge WorksAt: Person -> Company ); } +/// `optimize` Phase B β†’ Phase C residual: `compact_files` advanced the Lance +/// HEAD but the manifest publish hasn't run. The `Optimize` recovery sidecar +/// (loose-match, like SchemaApply/EnsureIndices) must roll the compacted version +/// forward on next open so the manifest tracks the Lance HEAD β€” and the healed +/// table must then accept a schema apply (the original bug's victim). +#[tokio::test] +async fn optimize_phase_b_failure_recovered_on_next_open() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let operation_id; + + // Seed: several separate Person inserts β†’ multiple fragments, so compaction + // has real work and advances the Lance HEAD. + { + let db = Omnigraph::init(&uri, helpers::TEST_SCHEMA).await.unwrap(); + for (name, age) in [("alice", 30), ("bob", 31), ("carol", 32), ("dave", 33)] { + db.mutate( + "main", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", name)], &[("$age", age)]), + ) + .await + .unwrap(); + } + } + + let pre_failure_version = { + let db = Omnigraph::open(&uri).await.unwrap(); + version_main(&db).await.unwrap() + }; + + // Failpoint fires AFTER compact_files advanced the Lance HEAD but BEFORE the + // manifest publish. The Optimize sidecar persists (only node:Person has + // compactable fragments, so exactly one sidecar is written). + { + let db = Omnigraph::open(&uri).await.unwrap(); + let _failpoint = + ScopedFailPoint::new("optimize.post_phase_b_pre_manifest_commit", "return"); + let err = db.optimize().await.unwrap_err(); + assert!( + err.to_string() + .contains("injected failpoint triggered: optimize.post_phase_b_pre_manifest_commit"), + "unexpected error: {err}" + ); + + let recovery_dir = dir.path().join("__recovery"); + let sidecars: Vec<_> = std::fs::read_dir(&recovery_dir) + .unwrap() + .filter_map(|e| e.ok()) + .collect(); + assert_eq!( + sidecars.len(), + 1, + "exactly one Optimize sidecar must persist after optimize failure" + ); + operation_id = single_sidecar_operation_id(dir.path()); + } + + // Recovery: reopen runs the sweep. The Optimize sidecar classifies + // RolledPastExpected (loose-match) β†’ RollForward β†’ manifest extends to the + // compacted Lance HEAD. + let db = Omnigraph::open(&uri).await.unwrap(); + let post_recovery_version = version_main(&db).await.unwrap(); + assert!( + post_recovery_version > pre_failure_version, + "manifest version must advance post-recovery (compaction rolled forward); \ + pre={pre_failure_version}, post={post_recovery_version}", + ); + drop(db); + + assert_post_recovery_invariants( + dir.path(), + &operation_id, + RecoveryExpectation::RolledForward { + tables: vec![TableExpectation::main("node:Person")], + }, + ) + .await + .unwrap(); + + // The healed table accepts an additive schema apply β€” its HEAD-vs-manifest + // precondition is satisfied because recovery published the compacted version. + let db = Omnigraph::open(&uri).await.unwrap(); + let desired = helpers::TEST_SCHEMA.replace( + " age: I32?\n}", + " age: I32?\n nickname: String?\n}", + ); + db.apply_schema(&desired) + .await + .expect("schema apply after optimize recovery must succeed"); +} + #[tokio::test] async fn branch_merge_phase_b_failure_recovered_on_next_open() { use omnigraph::loader::{LoadMode, load_jsonl}; diff --git a/crates/omnigraph/tests/fixtures/search.gq b/crates/omnigraph/tests/fixtures/search.gq index c39af82..d53fbc9 100644 --- a/crates/omnigraph/tests/fixtures/search.gq +++ b/crates/omnigraph/tests/fixtures/search.gq @@ -42,3 +42,17 @@ query hybrid_search($vq: Vector(4), $tq: String) { order { rrf(nearest($d.embedding, $vq), bm25($d.title, $tq)) } limit 3 } + +query rrf_two_fts($q: String) { + match { $d: Doc } + return { $d.slug, $d.title } + order { rrf(bm25($d.title, $q), bm25($d.body, $q)) } + limit 3 +} + +query rrf_two_vectors($q1: Vector(4), $q2: Vector(4)) { + match { $d: Doc } + return { $d.slug, $d.title } + order { rrf(nearest($d.embedding, $q1), nearest($d.embedding, $q2)) } + limit 3 +} diff --git a/crates/omnigraph/tests/helpers/mod.rs b/crates/omnigraph/tests/helpers/mod.rs index c97ff72..0e04aa2 100644 --- a/crates/omnigraph/tests/helpers/mod.rs +++ b/crates/omnigraph/tests/helpers/mod.rs @@ -236,6 +236,15 @@ pub fn vector_param(name: &str, values: &[f32]) -> ParamMap { map } +/// Build a ParamMap with two vector params. +pub fn two_vector_params(name1: &str, vals1: &[f32], name2: &str, vals2: &[f32]) -> ParamMap { + let mut map = vector_param(name1, vals1); + let key = name2.strip_prefix('$').unwrap_or(name2).to_string(); + let lit = Literal::List(vals2.iter().map(|v| Literal::Float(*v as f64)).collect()); + map.insert(key, lit); + map +} + /// Build a ParamMap with a vector param and a string param. pub fn vector_and_string_params( vec_name: &str, diff --git a/crates/omnigraph/tests/helpers/recovery.rs b/crates/omnigraph/tests/helpers/recovery.rs index c76009e..90d9a25 100644 --- a/crates/omnigraph/tests/helpers/recovery.rs +++ b/crates/omnigraph/tests/helpers/recovery.rs @@ -181,6 +181,9 @@ pub async fn assert_post_recovery_invariants( "audit row for {operation_id} recorded the wrong recovery_kind", ); assert_rollback_outcomes_record_drift(&audit); + // Roll-back now publishes the restored HEAD, so manifest == Lance + // HEAD afterward (symmetric with roll-forward) β€” no residual drift. + assert_manifest_pins_match_lance_heads(graph_root, &tables).await?; assert_recovery_commit_shape(graph_root, &audit, &tables).await?; assert_non_main_did_not_move_main(graph_root, &tables).await?; assert_idempotent_reopen(graph_root, operation_id).await?; diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index aa45d81..370f9e7 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -30,9 +30,13 @@ use arrow_schema::{DataType, Field, Schema}; use lance::Dataset; use lance::dataset::builder::DatasetBuilder; use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::dataset::transaction::Operation; use lance::dataset::write::delete::DeleteResult; use lance::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams}; +use lance::index::DatasetIndexExt; use lance_file::version::LanceFileVersion; +use lance_index::IndexType; +use lance_index::scalar::ScalarIndexParams; use lance_namespace::LanceNamespace; use lance_table::io::commit::ManifestNamingScheme; @@ -222,6 +226,33 @@ async fn _compile_compact_files_signature() -> lance::Result<()> { Ok(()) } +// --- Guard 7b: transaction history exposes repair's classification surface - +// +// `db/omnigraph/repair.rs` reads Lance transactions between manifest and HEAD +// and treats only `ReserveFragments` + `Rewrite` as safe maintenance drift. +// Compile-only. + +#[allow( + dead_code, + unreachable_code, + unused_variables, + unused_mut, + clippy::diverging_sub_expression +)] +async fn _compile_transaction_history_for_repair_signature() -> lance::Result<()> { + let ds: Dataset = unimplemented!(); + let tx = ds.read_transaction_by_version(1u64).await?; + if let Some(tx) = tx { + let operation = tx.operation; + let _name: &str = operation.name(); + match operation { + Operation::Rewrite { .. } | Operation::ReserveFragments { .. } => {} + _ => {} + } + } + Ok(()) +} + // --- Guard 8: Dataset::delete returns DeleteResult { new_dataset, num_deleted_rows } --- // // `table_store.rs::delete_where` consumes both fields. When MR-A migrates @@ -378,3 +409,135 @@ async fn compact_files_still_fails_on_blob_columns() { shifted): {err}" ); } + +// --- Guard 11: scalar-index coverage surface (physical_rows + index details) --- +// +// `table_store.rs::key_column_index_coverage` mirrors Lance's `create_filter_plan` +// C6 fallback: it reads `fragment.physical_rows` (the field whose absence on ANY +// fragment disables the scalar index for the whole scan) and sniffs the BTREE via +// `load_indices()` β†’ `index.fields` / `index.index_details.type_url`. This is the +// one real Lance-internal coupling on the indexed-traversal read path. If any of +// these surfaces renames or changes type, the coverage check (and the cost-based +// traversal chooser that consumes it) silently misclassifies. Compile-only. + +#[allow( + dead_code, + unreachable_code, + unused_variables, + unused_mut, + clippy::diverging_sub_expression +)] +async fn _compile_scalar_index_coverage_surface() -> lance::Result<()> { + let ds: Dataset = unimplemented!(); + // The create_filter_plan coupling: a fragment lacking `physical_rows` + // disables the scalar index for the entire scan. + for frag in ds.fragments().iter() { + let _physical_rows: Option = frag.physical_rows; + // `key_column_index_coverage` checks each current fragment id against the + // index `fragment_bitmap`. + let _id: u64 = frag.id; + } + // The index sniff: BTREE presence is detected by single-field index whose + // details type_url ends with "BTreeIndexDetails". The fragment coverage check + // reads `fragment_bitmap` (Option) and calls `.contains(u32)`. + let indices = ds.load_indices().await?; + for index in indices.iter() { + let _fields: &Vec = &index.fields; + if let Some(details) = index.index_details.as_ref() { + let _type_url: &str = details.type_url.as_str(); + } + let _covered: Option = index.fragment_bitmap.as_ref().map(|b| b.contains(0u32)); + } + Ok(()) +} + +// --- Guard 12: can a scalar BTREE be built on a system version column? -------- +// +// The deferred persisted-adjacency artifact plan assumed a cheap delta read of +// `_row_last_updated_at_version > V` could be a BTREE range lookup. Lance resolves +// index columns from the dataset schema, and the version columns are system +// metadata β€” so this probe documents whether the assumption holds. The outcome is +// the load-bearing fact, not a pass/fail of intent: if this starts SUCCEEDING when +// it currently errors (or vice versa), the artifact's delta-cost story changes. + +#[tokio::test] +async fn scalar_index_on_system_version_column_probe() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("guard12.lance"); + let mut ds = fresh_dataset(uri.to_str().unwrap()).await; + + // Sanity: the system version column is present (stable row ids + V2_2). + assert!( + ds.schema().field("_row_last_updated_at_version").is_none(), + "PROBE NOTE: `_row_last_updated_at_version` is NOT in the user schema \ + (it is system metadata); indexing it resolves through a different path." + ); + + let result = ds + .create_index_builder( + &["_row_last_updated_at_version"], + IndexType::BTree, + &ScalarIndexParams::default(), + ) + .replace(true) + .await; + + // Pin the observed behavior: a scalar index on the system version column is + // NOT buildable via the normal create-index path in this Lance. If this turns + // green (Ok), the artifact delta CAN use a version-column BTREE β€” revisit the + // deferred plan's Phase-2 delta-cost note in docs/dev/traversal handoff. + assert!( + result.is_err(), + "create_index on `_row_last_updated_at_version` unexpectedly SUCCEEDED β€” \ + a system-column scalar index is now buildable; the persisted-artifact \ + delta read could use it. Update the deferred-design notes." + ); +} + +// --- Guard 13: per-fragment deletion metadata is exposed without a scan ------- +// +// The deferred artifact's delete-correctness coverage model needs to detect, +// cheaply (O(fragments), no row scan), that a covered fragment acquired new +// deletions. That hinges on Lance tracking deletions at fragment-metadata level. +// This pins that a delete populates `fragment.deletion_file`, and probes whether +// the deleted-row COUNT is available as metadata (`num_deleted_rows`) β€” the +// difference between an O(fragments) coverage check and an O(|E|) scan. + +#[tokio::test] +async fn fragment_deletion_metadata_is_available() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("guard13.lance"); + let ds = fresh_dataset(uri.to_str().unwrap()).await; // 2 rows: alice, bob + + let deleted: DeleteResult = { + let mut ds = ds; + ds.delete("id = 'alice'").await.unwrap() + }; + assert_eq!(deleted.num_deleted_rows, 1, "one row deleted"); + let ds = deleted.new_dataset; + + // A delete must be tracked at fragment-metadata level (not only in data). + let with_deletion = ds + .fragments() + .iter() + .find(|f| f.deletion_file.is_some()) + .expect( + "after a delete, some fragment must carry a deletion_file β€” if not, \ + Lance changed deletion tracking; the artifact coverage model's \ + cheap delete-detection assumption is invalid.", + ); + + // Probe: is the deleted-row count available as metadata (cheap), or must the + // deletion vector be read? Pin whichever holds so the artifact plan knows. + let count: Option = with_deletion + .deletion_file + .as_ref() + .and_then(|df| df.num_deleted_rows); + assert_eq!( + count, + Some(1), + "PROBE: deletion_file.num_deleted_rows is not a populated metadata count \ + (got {count:?}); the artifact coverage model cannot cheaply detect \ + per-fragment deletions and would need to read the deletion vector.", + ); +} diff --git a/crates/omnigraph/tests/literal_filters.rs b/crates/omnigraph/tests/literal_filters.rs new file mode 100644 index 0000000..a0b2bd7 --- /dev/null +++ b/crates/omnigraph/tests/literal_filters.rs @@ -0,0 +1,96 @@ +//! Execution goldens for filtering by non-string/non-integer scalar LITERALS +//! (F64, F32, Bool, Date, DateTime), across both the in-memory comparison arm +//! (standalone `$m.prop op lit`) and the Lance-pushdown arm (inline binding +//! `Metric { prop: lit }`). Param-bound scalar filters and list-column +//! `contains` are already covered elsewhere; this closes the literal-RHS gap. + +mod helpers; + +use arrow_array::{Array, StringArray}; + +use omnigraph::db::Omnigraph; +use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph_compiler::ir::ParamMap; + +use helpers::*; + +const SCHEMA: &str = r#" +node Metric { + name: String @key + score: F64? + ratio: F32? + active: Bool? + born: Date? + seen: DateTime? +} +"#; + +// Seeds partition every predicate, so a dropped filter returns all 4 rows. +const DATA: &str = r#"{"type":"Metric","data":{"name":"m1","score":2.5,"ratio":0.5,"active":true,"born":"2024-06-01","seen":"2024-06-01T12:00:00Z"}} +{"type":"Metric","data":{"name":"m2","score":1.0,"ratio":0.25,"active":false,"born":"2023-01-01","seen":"2023-01-01T00:00:00Z"}} +{"type":"Metric","data":{"name":"m3","score":3.0,"ratio":0.75,"active":true,"born":"2025-01-01","seen":"2025-01-01T00:00:00Z"}} +{"type":"Metric","data":{"name":"m4","score":0.5,"ratio":0.1,"active":false,"born":"2022-12-31","seen":"2022-01-01T00:00:00Z"}}"#; + +async fn metric_db(dir: &tempfile::TempDir) -> Omnigraph { + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, SCHEMA).await.unwrap(); + load_jsonl(&mut db, DATA, LoadMode::Overwrite).await.unwrap(); + db +} + +async fn sorted_metric_names(db: &mut Omnigraph, queries: &str, name: &str) -> Vec { + let r = query_main(db, queries, name, &ParamMap::new()).await.unwrap(); + if r.num_rows() == 0 { + return Vec::new(); + } + let b = r.concat_batches().unwrap(); + let col = b.column(0).as_any().downcast_ref::().unwrap(); + let mut v: Vec = (0..col.len()).map(|i| col.value(i).to_string()).collect(); + v.sort(); + v +} + +#[tokio::test] +async fn float_literal_filters_execute() { + let dir = tempfile::tempdir().unwrap(); + let mut db = metric_db(&dir).await; + let q = r#" +query gt() { match { $m: Metric $m.score > 1.5 } return { $m.name } } +query le() { match { $m: Metric $m.ratio <= 0.25 } return { $m.name } } +query inline() { match { $m: Metric { score: 3.0 } } return { $m.name } } +"#; + // F64 standalone: scores 2.5, 3.0 > 1.5 + assert_eq!(sorted_metric_names(&mut db, q, "gt").await, vec!["m1", "m3"]); + // F32 standalone: ratios 0.25, 0.1 <= 0.25 + assert_eq!(sorted_metric_names(&mut db, q, "le").await, vec!["m2", "m4"]); + // F64 inline-binding pushdown: score == 3.0 + assert_eq!(sorted_metric_names(&mut db, q, "inline").await, vec!["m3"]); +} + +#[tokio::test] +async fn bool_literal_filters_execute() { + let dir = tempfile::tempdir().unwrap(); + let mut db = metric_db(&dir).await; + let q = r#" +query standalone() { match { $m: Metric $m.active = true } return { $m.name } } +query inline() { match { $m: Metric { active: true } } return { $m.name } } +query negated() { match { $m: Metric $m.active != true } return { $m.name } } +"#; + assert_eq!(sorted_metric_names(&mut db, q, "standalone").await, vec!["m1", "m3"]); + assert_eq!(sorted_metric_names(&mut db, q, "inline").await, vec!["m1", "m3"]); + assert_eq!(sorted_metric_names(&mut db, q, "negated").await, vec!["m2", "m4"]); +} + +#[tokio::test] +async fn date_and_datetime_literal_filters_execute() { + let dir = tempfile::tempdir().unwrap(); + let mut db = metric_db(&dir).await; + let q = r#" +query born_ge() { match { $m: Metric $m.born >= date("2024-01-01") } return { $m.name } } +query seen_lt() { match { $m: Metric $m.seen < datetime("2024-01-01T00:00:00Z") } return { $m.name } } +"#; + // born: m1 2024-06, m3 2025 >= 2024-01-01 + assert_eq!(sorted_metric_names(&mut db, q, "born_ge").await, vec!["m1", "m3"]); + // seen: m2 2023, m4 2022 < 2024-01-01 + assert_eq!(sorted_metric_names(&mut db, q, "seen_lt").await, vec!["m2", "m4"]); +} diff --git a/crates/omnigraph/tests/maintenance.rs b/crates/omnigraph/tests/maintenance.rs index 3e61677..13c9de7 100644 --- a/crates/omnigraph/tests/maintenance.rs +++ b/crates/omnigraph/tests/maintenance.rs @@ -8,10 +8,16 @@ mod helpers; use std::time::Duration; use lance::Dataset; -use omnigraph::db::{CleanupPolicyOptions, Omnigraph, SkipReason}; +use lance::dataset::optimize::{CompactionOptions, compact_files}; +use omnigraph::db::{ + CleanupPolicyOptions, Omnigraph, ReadTarget, RepairAction, RepairClassification, RepairOptions, + SkipReason, +}; use omnigraph::loader::{LoadMode, load_jsonl}; -use helpers::{TEST_DATA, TEST_SCHEMA, count_rows, init_and_load}; +use helpers::{ + MUTATION_QUERIES, TEST_DATA, TEST_SCHEMA, count_rows, init_and_load, mixed_params, mutate_main, +}; /// Filesystem URI of a node sub-table, mirroring the engine's layout /// (FNV-1a of the type name under `nodes/`). Matches the helper in @@ -25,11 +31,64 @@ fn node_table_uri(root: &str, type_name: &str) -> String { format!("{}/nodes/{hash:016x}", root.trim_end_matches('/')) } +async fn person_manifest_and_head(db: &Omnigraph, root: &str) -> (u64, u64, String) { + let snap = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let entry = snap.entry("node:Person").unwrap(); + let full = format!("{}/{}", root.trim_end_matches('/'), entry.table_path); + let head = Dataset::open(&full).await.unwrap().version().version; + (entry.table_version, head, full) +} + +async fn add_person_fragments(db: &mut Omnigraph) { + for (name, age) in [("Eve", 40), ("Frank", 41), ("Grace", 42), ("Heidi", 43)] { + mutate_main( + db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", name)], &[("$age", age as i64)]), + ) + .await + .expect("insert"); + } +} + +async fn forge_person_compaction_drift(db: &mut Omnigraph, root: &str) -> (u64, u64, String) { + add_person_fragments(db).await; + let (manifest_version, _, full) = person_manifest_and_head(db, root).await; + let mut ds = Dataset::open(&full).await.unwrap(); + let metrics = compact_files(&mut ds, CompactionOptions::default(), None) + .await + .expect("raw Lance compaction"); + let lance_head_version = ds.version().version; + assert!( + lance_head_version > manifest_version, + "raw Lance compaction should advance HEAD beyond manifest" + ); + assert!( + metrics.fragments_removed > 0 || metrics.fragments_added > 0, + "test precondition: raw compaction should rewrite fragments" + ); + (manifest_version, lance_head_version, full) +} + +async fn forge_person_delete_drift(db: &Omnigraph, root: &str) -> (u64, u64, String) { + let (manifest_version, _, full) = person_manifest_and_head(db, root).await; + let mut ds = Dataset::open(&full).await.unwrap(); + let deleted = ds.delete("name = 'Alice'").await.expect("raw Lance delete"); + assert_eq!(deleted.num_deleted_rows, 1, "fixture should delete Alice"); + let lance_head_version = deleted.new_dataset.version().version; + assert!( + lance_head_version > manifest_version, + "raw Lance delete should advance HEAD beyond manifest" + ); + (manifest_version, lance_head_version, full) +} + #[tokio::test] async fn optimize_on_empty_graph_returns_stats_per_table_with_no_changes() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); - let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + let db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); let stats = db.optimize().await.unwrap(); @@ -45,7 +104,7 @@ async fn optimize_on_empty_graph_returns_stats_per_table_with_no_changes() { #[tokio::test] async fn optimize_after_load_then_again_is_idempotent() { let dir = tempfile::tempdir().unwrap(); - let mut db = init_and_load(&dir).await; + let db = init_and_load(&dir).await; // First pass may compact (load wrote real fragments). let _first = db.optimize().await.unwrap(); @@ -163,6 +222,404 @@ node Tag {\n slug: String @key\n}\n"; assert_eq!(tag.skipped, None, "non-blob table must not be skipped"); } +// Regression: `optimize` must publish its compaction to the `__manifest` so the +// manifest's recorded `table_version` tracks the compacted Lance HEAD. +// +// Lance `compact_files` advances the *dataset's* version (reserve-fragments + +// rewrite commits) but knows nothing about OmniGraph's `__manifest`. If optimize +// does not publish a manifest update, the manifest's `table_version` lags the +// Lance HEAD: reads stay pinned to the pre-compaction version (compaction is +// invisible to them) and any subsequent schema apply / strict update/delete +// fails its HEAD-vs-manifest precondition with +// "stale view of '': expected manifest table version X but current is Y". +// This pins the fix β€” optimize publishes the compacted version, so manifest == +// HEAD and migrations after a compaction succeed. +#[tokio::test] +async fn optimize_publishes_compaction_to_manifest_so_schema_apply_succeeds() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let mut db = init_and_load(&dir).await; + + // Several separate inserts β†’ multiple Person fragments, so `compact_files` + // actually merges and moves the Lance HEAD (a single fragment is a no-op). + for (name, age) in [("Eve", 40), ("Frank", 41), ("Grace", 42), ("Heidi", 43)] { + mutate_main( + &mut db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", name)], &[("$age", age as i64)]), + ) + .await + .expect("insert"); + } + + let stats = db.optimize().await.unwrap(); + let person = stats + .iter() + .find(|s| s.table_key == "node:Person") + .expect("Person stat present"); + assert!( + person.committed, + "Person is multi-fragment, so optimize must have compacted it" + ); + + // After optimize, the manifest's recorded table_version must equal the actual + // Lance HEAD β€” optimize published its compaction, so there is no drift. + let snap = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let entry = snap.entry("node:Person").unwrap(); + let manifest_version = entry.table_version; + let full = format!("{}/{}", root, entry.table_path); + let lance_head = Dataset::open(&full).await.unwrap().version().version; + assert_eq!( + manifest_version, lance_head, + "after optimize, manifest table_version ({manifest_version}) must equal Lance HEAD ({lance_head})", + ); + + // Reads observe the compacted version with rows preserved (4 seed + 4 inserts). + assert_eq!(count_rows(&db, "node:Person").await, 8); + + // The headline: an additive (nullable property) migration touching the + // just-compacted table succeeds, where it previously failed with "stale view". + let desired = TEST_SCHEMA.replace( + " age: I32?\n}", + " age: I32?\n nickname: String?\n}", + ); + let result = db + .apply_schema(&desired) + .await + .expect("additive schema apply after optimize must succeed"); + assert!(result.applied, "schema apply should report applied=true"); +} + +#[tokio::test] +async fn optimize_skips_preexisting_manifest_head_drift() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let mut db = init_and_load(&dir).await; + let (manifest_before, head_before, _) = forge_person_compaction_drift(&mut db, &root).await; + + let stats = db.optimize().await.unwrap(); + let person = stats + .iter() + .find(|s| s.table_key == "node:Person") + .expect("Person stat present"); + assert_eq!(person.skipped, Some(SkipReason::DriftNeedsRepair)); + assert!(!person.committed); + assert_eq!(person.manifest_version, Some(manifest_before)); + assert_eq!(person.lance_head_version, Some(head_before)); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!( + manifest_after, manifest_before, + "optimize must not publish uncovered drift" + ); + assert_eq!( + head_after, head_before, + "optimize must not move drifted HEAD" + ); +} + +#[tokio::test] +async fn repair_preview_reports_verified_maintenance_drift_without_healing() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let mut db = init_and_load(&dir).await; + let (manifest_before, head_before, _) = forge_person_compaction_drift(&mut db, &root).await; + + let stats = db + .repair(RepairOptions { + confirm: false, + force: false, + }) + .await + .unwrap(); + assert_eq!(stats.manifest_version, None); + let person = stats + .tables + .iter() + .find(|s| s.table_key == "node:Person") + .expect("Person repair stat present"); + assert_eq!( + person.classification, + RepairClassification::VerifiedMaintenance + ); + assert_eq!(person.action, RepairAction::Preview); + assert_eq!(person.manifest_version, manifest_before); + assert_eq!(person.lance_head_version, head_before); + assert!( + person + .operations + .iter() + .all(|op| op == "ReserveFragments" || op == "Rewrite"), + "maintenance drift should only include Lance maintenance operations: {:?}", + person.operations + ); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!(manifest_after, manifest_before); + assert_eq!(head_after, head_before); +} + +#[tokio::test] +async fn repair_confirm_heals_verified_maintenance_drift() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let mut db = init_and_load(&dir).await; + let (_, head_before, _) = forge_person_compaction_drift(&mut db, &root).await; + + let stats = db + .repair(RepairOptions { + confirm: true, + force: false, + }) + .await + .unwrap(); + assert!( + stats.manifest_version.is_some(), + "confirmed repair should publish one manifest commit" + ); + let person = stats + .tables + .iter() + .find(|s| s.table_key == "node:Person") + .expect("Person repair stat present"); + assert_eq!( + person.classification, + RepairClassification::VerifiedMaintenance + ); + assert_eq!(person.action, RepairAction::Healed); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!(manifest_after, head_before); + assert_eq!(head_after, head_before); + + let desired = TEST_SCHEMA.replace( + " age: I32?\n}", + " age: I32?\n nickname: String?\n}", + ); + let result = db + .apply_schema(&desired) + .await + .expect("strict schema apply should succeed after repair"); + assert!(result.applied); +} + +#[tokio::test] +async fn repair_refuses_raw_delete_without_force() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let db = init_and_load(&dir).await; + let (manifest_before, head_before, _) = forge_person_delete_drift(&db, &root).await; + + let stats = db + .repair(RepairOptions { + confirm: true, + force: false, + }) + .await + .unwrap(); + assert_eq!(stats.manifest_version, None); + let person = stats + .tables + .iter() + .find(|s| s.table_key == "node:Person") + .expect("Person repair stat present"); + assert_eq!(person.classification, RepairClassification::Suspicious); + assert_eq!(person.action, RepairAction::Refused); + assert!( + person.operations.iter().any(|op| op == "Delete"), + "raw Lance delete should be reported as a suspicious operation: {:?}", + person.operations + ); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!(manifest_after, manifest_before); + assert_eq!(head_after, head_before); + assert_eq!( + count_rows(&db, "node:Person").await, + 4, + "manifest-pinned reads should still see the pre-delete version" + ); +} + +#[tokio::test] +async fn repair_force_heals_suspicious_drift() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let db = init_and_load(&dir).await; + let (_, head_before, _) = forge_person_delete_drift(&db, &root).await; + + let stats = db + .repair(RepairOptions { + confirm: true, + force: true, + }) + .await + .unwrap(); + let person = stats + .tables + .iter() + .find(|s| s.table_key == "node:Person") + .expect("Person repair stat present"); + assert_eq!(person.classification, RepairClassification::Suspicious); + assert_eq!(person.action, RepairAction::Forced); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!(manifest_after, head_before); + assert_eq!(head_after, head_before); + assert_eq!( + count_rows(&db, "node:Person").await, + 3, + "forced repair publishes the raw delete's HEAD" + ); +} + +#[tokio::test] +async fn non_strict_load_refuses_uncovered_drift_before_folding_it() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let mut db = init_and_load(&dir).await; + let (manifest_before, head_before, _) = forge_person_compaction_drift(&mut db, &root).await; + + let err = load_jsonl( + &mut db, + "{\"type\":\"Person\",\"data\":{\"name\":\"Ivan\",\"age\":44}}", + LoadMode::Merge, + ) + .await + .expect_err("merge load must not silently fold uncovered drift"); + assert!( + err.to_string().contains("omnigraph repair"), + "error should point at explicit repair; got: {err}" + ); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!(manifest_after, manifest_before); + assert_eq!(head_after, head_before); +} + +#[tokio::test] +async fn delete_only_mutation_refuses_uncovered_drift_before_inline_commit() { + let dir = tempfile::tempdir().unwrap(); + let root = dir + .path() + .to_str() + .unwrap() + .trim_end_matches('/') + .to_string(); + let mut db = init_and_load(&dir).await; + let (manifest_before, head_before, _) = forge_person_compaction_drift(&mut db, &root).await; + + let err = mutate_main( + &mut db, + MUTATION_QUERIES, + "remove_person", + &mixed_params(&[("$name", "Alice")], &[]), + ) + .await + .expect_err("strict delete must reject uncovered drift before delete_where"); + assert!( + err.to_string().contains("expected"), + "delete should fail as a strict stale-version write; got: {err}" + ); + + let (manifest_after, head_after, _) = person_manifest_and_head(&db, &root).await; + assert_eq!(manifest_after, manifest_before); + assert_eq!( + head_after, head_before, + "delete_where must not run after the strict drift guard fails" + ); + assert_eq!( + count_rows(&db, "node:Person").await, + 8, + "manifest-pinned reads should still see all rows present before the failed delete" + ); +} + +// Regression: `optimize` must REFUSE when an unresolved recovery sidecar is +// pending. Operating on an unrecovered graph could publish a partial write that +// the all-or-nothing recovery sweep would roll back; the operator must reopen +// (run the recovery sweep) first. +#[tokio::test] +async fn optimize_defers_when_recovery_sidecar_is_pending() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let db = init_and_load(&dir).await; + + // Simulate an in-process failed write that left a recovery sidecar on disk. + let recovery_dir = dir.path().join("__recovery"); + std::fs::create_dir_all(&recovery_dir).unwrap(); + let person_path = node_table_uri(uri, "Person"); + let sidecar_json = format!( + r#"{{ + "schema_version": 1, + "operation_id": "01H000000000000000000DEFR", + "started_at": "0", + "branch": null, + "actor_id": "act-test", + "writer_kind": "Mutation", + "tables": [ + {{ + "table_key": "node:Person", + "table_path": "{}", + "expected_version": 1, + "post_commit_pin": 2 + }} + ] + }}"#, + person_path + ); + std::fs::write( + recovery_dir.join("01H000000000000000000DEFR.json"), + sidecar_json, + ) + .unwrap(); + + let err = db + .optimize() + .await + .expect_err("optimize must defer (error) while a recovery sidecar is pending"); + assert!( + err.to_string().to_lowercase().contains("recovery"), + "optimize defer error should mention recovery; got: {err}", + ); +} + #[tokio::test] async fn cleanup_without_any_policy_option_errors() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph/tests/merge_truth_table.rs b/crates/omnigraph/tests/merge_truth_table.rs index 068b439..e2df882 100644 --- a/crates/omnigraph/tests/merge_truth_table.rs +++ b/crates/omnigraph/tests/merge_truth_table.rs @@ -941,8 +941,8 @@ async fn merge_pair_truth_table() { unsupported_cells, 45, "expected 45 cells involving dropProperty/addLabel/removeLabel" ); - assert!( - elapsed.as_secs() < 30, - "merge truth table exceeded 30s budget: {elapsed:?}" - ); + // No wall-clock assertion here: `elapsed` is logged above for visibility, but + // a fixed time budget in a correctness test flakes under parallel test load + // (it tripped at ~31s in the full `--test-threads=4` gate while passing at + // ~20s in isolation). Merge-perf regressions belong in a bench, not here. } diff --git a/crates/omnigraph/tests/ordering.rs b/crates/omnigraph/tests/ordering.rs new file mode 100644 index 0000000..4e9296b --- /dev/null +++ b/crates/omnigraph/tests/ordering.rs @@ -0,0 +1,134 @@ +//! ORDER BY golden coverage: descending, multi-key precedence, deterministic +//! tie-break (total order), and NULL placement. +//! +//! These pin the observable output-ordering contract (deny-list: "output +//! ordering … become dependencies once shipped"). `apply_ordering` appends the +//! bound entities' key columns as an ascending tie-break, so equal user-sort +//! keys yield a TOTAL, deterministic order (and `ORDER … LIMIT` is +//! deterministic). NULL placement is `nulls_first = !descending` (NULLs first +//! under ASC, last under DESC). Both are documented in +//! `docs/user/query-language.md`. + +mod helpers; + +use arrow_array::{Array, StringArray}; + +use omnigraph::db::Omnigraph; +use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph_compiler::ir::ParamMap; +use omnigraph_compiler::result::QueryResult; + +use helpers::*; + +/// Names in result ROW order (not sorted) β€” these tests assert positional order. +fn names_in_order(result: &QueryResult) -> Vec { + let batch = result.concat_batches().unwrap(); + if batch.num_rows() == 0 { + return Vec::new(); + } + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + (0..col.len()).map(|i| col.value(i).to_string()).collect() +} + +/// Init the standard schema and load a custom Person-only dataset. +async fn init_people(dir: &tempfile::TempDir, jsonl: &str) -> Omnigraph { + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl(&mut db, jsonl, LoadMode::Overwrite).await.unwrap(); + db +} + +#[tokio::test] +async fn ordering_descending() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + let q = r#" +query q() { + match { $p: Person } + return { $p.name } + order { $p.age desc } +} +"#; + let got = names_in_order(&query_main(&mut db, q, "q", &ParamMap::new()).await.unwrap()); + // Charlie(35), Alice(30), Diana(28), Bob(25) + assert_eq!(got, vec!["Charlie", "Alice", "Diana", "Bob"]); +} + +#[tokio::test] +async fn ordering_multi_key_age_desc_name_asc() { + let dir = tempfile::tempdir().unwrap(); + // Alice & Bob tie at age 30; loaded Bob-first so the expected output order + // cannot be the load order. + let data = r#"{"type":"Person","data":{"name":"Bob","age":30}} +{"type":"Person","data":{"name":"Alice","age":30}} +{"type":"Person","data":{"name":"Charlie","age":25}}"#; + let mut db = init_people(&dir, data).await; + let q = r#" +query q() { + match { $p: Person } + return { $p.name } + order { $p.age desc, $p.name asc } +} +"#; + let got = names_in_order(&query_main(&mut db, q, "q", &ParamMap::new()).await.unwrap()); + // age desc -> [30,30,25]; the 30-tie broken by name asc -> Alice before Bob. + assert_eq!(got, vec!["Alice", "Bob", "Charlie"]); +} + +#[tokio::test] +async fn ordering_tiebreak_by_key_is_deterministic() { + let dir = tempfile::tempdir().unwrap(); + // Same tie at age 30, NO secondary sort key. Loaded Bob-first; the tie must + // break by the entity key (name) ascending -> Alice before Bob, regardless + // of load order. This locks the total-order tie-break in apply_ordering. + let data = r#"{"type":"Person","data":{"name":"Bob","age":30}} +{"type":"Person","data":{"name":"Alice","age":30}} +{"type":"Person","data":{"name":"Charlie","age":25}}"#; + let mut db = init_people(&dir, data).await; + let q = r#" +query q() { + match { $p: Person } + return { $p.name } + order { $p.age asc } +} +"#; + let got = names_in_order(&query_main(&mut db, q, "q", &ParamMap::new()).await.unwrap()); + // age asc -> Charlie(25), then the 30-tie broken by key asc -> Alice, Bob. + assert_eq!(got, vec!["Charlie", "Alice", "Bob"]); +} + +#[tokio::test] +async fn ordering_nulls_placement_asc_and_desc() { + let dir = tempfile::tempdir().unwrap(); + // Bob has a NULL age. + let data = r#"{"type":"Person","data":{"name":"Alice","age":30}} +{"type":"Person","data":{"name":"Bob","age":null}} +{"type":"Person","data":{"name":"Charlie","age":25}}"#; + let mut db = init_people(&dir, data).await; + + let asc = r#" +query q() { + match { $p: Person } + return { $p.name } + order { $p.age asc } +} +"#; + let got_asc = names_in_order(&query_main(&mut db, asc, "q", &ParamMap::new()).await.unwrap()); + // ASC: nulls_first -> Bob(null), then 25, 30. + assert_eq!(got_asc, vec!["Bob", "Charlie", "Alice"]); + + let desc = r#" +query q() { + match { $p: Person } + return { $p.name } + order { $p.age desc } +} +"#; + let got_desc = names_in_order(&query_main(&mut db, desc, "q", &ParamMap::new()).await.unwrap()); + // DESC: nulls last -> 30, 25, then Bob(null). + assert_eq!(got_desc, vec!["Alice", "Charlie", "Bob"]); +} diff --git a/crates/omnigraph/tests/proptest_equivalence.rs b/crates/omnigraph/tests/proptest_equivalence.rs new file mode 100644 index 0000000..3423a2f --- /dev/null +++ b/crates/omnigraph/tests/proptest_equivalence.rs @@ -0,0 +1,311 @@ +//! Property-based query-correctness invariants over generated graphs. +//! +//! The cross-type id-collision bug (fixed in f6a0e53) was a silent wrong-result +//! divergence between the two Expand modes, caught only because someone +//! hand-built the one colliding fixture. This turns that single example into a +//! search over the whole class: node keys for BOTH types are drawn from a small +//! SHARED alphabet, so cross-type collisions β€” plus cycles and self-loops β€” +//! arise frequently. The invariants make any future fork divergence (the planned +//! third ExpandMode, the anti-join fast/slow fork) fail loudly instead of +//! silently. +//! +//! Each test is a sync `#[test]` + `#[serial]`: it builds its own runtime and +//! `block_on`s per generated case (proptest closures are sync), and the +//! mode-equivalence test writes `OMNIGRAPH_TRAVERSAL_MODE`, so serial execution +//! keeps env writes from racing other tests in this binary. + +mod helpers; + +use std::collections::HashSet; + +use arrow_array::{Array, StringArray}; +use proptest::prelude::*; +use proptest::test_runner::{Config, TestRunner}; +use serial_test::serial; + +use omnigraph::db::{Omnigraph, ReadTarget}; +use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph_compiler::ir::ParamMap; +use omnigraph_compiler::query::ast::Literal; + +use helpers::*; + +/// Small SHARED key alphabet β€” Person and Company keys are both drawn from this, +/// so cross-type id collisions are common. +const KEYS: &[&str] = &["a", "b", "c", "d", "e"]; + +const QUERIES: &str = r#" +query friends($name: String) { + match { + $p: Person { name: $name } + $p knows{1,3} $f + } + return { $f.name } +} +query employers($name: String) { + match { + $p: Person { name: $name } + $p worksAt{1,2} $c + } + return { $c.name } +} +query all_persons() { + match { $p: Person } + return { $p.name } +} +query employed() { + match { + $p: Person + $p worksAt $c + } + return { $p.name } +} +query unemployed() { + match { + $p: Person + not { $p worksAt $_ } + } + return { $p.name } +} +"#; + +#[derive(Debug, Clone)] +struct GenGraph { + persons: Vec, + companies: Vec, + knows: Vec<(usize, usize)>, // indices into persons (self-loops & cycles allowed) + works_at: Vec<(usize, usize)>, // (person idx, company idx) +} + +impl GenGraph { + fn to_jsonl(&self) -> String { + let mut s = String::new(); + for p in &self.persons { + s.push_str(&format!("{{\"type\":\"Person\",\"data\":{{\"name\":\"{p}\"}}}}\n")); + } + for c in &self.companies { + s.push_str(&format!("{{\"type\":\"Company\",\"data\":{{\"name\":\"{c}\"}}}}\n")); + } + // Dedup exact-duplicate edge rows (the loader rejects intra-batch + // duplicate keys); collisions/cycles/self-loops are unaffected. + let mut seen = HashSet::new(); + for &(a, b) in &self.knows { + if seen.insert(("k", a, b)) { + s.push_str(&format!( + "{{\"edge\":\"Knows\",\"from\":\"{}\",\"to\":\"{}\"}}\n", + self.persons[a], self.persons[b] + )); + } + } + for &(a, b) in &self.works_at { + if seen.insert(("w", a, b)) { + s.push_str(&format!( + "{{\"edge\":\"WorksAt\",\"from\":\"{}\",\"to\":\"{}\"}}\n", + self.persons[a], self.companies[b] + )); + } + } + s + } +} + +fn arb_keys() -> impl Strategy> { + proptest::sample::subsequence(KEYS.to_vec(), 1..=KEYS.len()) + .prop_map(|v| v.into_iter().map(String::from).collect()) +} + +fn arb_graph() -> impl Strategy { + (arb_keys(), arb_keys()).prop_flat_map(|(persons, companies)| { + let np = persons.len(); + let nc = companies.len(); + let knows = prop::collection::vec((0..np, 0..np), 0..=10); + let works = prop::collection::vec((0..np, 0..nc), 0..=10); + (Just(persons), Just(companies), knows, works).prop_map( + |(persons, companies, knows, works_at)| GenGraph { + persons, + companies, + knows, + works_at, + }, + ) + }) +} + +fn config() -> Config { + Config { + cases: 48, + ..Config::default() + } +} + +fn clear_mode() { + unsafe { std::env::remove_var("OMNIGRAPH_TRAVERSAL_MODE") }; +} + +/// RAII guard that sets `OMNIGRAPH_TRAVERSAL_MODE` and clears it on drop β€” so a +/// panic mid-case (e.g. a query `unwrap`) cannot leak the forced mode into +/// proptest's subsequent shrink/cases and mask the divergence under test. SAFE: +/// every test in this binary is `#[serial]`, so no thread reads the env during +/// the write. +struct ModeGuard; +impl ModeGuard { + fn set(mode: &str) -> Self { + unsafe { std::env::set_var("OMNIGRAPH_TRAVERSAL_MODE", mode) }; + ModeGuard + } +} +impl Drop for ModeGuard { + fn drop(&mut self) { + unsafe { std::env::remove_var("OMNIGRAPH_TRAVERSAL_MODE") }; + } +} + +async fn load_graph(graph: &GenGraph) -> (tempfile::TempDir, Omnigraph) { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl(&mut db, &graph.to_jsonl(), LoadMode::Overwrite) + .await + .unwrap(); + (dir, db) +} + +fn one_param(val: &str) -> ParamMap { + let mut m = ParamMap::new(); + m.insert("name".to_string(), Literal::String(val.to_string())); + m +} + +/// First-column strings, sorted (MULTISET β€” preserves duplicate-row count so +/// mode comparisons catch dedup divergence, not just set divergence). +async fn col0_sorted(db: &mut Omnigraph, name: &str, params: &ParamMap) -> Vec { + let r = db + .query(ReadTarget::branch("main"), QUERIES, name, params) + .await + .unwrap(); + if r.num_rows() == 0 { + return Vec::new(); + } + let b = r.concat_batches().unwrap(); + let col = b.column(0).as_any().downcast_ref::().unwrap(); + let mut v: Vec = (0..col.len()).map(|i| col.value(i).to_string()).collect(); + v.sort(); + v +} + +async fn col0_set(db: &mut Omnigraph, name: &str, params: &ParamMap) -> HashSet { + col0_sorted(db, name, params).await.into_iter().collect() +} + +// INVARIANT 1: mode equivalence. For any generated graph and start key, the +// CSR, indexed, and auto paths return identical result multisets β€” over both a +// same-type traversal (knows{1,3}, exercises cycles/self-loops) and a cross-type +// one (worksAt{1,2}, collision-prone). This is the search-over-the-class version +// of the hand-built cross-type-collision fixture. +#[test] +#[serial] +fn prop_expand_indexed_eq_csr() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut runner = TestRunner::new(config()); + runner + .run(&arb_graph(), |graph| { + let mismatch = rt.block_on(async { + let (_dir, mut db) = load_graph(&graph).await; + for start in graph.persons.clone() { + let p = one_param(&start); + for q in ["friends", "employers"] { + // Each guard clears the mode on drop (end of the block, + // or on panic), so a forced mode never leaks across runs. + let csr = { + let _g = ModeGuard::set("csr"); + col0_sorted(&mut db, q, &p).await + }; + let indexed = { + let _g = ModeGuard::set("indexed"); + col0_sorted(&mut db, q, &p).await + }; + // No guard β†’ env unset β†’ auto (cost-based) path. + let auto = col0_sorted(&mut db, q, &p).await; + if csr != indexed || csr != auto { + return Some((start, q, csr, indexed, auto)); + } + } + } + None + }); + prop_assert!( + mismatch.is_none(), + "Expand mode divergence: {:?}", + mismatch + ); + Ok(()) + }) + .unwrap(); +} + +// INVARIANT 2: no phantom rows. Every key a traversal returns must belong to the +// destination type's loaded key set β€” independent of the two-mode comparison, so +// it catches over-emission even if both modes are wrong identically. +#[test] +#[serial] +fn prop_results_subset_of_existing_nodes() { + clear_mode(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut runner = TestRunner::new(config()); + runner + .run(&arb_graph(), |graph| { + let bad = rt.block_on(async { + let (_dir, mut db) = load_graph(&graph).await; + let persons: HashSet = graph.persons.iter().cloned().collect(); + let companies: HashSet = graph.companies.iter().cloned().collect(); + for start in graph.persons.clone() { + let p = one_param(&start); + for f in col0_set(&mut db, "friends", &p).await { + if !persons.contains(&f) { + return Some(("friends", start, f)); + } + } + for c in col0_set(&mut db, "employers", &p).await { + if !companies.contains(&c) { + return Some(("employers", start, c)); + } + } + } + None + }); + prop_assert!(bad.is_none(), "phantom row: {:?}", bad); + Ok(()) + }) + .unwrap(); +} + +// INVARIANT 3: anti-join complement. `not { $p worksAt $_ }` and its complement +// (persons WITH a worksAt) must be disjoint and together cover all persons. +#[test] +#[serial] +fn prop_antijoin_partitions_persons() { + clear_mode(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut runner = TestRunner::new(config()); + runner + .run(&arb_graph(), |graph| { + let err = rt.block_on(async { + let (_dir, mut db) = load_graph(&graph).await; + let all = col0_set(&mut db, "all_persons", &ParamMap::new()).await; + let unemployed = col0_set(&mut db, "unemployed", &ParamMap::new()).await; + let employed = col0_set(&mut db, "employed", &ParamMap::new()).await; + let overlap: Vec<_> = unemployed.intersection(&employed).cloned().collect(); + let union: HashSet<_> = unemployed.union(&employed).cloned().collect(); + if !overlap.is_empty() { + return Some(format!("overlap {overlap:?}")); + } + if union != all { + return Some(format!("union {union:?} != all {all:?}")); + } + None + }); + prop_assert!(err.is_none(), "anti-join partition broken: {:?}", err); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/omnigraph/tests/recovery.rs b/crates/omnigraph/tests/recovery.rs index a090178..f6b19e8 100644 --- a/crates/omnigraph/tests/recovery.rs +++ b/crates/omnigraph/tests/recovery.rs @@ -278,6 +278,97 @@ async fn recovery_rolls_back_synthetic_drift_on_open() { ); } +/// Regression: recovery roll-back must PUBLISH the restored version so +/// `manifest == Lance HEAD` afterward (no residual "orphaned drift"). Before the +/// fix, roll-back restored via `Dataset::restore` but left the manifest pin +/// behind HEAD, so a subsequent strict write / schema apply failed its +/// HEAD-vs-manifest precondition ("stale view … refresh and retry") β€” and a +/// failed schema apply's own roll-back leaked +1 each retry (the original bug's +/// loop). With convergence, one roll-back leaves `manifest == HEAD` and the +/// follow-up succeeds. +#[tokio::test] +async fn recovery_rollback_converges_manifest_so_schema_apply_succeeds() { + use omnigraph::db::ReadTarget; + use omnigraph::loader::{LoadMode, load_jsonl}; + use omnigraph::table_store::TableStore; + + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl( + &mut db, + r#"{"type":"Person","data":{"name":"alice","age":30}} +{"type":"Person","data":{"name":"bob","age":25}} +"#, + LoadMode::Append, + ) + .await + .unwrap(); + drop(db); + + // Forge a Phase-B residual: advance Person's Lance HEAD without publishing to + // the manifest (the manifest pin stays at the load's committed version). + let person_uri = node_table_uri(uri, "Person"); + let store = TableStore::new(uri); + let mut ds = Dataset::open(&person_uri).await.unwrap(); + let manifest_pin = ds.version().version; + let _ = store + .delete_where(&person_uri, &mut ds, "1 = 2") + .await + .unwrap(); + drop(ds); + + // Roll-back-classified sidecar (post_commit_pin != observed head β‡’ + // UnexpectedAtP1 β‡’ RollBack). + let sidecar_json = format!( + r#"{{ + "schema_version": 1, + "operation_id": "01H0000000000000000000CVG", + "started_at": "0", + "branch": null, + "actor_id": "act-test", + "writer_kind": "Mutation", + "tables": [ + {{ + "table_key": "node:Person", + "table_path": "{}", + "expected_version": {}, + "post_commit_pin": {} + }} + ] + }}"#, + person_uri, manifest_pin, manifest_pin + ); + write_sidecar_file(dir.path(), "01H0000000000000000000CVG", &sidecar_json); + + // Reopen runs the sweep: restore Person to manifest_pin, then PUBLISH so the + // manifest tracks the restored Lance HEAD. + let db = Omnigraph::open(uri).await.unwrap(); + + // Convergence: manifest pin == Lance HEAD. Fails before the fix β€” the + // manifest stays at manifest_pin while HEAD advanced past it. + let snap = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let entry = snap.entry("node:Person").unwrap(); + let lance_head = Dataset::open(&person_uri).await.unwrap().version().version; + assert_eq!( + entry.table_version, lance_head, + "roll-back must publish so manifest pin ({}) == Lance HEAD ({})", + entry.table_version, lance_head, + ); + + // The +1-loop victim: an additive schema apply must now succeed (its + // HEAD-vs-manifest precondition is satisfied). Before the fix this failed + // with "stale view … refresh and retry". + let desired = TEST_SCHEMA.replace( + " age: I32?\n}", + " age: I32?\n nickname: String?\n}", + ); + db.apply_schema(&desired) + .await + .expect("schema apply after a converging roll-back must succeed"); +} + // ===================================================================== // Phase 4 β€” roll-forward path + audit row recording // ===================================================================== diff --git a/crates/omnigraph/tests/search.rs b/crates/omnigraph/tests/search.rs index c4454cf..480ec3c 100644 --- a/crates/omnigraph/tests/search.rs +++ b/crates/omnigraph/tests/search.rs @@ -556,6 +556,111 @@ async fn bm25_returns_ranked_results() { assert!(result.num_rows() <= 3, "bm25 should respect limit 3"); } +// Full rank-ORDER golden (not just top-1 / non-empty): pins ranks 2..k so a +// regression corrupting the tail or reversing the sort direction fails loudly. +// nearest skips apply_ordering (is_search_ordered) and returns Lance native +// order, so result_slugs row order == rank order. +#[tokio::test] +#[serial] +async fn nearest_full_rank_order() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_search_db(&dir).await; + let result = query_main( + &mut db, + SEARCH_QUERIES, + "vector_search", + &vector_param("$q", &[0.1, 0.2, 0.3, 0.4]), + ) + .await + .unwrap(); + // [0.1,0.2,0.3,0.4] == ml-intro's embedding (dist 0); the rest by ascending L2. + assert_eq!(result_slugs(&result), vec!["ml-intro", "nlp-guide", "rl-intro"]); +} + +#[tokio::test] +#[serial] +async fn bm25_full_rank_order() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_search_db(&dir).await; + let result = query_main( + &mut db, + SEARCH_QUERIES, + "bm25_search", + ¶ms(&[("$q", "Learning")]), + ) + .await + .unwrap(); + // Descending BM25 score order. + assert_eq!(result_slugs(&result), vec!["rl-intro", "ml-intro", "dl-basics"]); +} + +// Characterization: fuzzy() does NOT match under the default tokenizer/index in +// this setup β€” a one-edit typo ("Introductio" for "Introduction") returns no +// rows. (`search`/`match_text` DO work, so FTS itself is fine; fuzzy term +// queries specifically are inert here.) This pins that documented limitation +// instead of leaving fuzzy silently unasserted: if a Lance/tokenizer change +// makes fuzzy match, this turns red and should be promoted to a real +// matched-set + exclusion golden. +#[tokio::test] +#[serial] +async fn fuzzy_does_not_match_under_default_tokenizer() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_search_db(&dir).await; + let r = query_main(&mut db, SEARCH_QUERIES, "fuzzy_search", ¶ms(&[("$q", "Introductio")])) + .await + .unwrap(); + assert!( + result_slugs(&r).is_empty(), + "fuzzy now matches β€” promote this to a real matched-set/exclusion golden" + ); +} + +// match_text is a FILTER on the body: assert the exact matched set, not contains. +#[tokio::test] +#[serial] +async fn match_text_matches_exact_set_excludes_unrelated() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_search_db(&dir).await; + // "neural" appears only in dl-basics's body ("neural networks"). + let r = query_main(&mut db, SEARCH_QUERIES, "phrase_search", ¶ms(&[("$q", "neural")])) + .await + .unwrap(); + let mut got = result_slugs(&r); + got.sort(); + assert_eq!(got, vec!["dl-basics"]); +} + +// RRF fuses arms OTHER than the default nearest+bm25: two FTS arms (title+body). +// Proves primary_var resolves when neither arm is `nearest`, and fusion runs. +#[tokio::test] +#[serial] +async fn rrf_fuses_two_fts_fields() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_search_db(&dir).await; + let r = query_main(&mut db, SEARCH_QUERIES, "rrf_two_fts", ¶ms(&[("$q", "learning")])) + .await + .unwrap(); + assert_eq!(result_slugs(&r), vec!["dl-basics", "ml-intro", "rl-intro"]); +} + +// RRF fuses two vector arms (no embedding creds β€” explicit vectors). A doc near +// BOTH query vectors out-ranks one near only one. +#[tokio::test] +#[serial] +async fn rrf_fuses_two_vector_queries() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_search_db(&dir).await; + let r = query_main( + &mut db, + SEARCH_QUERIES, + "rrf_two_vectors", + &two_vector_params("$q1", &[0.1, 0.2, 0.3, 0.4], "$q2", &[0.5, 0.6, 0.7, 0.8]), + ) + .await + .unwrap(); + assert_eq!(result_slugs(&r), vec!["rl-intro", "ml-intro", "dl-basics"]); +} + #[tokio::test] #[serial] async fn mutation_commit_refreshes_search_indices_without_manual_ensure() { diff --git a/crates/omnigraph/tests/traversal.rs b/crates/omnigraph/tests/traversal.rs index 6efe7de..2f518fd 100644 --- a/crates/omnigraph/tests/traversal.rs +++ b/crates/omnigraph/tests/traversal.rs @@ -46,6 +46,194 @@ query not_at_acme() { assert_eq!(names_vec, vec!["Bob", "Charlie", "Diana"]); } +// Nested anti-join (double negation): proves `not { … not { … } }` recurses +// through execute_pipeline. "People who do NOT work at any NON-Acme company": +// inner `not { $c.name = "Acme" }` keeps the non-Acme employers, the outer `not` +// removes anyone who has one. Alice (Acme only), Charlie & Diana (no employer) +// remain β€” distinct from plain unemployed {Charlie, Diana}. +#[tokio::test] +async fn nested_anti_join_double_negation() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + + let queries = r#" +query no_nonacme_employer() { + match { + $p: Person + not { + $p worksAt $c + not { + $c.name = "Acme" + } + } + } + return { $p.name } +} +"#; + let result = query_main(&mut db, queries, "no_nonacme_employer", &ParamMap::new()) + .await + .unwrap(); + + let batch = result.concat_batches().unwrap(); + let names = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let mut names_vec: Vec<&str> = (0..names.len()).map(|i| names.value(i)).collect(); + names_vec.sort(); + assert_eq!(names_vec, vec!["Alice", "Charlie", "Diana"]); +} + +// The anti-join has two execution forks: the CSR `has_neighbors` fast path +// (bare single-op Expand inner) and the set-oriented inner-pipeline replay (when +// dst_filters force a multi-op inner). They must agree. `not { $p worksAt $_ }` +// takes the fast path; the same negation with an always-true dst filter +// (`$c.name != ""`) is semantically identical but forces the slow path. +#[tokio::test] +async fn anti_join_fast_and_slow_paths_agree() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + + let queries = r#" +query fast() { + match { + $p: Person + not { $p worksAt $_ } + } + return { $p.name } +} +query slow() { + match { + $p: Person + not { + $p worksAt $c + $c.name != "" + } + } + return { $p.name } +} +"#; + let names = |result: omnigraph_compiler::result::QueryResult| { + let batch = result.concat_batches().unwrap(); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let mut v: Vec = (0..col.len()).map(|i| col.value(i).to_string()).collect(); + v.sort(); + v + }; + + let fast = names(query_main(&mut db, queries, "fast", &ParamMap::new()).await.unwrap()); + let slow = names(query_main(&mut db, queries, "slow", &ParamMap::new()).await.unwrap()); + + assert_eq!(fast, slow, "anti-join fast and slow paths must agree"); + // Alice->Acme, Bob->Globex employed; Charlie & Diana have no employer. + assert_eq!(fast, vec!["Charlie", "Diana"]); +} + +// Regression: nested slow-path anti-joins must not collide on the synthetic +// correlation tag. The outer anti-join tags rows with a correlation column that +// rides through its inner pipeline; when the inner pipeline contains ANOTHER +// slow-path anti-join, a fixed tag name would duplicate, and reading it by name +// returns the OUTER tag β€” mis-correlating the inner negation. Fan-out (p1 works +// at two companies) makes the inner row indices diverge from the outer tags, so +// the bug produces a different person set than the correct one. +#[tokio::test] +async fn nested_anti_join_with_fanout_correlates_correctly() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + // p1 -> {Acme, Globex} (fan-out), p2 -> Globex, p3 -> Acme, p4 -> (none). + let data = r#"{"type":"Person","data":{"name":"p1"}} +{"type":"Person","data":{"name":"p2"}} +{"type":"Person","data":{"name":"p3"}} +{"type":"Person","data":{"name":"p4"}} +{"type":"Company","data":{"name":"Acme"}} +{"type":"Company","data":{"name":"Globex"}} +{"edge":"WorksAt","from":"p1","to":"Acme"} +{"edge":"WorksAt","from":"p1","to":"Globex"} +{"edge":"WorksAt","from":"p2","to":"Globex"} +{"edge":"WorksAt","from":"p3","to":"Acme"}"#; + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl(&mut db, data, LoadMode::Overwrite).await.unwrap(); + + let queries = r#" +query no_nonacme_employer() { + match { + $p: Person + not { + $p worksAt $c + not { + $c.name = "Acme" + } + } + } + return { $p.name } +} +"#; + let result = query_main(&mut db, queries, "no_nonacme_employer", &ParamMap::new()) + .await + .unwrap(); + let batch = result.concat_batches().unwrap(); + let names = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let mut names_vec: Vec<&str> = (0..names.len()).map(|i| names.value(i)).collect(); + names_vec.sort(); + // p1 & p2 have a non-Acme employer (Globex) -> excluded; p3 (Acme only) and + // p4 (no employer) remain. + assert_eq!(names_vec, vec!["p3", "p4"]); +} + +// Regression: a multi-hop anti-join must not take the bulk fast path. The fast +// path answers via `has_neighbors` (ONE-hop existence), so `not { $p knows{2,2} +// $x }` would wrongly drop a node that has a 1-hop neighbor but no 2-hop path. +// Graph: a->b (b is a sink, so a has no 2-hop path), c->d->e (c has a 2-hop +// path). Only c has a 2-hop knows path, so only c is removed. +#[tokio::test] +async fn anti_join_respects_multi_hop_bounds() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let data = r#"{"type":"Person","data":{"name":"a"}} +{"type":"Person","data":{"name":"b"}} +{"type":"Person","data":{"name":"c"}} +{"type":"Person","data":{"name":"d"}} +{"type":"Person","data":{"name":"e"}} +{"edge":"Knows","from":"a","to":"b"} +{"edge":"Knows","from":"c","to":"d"} +{"edge":"Knows","from":"d","to":"e"}"#; + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl(&mut db, data, LoadMode::Overwrite).await.unwrap(); + + let queries = r#" +query no_two_hop() { + match { + $p: Person + not { $p knows{2,2} $x } + } + return { $p.name } +} +"#; + let result = query_main(&mut db, queries, "no_two_hop", &ParamMap::new()) + .await + .unwrap(); + let batch = result.concat_batches().unwrap(); + let names = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let mut names_vec: Vec<&str> = (0..names.len()).map(|i| names.value(i)).collect(); + names_vec.sort(); + // Only c has a 2-hop knows path β†’ removed; everyone else (incl. a, which has + // a 1-hop neighbor but no 2-hop path) is kept. + assert_eq!(names_vec, vec!["a", "b", "d", "e"]); +} + // ─── Variable-length hops ─────────────────────────────────────────────────── const CHAIN_SCHEMA: &str = r#" diff --git a/crates/omnigraph/tests/traversal_indexed.rs b/crates/omnigraph/tests/traversal_indexed.rs new file mode 100644 index 0000000..2ceed85 --- /dev/null +++ b/crates/omnigraph/tests/traversal_indexed.rs @@ -0,0 +1,327 @@ +//! BTREE-indexed Expand path (`execute_expand_indexed`) coverage. +//! +//! These tests force the Expand execution mode via `OMNIGRAPH_TRAVERSAL_MODE` +//! and assert the indexed path matches the CSR path (both are semantically +//! identical β€” the indexed path just serves neighbor lookups from the persisted +//! src/dst BTREE instead of an in-memory CSR). They live in their own test +//! binary and are all `#[serial]`, so the env writes never race a concurrent +//! reader: within this process serial execution serializes every env read, and +//! other test binaries (e.g. `traversal.rs`) are separate processes whose env +//! stays unset (β†’ CSR), validating the shared hydrate/align tail on the CSR path. + +mod helpers; + +use arrow_array::{Array, StringArray}; + +use omnigraph::db::Omnigraph; +use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph::table_store::{IndexCoverage, TableStore}; +use omnigraph_compiler::ir::ParamMap; +use serial_test::serial; + +use helpers::*; + +fn set_mode(mode: &str) { + // SAFE: every test here is #[serial] and this binary has no non-serial + // env reader, so no thread reads the environment during this write. + unsafe { std::env::set_var("OMNIGRAPH_TRAVERSAL_MODE", mode) }; +} + +fn clear_mode() { + unsafe { std::env::remove_var("OMNIGRAPH_TRAVERSAL_MODE") }; +} + +/// Run a name-returning query and return its first column, sorted. +async fn sorted_names(db: &mut Omnigraph, queries: &str, name: &str, params: &ParamMap) -> Vec { + let result = query_main(db, queries, name, params).await.unwrap(); + if result.num_rows() == 0 { + return Vec::new(); + } + let batch = result.concat_batches().unwrap(); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let mut v: Vec = (0..col.len()).map(|i| col.value(i).to_string()).collect(); + v.sort(); + v +} + +/// Run the same query under CSR, indexed, and auto (cost-chooser) modes; assert +/// all three produce identical results and return them. The auto pass exercises +/// `choose_expand_mode` end to end: whichever path it selects, the rows must +/// match the forced paths (the chooser changes which path runs, never the result). +async fn both_modes(db: &mut Omnigraph, queries: &str, name: &str, params: &ParamMap) -> Vec { + set_mode("csr"); + let csr = sorted_names(db, queries, name, params).await; + set_mode("indexed"); + let indexed = sorted_names(db, queries, name, params).await; + clear_mode(); + let auto = sorted_names(db, queries, name, params).await; + assert_eq!( + indexed, csr, + "indexed Expand must produce identical results to CSR for query '{name}'" + ); + assert_eq!( + auto, csr, + "auto (cost-chooser) Expand must produce identical results to the forced paths for query '{name}'" + ); + indexed +} + +// The C6 index-coverage guard: `key_column_index_coverage` must report whether +// a `key_col IN (...)` scan will use the persisted BTREE or silently full-scan. +// Not #[serial] β€” it calls the helper directly and reads no env. +#[tokio::test] +async fn key_column_index_coverage_detects_btree_presence() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + let snap = snapshot_main(&db).await.unwrap(); + + // Edge `src` gets a BTREE from ensure_indices on load β†’ Indexed. + let edge_ds = snap.open("edge:Knows").await.unwrap(); + let src_cov = TableStore::key_column_index_coverage(&edge_ds, "src") + .await + .unwrap(); + assert_eq!(src_cov, IndexCoverage::Indexed, "edge src is BTREE-indexed"); + + // A node property column with no scalar index β†’ Degraded (the warn path). + let node_ds = snap.open("node:Person").await.unwrap(); + let age_cov = TableStore::key_column_index_coverage(&node_ds, "age") + .await + .unwrap(); + assert!( + matches!(age_cov, IndexCoverage::Degraded { .. }), + "non-indexed column should be Degraded, got {age_cov:?}" + ); +} + +// An edge appended after the BTREE was built lands in a new fragment that the +// index does not cover (edge-index creation is skipped once a BTREE exists). The +// scan is then partly a full scan, so coverage must report `Degraded` β€” otherwise +// the cost chooser would price an unindexed-in-part scan as fully indexed. +// (Results stay correct regardless β€” `indexed_finds_unindexed_appended_edge`.) +#[tokio::test] +async fn coverage_degrades_for_appended_unindexed_fragment() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + + // Fresh load: the Knows BTREE covers every fragment β†’ Indexed. + let snap = snapshot_main(&db).await.unwrap(); + let edge_ds = snap.open("edge:Knows").await.unwrap(); + assert_eq!( + TableStore::key_column_index_coverage(&edge_ds, "src").await.unwrap(), + IndexCoverage::Indexed, + "freshly-loaded edge BTREE covers all fragments" + ); + + // Append an edge β†’ a new, unindexed fragment outside the index fragment_bitmap. + mutate_main( + &mut db, + MUTATION_QUERIES, + "add_friend", + ¶ms(&[("$from", "Alice"), ("$to", "Diana")]), + ) + .await + .unwrap(); + + let snap2 = snapshot_main(&db).await.unwrap(); + let edge_ds2 = snap2.open("edge:Knows").await.unwrap(); + let cov = TableStore::key_column_index_coverage(&edge_ds2, "src").await.unwrap(); + assert!( + matches!(cov, IndexCoverage::Degraded { .. }), + "appended unindexed fragment must degrade coverage, got {cov:?}" + ); +} + +#[tokio::test] +#[serial] +async fn indexed_matches_csr_one_hop_same_type() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + // friends_of: `$p knows $f` (Person -> Person, single hop). + let got = both_modes(&mut db, TEST_QUERIES, "friends_of", ¶ms(&[("$name", "Alice")])).await; + assert_eq!(got, vec!["Bob", "Charlie"], "Alice knows Bob and Charlie"); +} + +#[tokio::test] +#[serial] +async fn indexed_matches_csr_multi_hop_same_type() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + let queries = r#" +query reach($name: String) { + match { + $p: Person { name: $name } + $p knows{1,2} $f + } + return { $f.name } +} +"#; + // Alice -> Bob, Charlie (1 hop); Bob -> Diana (2 hops). + let got = both_modes(&mut db, queries, "reach", ¶ms(&[("$name", "Alice")])).await; + assert_eq!(got, vec!["Bob", "Charlie", "Diana"]); +} + +#[tokio::test] +#[serial] +async fn indexed_matches_csr_cross_type() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + let queries = r#" +query employer($name: String) { + match { + $p: Person { name: $name } + $p worksAt $c + } + return { $c.name } +} +"#; + let got = both_modes(&mut db, queries, "employer", ¶ms(&[("$name", "Alice")])).await; + assert_eq!(got, vec!["Acme"], "Alice works at Acme"); +} + +#[tokio::test] +#[serial] +async fn indexed_matches_csr_no_match() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + // Diana has no outgoing Knows edges β†’ empty in both modes. + let got = both_modes(&mut db, TEST_QUERIES, "friends_of", ¶ms(&[("$name", "Diana")])).await; + assert!(got.is_empty(), "Diana knows no one"); +} + +#[tokio::test] +#[serial] +async fn indexed_finds_unindexed_appended_edge() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + + // Append Alice -> Diana AFTER the initial load. `ensure_indices`' existence + // guard means the src/dst BTREE built on the first load does NOT cover this + // new fragment. The indexed path must still find it via Lance's + // unindexed-fragment scan (fast_search=false default), so partial index + // coverage never silently drops rows. + mutate_main( + &mut db, + MUTATION_QUERIES, + "add_friend", + ¶ms(&[("$from", "Alice"), ("$to", "Diana")]), + ) + .await + .unwrap(); + + set_mode("indexed"); + let got = sorted_names(&mut db, TEST_QUERIES, "friends_of", ¶ms(&[("$name", "Alice")])).await; + clear_mode(); + + assert_eq!( + got, + vec!["Bob", "Charlie", "Diana"], + "indexed traversal must see the freshly-appended, unindexed edge" + ); +} + +// Regression: a node `id` is unique only WITHIN a type, so a `Person` and a +// `Company` can share an id string. A variable-length traversal over a +// cross-type edge (`worksAt`, Person -> Company) must structurally stop after +// one hop β€” a Company is not a `worksAt` source β€” so `worksAt{1,2}` returns +// exactly the one-hop companies. Before the structural hop-cap, the indexed +// path's single string interner de-interned the hop-1 Company id back to the +// colliding Person id and ran a hop-2 `worksAt src IN (...)` scan that matched +// that same-string Person's edges, emitting a spurious second-hop company the +// CSR path never produces. `both_modes` (csr == indexed == auto) plus the +// golden assert catch both the divergence and an over-emitting shared bug. +#[tokio::test] +#[serial] +async fn cross_type_id_collision_does_not_bleed_into_second_hop() { + const SCHEMA: &str = r#" +node Person { name: String @key } +node Company { name: String @key } +edge WorksAt: Person -> Company +"#; + // `shared` is BOTH a Person id and a Company id. alice worksAt the Company + // `shared`; the Person `shared` worksAt the Company `other`. + const DATA: &str = r#"{"type":"Person","data":{"name":"alice"}} +{"type":"Person","data":{"name":"shared"}} +{"type":"Company","data":{"name":"shared"}} +{"type":"Company","data":{"name":"other"}} +{"edge":"WorksAt","from":"alice","to":"shared"} +{"edge":"WorksAt","from":"shared","to":"other"}"#; + const QUERY: &str = r#" +query reach($name: String) { + match { + $p: Person { name: $name } + $p worksAt{1,2} $c + } + return { $c.name } +} +"#; + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, SCHEMA).await.unwrap(); + load_jsonl(&mut db, DATA, LoadMode::Overwrite).await.unwrap(); + + let got = both_modes(&mut db, QUERY, "reach", ¶ms(&[("$name", "alice")])).await; + assert_eq!( + got, + vec!["shared"], + "cross-type worksAt{{1,2}} must return only the one-hop company; a hop-2 \ + result means the id-string collision bled across types" + ); +} + +const REACH_5: &str = r#" +query reach($name: String) { + match { + $p: Person { name: $name } + $p knows{1,5} $f + } + return { $f.name } +} +"#; + +// A directed 3-cycle a->b->c->a, traversed with a hop ceiling (5) ABOVE the cycle +// length. Variable-length traversal must terminate and dedup (the source is +// seeded into `visited`, so the c->a back-edge does not re-emit a). Uses a +// bounded range deliberately: an unbounded `{1,}` is a typecheck error, not a +// runtime path. `both_modes` also confirms indexed == csr on the cycle. +#[tokio::test] +#[serial] +async fn variable_hops_terminate_and_dedup_on_cycle() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let data = r#"{"type":"Person","data":{"name":"a"}} +{"type":"Person","data":{"name":"b"}} +{"type":"Person","data":{"name":"c"}} +{"edge":"Knows","from":"a","to":"b"} +{"edge":"Knows","from":"b","to":"c"} +{"edge":"Knows","from":"c","to":"a"}"#; + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl(&mut db, data, LoadMode::Overwrite).await.unwrap(); + + let got = both_modes(&mut db, REACH_5, "reach", ¶ms(&[("$name", "a")])).await; + // From a: b (1 hop), c (2 hops); the c->a back-edge hits the seeded source + // and is not re-emitted. No infinite loop, each node at most once. + assert_eq!(got, vec!["b", "c"]); +} + +// A self-loop a->a plus a->b. Variable-length traversal must not loop forever and +// must not re-emit the seeded source. +#[tokio::test] +#[serial] +async fn variable_hops_handle_self_loop() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let data = r#"{"type":"Person","data":{"name":"a"}} +{"type":"Person","data":{"name":"b"}} +{"edge":"Knows","from":"a","to":"a"} +{"edge":"Knows","from":"a","to":"b"}"#; + let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + load_jsonl(&mut db, data, LoadMode::Overwrite).await.unwrap(); + + let got = both_modes(&mut db, REACH_5, "reach", ¶ms(&[("$name", "a")])).await; + // a->a hits the seeded source (pruned); only b is reached. + assert_eq!(got, vec!["b"]); +} diff --git a/crates/omnigraph/tests/writes.rs b/crates/omnigraph/tests/writes.rs index 13cb10f..d76ad46 100644 --- a/crates/omnigraph/tests/writes.rs +++ b/crates/omnigraph/tests/writes.rs @@ -6,8 +6,8 @@ //! What this file covers: //! - No `__run__*` branches are created by load or mutate. //! - Cancellation of a mutation future leaves no graph-level state. -//! - Concurrent writers to the same table land exactly one publish; the -//! loser surfaces `ManifestConflictDetails::ExpectedVersionMismatch`. +//! - Concurrent non-strict inserts/merges rebase under the per-table queue; +//! strict updates/deletes surface `ExpectedVersionMismatch` on stale state. //! - Failed mutations and loads leave the target unchanged. //! - Multi-statement mutations are atomic (one commit per query). //! - actor_id propagates through to the commit graph. @@ -17,7 +17,7 @@ mod helpers; use arrow_array::Array; use omnigraph::db::commit_graph::CommitGraph; use omnigraph::db::{Omnigraph, ReadTarget}; -use omnigraph::error::{ManifestConflictDetails, ManifestErrorKind, OmniError}; +use omnigraph::error::OmniError; use omnigraph::loader::{LoadMode, load_jsonl}; use helpers::*; @@ -241,18 +241,11 @@ async fn partial_failure_leaves_target_queryable_and_unblocks_next_mutation() { assert_eq!(frank.num_rows(), 1, "Frank must be visible after publish"); } -/// Concurrent writers to the same `(table, branch)` produce exactly one -/// success and one `ExpectedVersionMismatch`. The replacement for the old -/// `concurrent_conflicting_run_publish_fails_cleanly` test β€” the OCC fence -/// has moved from a graph-level run-publish merge into the publisher's -/// per-table CAS. -/// -/// Drives the race by interleaving two handles that captured the same -/// pre-write manifest snapshot: A commits first; B's commit then sees -/// `expected_versions[node:Person] = pre` while the manifest is at -/// `pre + 1`, and the publisher rejects. +/// Stale non-strict writers rebase to the live manifest pin under the +/// per-table queue instead of folding raw drift or returning a false 409. +/// Strict update/delete semantics are covered by the consistency/server tests. #[tokio::test] -async fn concurrent_writers_one_succeeds_one_gets_expected_version_mismatch() { +async fn stale_non_strict_insert_rebases_to_live_manifest_pin() { let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_string_lossy().into_owned(); @@ -281,40 +274,30 @@ async fn concurrent_writers_one_succeeds_one_gets_expected_version_mismatch() { .unwrap(); } - // Writer B's coordinator is still at the pre-A snapshot. Its mutation - // captures expected_versions[node:Person] = pre (stale), then publishes - // β€” the publisher's CAS pre-check sees the manifest is now at post and - // rejects with ExpectedVersionMismatch. - let result_b = db_b - .mutate( - "main", - MUTATION_QUERIES, - "insert_person", - &mixed_params(&[("$name", "WriterB")], &[("$age", 42)]), - ) - .await; + // Writer B's coordinator is still at the pre-A snapshot, but Insert is + // non-strict: commit_all re-reads the live manifest pin under the queue, + // verifies Lance HEAD equals that pin, and then lets Lance rebase the + // staged append. + db_b.mutate( + "main", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "WriterB")], &[("$age", 42)]), + ) + .await + .unwrap(); - let err = result_b.expect_err("stale writer must hit ExpectedVersionMismatch"); - let OmniError::Manifest(manifest_err) = err else { - panic!("expected Manifest error, got {err:?}"); - }; - assert_eq!(manifest_err.kind, ManifestErrorKind::Conflict); - let Some(ManifestConflictDetails::ExpectedVersionMismatch { - ref table_key, - expected, - actual, - }) = manifest_err.details - else { - panic!( - "expected ExpectedVersionMismatch, got {:?}", - manifest_err.details, - ); - }; - assert_eq!(table_key, "node:Person"); - assert!( - actual > expected, - "actual ({actual}) should be ahead of expected ({expected})", - ); + for name in ["WriterA", "WriterB"] { + let person = query_main( + &mut db_b, + TEST_QUERIES, + "get_person", + ¶ms(&[("$name", name)]), + ) + .await + .unwrap(); + assert_eq!(person.num_rows(), 1, "{name} should be visible"); + } } /// The cancellation hole that motivated removing the Run state machine: dropping a mutation future @@ -371,11 +354,10 @@ async fn cancelled_mutation_future_leaves_no_state() { // Cancel-safety property: no graph-level run/staging state remains. // - // Note: `branch_list()` already filters `__run__*` via - // `is_internal_system_branch`, so a runtime "no `__run__` branches" check - // would be vacuous. The structural property that no `__run__` branches - // can ever be created is enforced by deletion of `begin_run` etc. in - // (verified by the build itself β€” those symbols no longer exist). + // No `__run__` branches can ever be created: the Run state machine + // (`begin_run` etc.) was deleted in MR-771 β€” verified by the build itself, + // those symbols no longer exist. Any legacy `__run__*` branch on an + // upgraded graph is swept by the v2β†’v3 manifest migration. // // (1) The branch list is unchanged: cancellation/completion cannot // synthesize new public branches. @@ -442,34 +424,40 @@ async fn repeated_loads_do_not_accumulate_branches() { assert_eq!(db.branch_list().await.unwrap(), vec!["main".to_string()]); } -/// User code must not be able to write to internal `__run__*` names. -/// The branch-name guard predicate is kept as defense-in-depth; it -/// will be removed once a future production sweep retires the legacy -/// branches. +/// After MR-770, `__run__*` is an ordinary branch name β€” the Run state machine +/// and its `is_internal_run_branch` guard are gone. The surviving internal-ref +/// guard still rejects the active `__schema_apply_lock__` branch on the public +/// create/merge APIs. #[tokio::test] -async fn public_branch_apis_reject_internal_run_refs() { +async fn public_branch_apis_reject_internal_system_refs() { let dir = tempfile::tempdir().unwrap(); let mut db = init_and_load(&dir).await; - let create_err = db.branch_create("__run__synthetic").await.unwrap_err(); + // `__run__*` is no longer reserved β€” creating it now succeeds. + db.branch_create("__run__formerly_reserved") + .await + .expect("__run__ prefix is a normal branch name post-MR-770"); + + // The schema-apply lock branch is still rejected on public branch APIs. + let create_err = db.branch_create("__schema_apply_lock__").await.unwrap_err(); let OmniError::Manifest(err) = create_err else { panic!("expected Manifest error"); }; assert!( - err.message.contains("internal run ref"), + err.message.contains("internal system ref"), "unexpected error: {}", err.message ); let merge_err = db - .branch_merge("__run__synthetic", "main") + .branch_merge("__schema_apply_lock__", "main") .await .unwrap_err(); let OmniError::Manifest(err) = merge_err else { panic!("expected Manifest error"); }; assert!( - err.message.contains("internal run refs"), + err.message.contains("internal system refs"), "unexpected error: {}", err.message ); diff --git a/docs/dev/branch-protection.md b/docs/dev/branch-protection.md index 9b2fa78..2b6cc37 100644 --- a/docs/dev/branch-protection.md +++ b/docs/dev/branch-protection.md @@ -8,7 +8,7 @@ This page explains what the policy says and how to change it. | Setting | Value | Why | |---|---|---| -| **Required status checks (strict)** | `Classify Changes`, `Check AGENTS.md Links`, `Test Workspace`, `Test omnigraph-server --features aws`, `CODEOWNERS / drift`, `CODEOWNERS / noedit` | Every PR must pass workspace tests, AGENTS.md link integrity, and the CODEOWNERS hygiene checks. `strict: true` requires the branch to be up-to-date with `main` before merge. | +| **Required status checks (strict)** | `Classify Changes`, `Check AGENTS.md Links`, `Test Workspace`, `Test omnigraph-server --features aws`, `CODEOWNERS matches source`, `CODEOWNERS not hand-edited` | Every PR must pass workspace tests, AGENTS.md link integrity, and the CODEOWNERS hygiene checks. The two CODEOWNERS contexts must equal the job `name:` values in `.github/workflows/codeowners.yml` **verbatim** β€” a context naming a job that never reports (the old `CODEOWNERS / drift` used the job *id*, and the job was path-filtered) leaves every PR permanently pending and forces admin overrides. `strict: true` requires the branch to be up-to-date with `main` before merge. | | **Required approving reviews** | `1` | At least one reviewer. With a 2-person team, going higher would block all merges when one person is unavailable. | | **Require code-owner reviews** | `true` | The reviewer must be a code owner per `.github/CODEOWNERS`. This is what makes the codeowners chassis enforced. | | **Dismiss stale reviews on new commits** | `true` | A push after approval invalidates the prior review. Prevents the "approve, then sneak in unreviewed changes" pattern. | @@ -16,7 +16,7 @@ This page explains what the policy says and how to change it. | **Disallow force pushes** | `true` | No history rewrites on `main`. | | **Disallow branch deletions** | `true` | `main` cannot be deleted. | | **Required conversation resolution** | `true` | All review comment threads must be resolved before merge. | -| **Enforce on admins** | `true` | Even repository admins go through the gates. The point is no bypasses. | +| **Enforce on admins** | `false` | Admins can override the gates (`enforce_admins: false` in the JSON). This is the intended escape hatch for the 2-person team; tightening to `true` is tracked under hardening below. | | **Required signed commits** | not yet | Not enabled. Would lock out maintainers until everyone enrolls GPG/SSH commit signing. Tracked as a follow-up. | ## How to apply diff --git a/docs/dev/cluster-axioms.md b/docs/dev/cluster-axioms.md new file mode 100644 index 0000000..a3793b4 --- /dev/null +++ b/docs/dev/cluster-axioms.md @@ -0,0 +1,97 @@ +# Cluster Control-Plane Axioms + +**Type:** Standing design filter +**Status:** Draft / thinking-in-progress +**Date:** 2026-06-07 +**Relationship:** the distilled axioms behind [cluster-config-specs.md](cluster-config-specs.md). The downstream implementation inventory and blast-radius assessment live in [cluster-config-implementation-spec.md](cluster-config-implementation-spec.md). The high-level spec is the argument; this is the checklist. Hold any config / control-plane / deployment proposal against these and cite them by number (e.g. "violates axiom 5"). + +This file is intentionally short and stable. The axioms are phrased so other +docs can reference "axiom 6" without churn. The motivating requirement comes +first; the core axioms are what the design is *based on*; the derived rules are +consequences that follow from them. + +> **Revision 2026-06-07 β€” committed to the Terraform paradigm.** State is now an +> **authoritative, locked ledger in a backend** (no longer framed as a +> "mostly-rebuildable projection"); `plan` is a **config ↔ state diff**; and +> **ETL pipelines** join schema as config-defined resources that trigger +> data-plane effects. Secrets live in a gitignored **`.env`** file (`${NAME}`), +> and **query exposure is a policy decision** (no registry `expose:` flag). +> Axioms **2, 5, 6** revised; **12, 13, 14** added. The earlier +> "state is just a rebuildable projection; config is the *only* truth" framing is +> superseded β€” see axiom 5. +> +> **Revision 2026-06-08 β€” JSON state first.** The baseline state backend is now +> Terraform-style JSON documents plus backend lock/CAS, not Lance control-plane +> datasets. Lance remains a possible later backend only if row-level history or +> queryability justifies the extra machinery. + +--- + +## Tenet 0 β€” the motivating requirement + +**0. The Sarah/Bob test.** If one operator changes schema / queries / policies / UI / pipelines / aliases, another operator (or their agent) must learn *what the deployment is and what changed* from **one source, one history, one diff**. Fragmentation across separate mechanisms is the failure the whole design exists to eliminate. Every other axiom is in service of passing this test. + +--- + +## Core axioms (what the design is based on) + +**1. The cluster is the unit of declarative state.** Not the graph (policies, queries, UI, and pipelines cross-cut graphs; "which graphs exist" has no per-graph home), not the fleet (the next scope up β€” named and deferred). The cluster is what two operators collaborate over; a graph is a *resource within* it. + +**2. Two sources of truth, for two different questions β€” config for *intent*, state for *deployed reality*.** The version-controlled **config** (a set of files in one folder) is the source of truth for what the cluster *should be*. The **state ledger** is the source of truth for what *is* currently deployed. Change flows one way only: you edit config and `apply` converges the cluster (**code β†’ cluster**, never edit-the-cluster-and-call-it-intent). But "what exists right now" is read from **state**, not re-derived from the world on every command. `plan` is the diff between the two. + +**3. Declarative, not imperative.** You describe the desired end state; the reconciler computes the steps. No runtime mutation API that makes the running system the place *intent* lives. + +**4. As-code is structural, not stylistic (the recursion argument).** Code is the base case; modeling the definition *as data* (a meta-graph describing graphs) recurses with no base case. Config must live **outside** the running system so it is reviewable (PRs), reproducible (clone + apply), diffable as text, and editable by an agent β€” without the system having to describe itself. + + +**5. The Terraform model: config / state / reconcile β€” and state is an authoritative, locked ledger.** Config (as code) = desired truth. **State = the authoritative record of what has been applied**, held in a **backend** β€” the cluster's own object-store backend *or* a separate cloud store, the operator's choice, exactly like a Terraform backend. The baseline representation is JSON documents (`state.json`, status/approval/recovery JSON records) protected by backend lock/CAS, not Lance control-plane tables. State is **locked** during apply so two operators cannot converge concurrently. `validate` parses and schema-checks desired config; `plan` = `diff(config, state)` as a structured artifact with resource digests, dependency edges, state observations, proposed changes, blast radius, and approval gates; `apply` converges the cluster from an accepted fresh plan and **updates state**, and does not acknowledge success until state has recorded the result. A cluster-hosted JSON backend is still a separate state CAS step from graph Lance manifest moves; failures surface a repair/import condition instead of being described as cross-object all-or-nothing. A future Lance-backed state backend or cluster manifest publisher is optional and must earn its complexity by needing row-level queryability/history or tighter publish fencing. Because OmniGraph's running cluster is self-describing (manifests, commit logs), state is *reconstructable* by import/refresh if lost β€” its edge over opaque-cloud Terraform β€” but it is **treated as the source of truth for current reality, not casually regenerated**. The one slice that can never be reconstructed (who approved an irreversible apply) lives in the durable audit ledger; state references it (axiom 11). + +**6. The control plane reconciles definition, not data β€” across two data-plane seams.** Definition β€” schema, policies, queries, UI, bindings, aliases, ETL **pipelines**, embeddings config, and the set of graphs β€” is reconciled. Data β€” rows, edges, vectors β€” is data-plane content, versioned by the commit DAG and produced by `load` / `mutate` and **pipeline execution**, sitting **outside** the reconcile loop. Exactly two definition kinds *trigger* a data-plane effect without owning data: **schema** (a migration conforms existing rows; `plan` previews its impact) and **ETL pipelines** (their execution ingests external data). The loop converges their *definitions*; the data they produce is never what it reconciles. + +**7. Operated by agent (agent-as-controller).** An agent authors config changes and drives reconciliation as an authenticated actor, subject to policy and approval gates β€” no human state-management burden. This fuses Terraform's as-code config with Kubernetes' continuous reconciliation. + +--- + +## Derived rules (consequences of the axioms) + +**8. The reversibility gradient gates apply β€” including drift correction.** Irreversible / data-loss operations (drop a graph, hard-drop schema data, a pipeline that overwrites) and compatibility-narrowing migrations (for example, future validated enum narrowing) are gated; reversible ones (recolor a dashboard) are not. The gate is keyed to physics, not to who operates it, and a reconciler "just fixing drift" is never an exception. + +**9. Atomicity and referential integrity are plan-time, not runtime.** `ApplyGroup` is the atomicity unit; cross-resource references *force* grouping (mandatory, not opt-in); references use typed resource/provider addresses (`graph.knowledge`, `query.knowledge.find_experts`, `provider.source.github_org`) so the planner can reject wrong-kind or missing targets before apply β€” bare names in a kind-fixed field are accepted shorthand and normalized to the typed address (fix 2026-06-08), while a kind-ambiguous value (e.g. `source: github`) is rejected; a reference to a missing or being-removed resource is a fail-closed `plan` error, not a deferred runtime failure. + +**10. Secrets live in a `.env` file; connection/identity is per-operator.** The committed cluster config carries **no secret values** β€” only `${NAME}` references. The values (embedding API keys, pipeline **source credentials**, per-deployment settings) live in a separate **`.env` file** β€” which is gitignored and supplied per deployment, never committed. Separately, an operator's own connection (which cluster, which token) is the per-operator layer, distinct from both the shared config and its `.env` file. + +**11. Approvals and audit live in a durable ledger, not inline in state.** State *references* the audit record by id. In the baseline, that ledger is append-only JSON records in the state backend; a future Lance table is an implementation option, not a requirement. This keeps the bulk of state reconstructable and keeps approval facts β€” "who authorized this irreversible apply" β€” where loss is impossible. + +**12. State lives in a backend and is locked.** The state ledger is stored in a configurable backend β€” the cluster's own backend, or a separate cloud store β€” and `plan`/`apply` acquire a **state lock** first, so concurrent applies serialize instead of racing. (Generalizes the existing `__schema_apply_lock__` from schema scope to cluster scope.) The backend choice is part of the safety model: the first backend should be JSON plus object-store lock/CAS; any Lance-backed state backend needs its own RFC-level proof that the table semantics are worth the control-plane complexity. + +**13. Pipelines are definition; their execution is data-plane.** An ETL pipeline (external source β†’ transform β†’ target graph) is **declared in config and reconciled like any resource**; *running* it produces ordinary data-plane writes (`load`/`mutate`) outside the reconcile loop. `apply` converges the pipeline's *definition* (create / update / delete / schedule); the rows it ingests are never reconciled. A fan-out run over several graphs is statusful rather than magically atomic: each target records commit id, status, retryability, and idempotency key unless the pipeline explicitly uses a branch/merge protocol that can fence the whole target set. Source credentials are secret references (axiom 10). + + +**14. Exposure is a policy decision, not a config flag.** Target design: which stored queries (and the tools/dashboards built on them) an actor may **list or invoke** is decided by the policy layer (Cedar: `invoke_query` + catalog visibility), not by a per-query `expose:` boolean. The registry only says a query *exists* (name β†’ file); **policy says who may see and run it**, so the MCP catalog (`GET /queries`) becomes each actor's policy-permitted set. This supersedes the engine's current `mcp.expose` flag only after per-query `invoke_query` scope and Cedar-filtered catalog listing land; until then, proposals must state the compatibility bridge to today's `mcp.expose` + coarse invocation gate. + +--- + +## The one-line compression + +**One cluster; config (a folder of files) is desired truth and a locked state ledger in a backend is deployed truth; `plan` diffs them, `apply` converges the cluster and updates state, an agent drives the loop β€” reconciling the cluster's *definition* (schema, policies, queries, UI, pipelines, …) and never its data β€” so any operator sees the whole system and its history from one place.** + +--- + +## How to use this file + +- **Reviewing a proposal:** walk axioms 0–14; any conflict is the burden of the proposer to justify. The most common tensions: + - Treating the *running system* as the source of truth for **intent** β†’ axioms 2, 4 (intent lives in config). + - Treating state as a throwaway derivation rather than an authoritative, locked, backend-held ledger β†’ axiom 5, 12. + - A runtime config-mutation API instead of declarative apply β†’ axiom 3. + - "State" meaning a per-operator selection rather than the applied-cluster ledger β†’ axiom 5. + - The control plane reconciling (or owning) data β€” including treating pipeline *rows* as reconciled state β†’ axiom 6, 13. + - Treating fan-out pipeline execution as atomic without a branch/merge protocol or per-target status ledger β†’ axiom 13. + - Per-graph or per-server scoping of cluster-level definition β†’ axiom 1. + - Bare string references that force the planner to guess whether `knowledge` means a graph, query, provider, or path β†’ axiom 9. + - A secret value (token, embedding key, pipeline source credential) inline in config instead of in the gitignored `.env` file β†’ axiom 10. + - A per-query `expose:`/visibility flag in target-state cluster config instead of governing list/invoke in policy; or failing to account for today's `mcp.expose` compatibility bridge β†’ axiom 14. + - Shipping `apply` before hermetic `validate` + read-only `plan` tests, or shipping graph/schema-moving apply before recovery tests for the graph/resource-moved-before-cluster-publish gap β†’ axiom 5 and axiom 12. +- **Citing:** reference axioms by number in PRs and review comments so the rationale is stable across renames and refactors. diff --git a/docs/dev/cluster-config-implementation-spec.md b/docs/dev/cluster-config-implementation-spec.md new file mode 100644 index 0000000..5121451 --- /dev/null +++ b/docs/dev/cluster-config-implementation-spec.md @@ -0,0 +1,705 @@ +# Cluster Config Implementation Spec And Blast Radius + +**Status:** Draft / implementation planning +**Type:** Downstream design spec +**Date:** 2026-06-08 +**Relationship:** companion to [cluster-config-specs.md](cluster-config-specs.md) +and [cluster-axioms.md](cluster-axioms.md). The high-level spec explains why +the cluster control plane should exist; this file names what must change +downstream and how large the blast radius is. + + + +## Executive Summary + +Overall blast radius: **very high**. + +This is not a small extension to `omnigraph.yaml`. The target design creates a +new shared cluster desired-state document, a locked state ledger, a cluster +manifest publisher, and a reconciler that coordinates resources above a single +graph. The existing config system remains useful, but its role changes: + +- `omnigraph.yaml` / global config remains the per-operator and startup bridge. +- `cluster.yaml` becomes shared desired state for a deployment. +- The cluster state ledger becomes the authoritative record of applied reality. +- Server/runtime surfaces eventually read from the cluster catalog instead of + only from process-start config. + +Safe rollout requires an additive path. Do not replace the current config, +server, or policy behavior in one step. + +## Current Surfaces Surveyed + +| Surface | Current behavior | Why it matters | +|---|---|---| +| `omnigraph-config::OmnigraphConfig` | Layered global/state/project config for CLI and server startup; strict `version: 1`; named maps replace wholesale | A cluster spec needs different ownership and merge semantics; do not stretch this type until it becomes ambiguous | +| `omnigraph-server::load_server_settings` | Opens either one selected graph or every configured embedded graph in multi mode | Cluster config changes startup, registry identity, and eventually runtime reconcile | +| `GraphRegistry` | Holds open graph handles; production registry is startup-only today; runtime insert is test-only | Cluster apply wants graph add/remove/reload as real control-plane operations | +| `omnigraph-queries::QueryRegistry` | Loads `.gq` files from `queries:` and honors `mcp.expose` for catalog listing | Target cluster config removes exposure from the registry and moves list/invoke to policy | +| `omnigraph-policy::PolicyAction` | Per-graph actions plus server-scoped `graph_list`; `invoke_query` is graph-scoped and coarse | Cluster plan/apply and per-query exposure need new policy scope without breaking coarse rules | +| Engine graph manifest | Graph-level atomic visibility via `__manifest`, expected table versions, and recovery sidecars | Cluster apply needs a higher-level publisher; Lance still commits per dataset | +| Schema apply | Existing plan/apply/lock shape for one graph; soft/hard drops already modeled | This is the prototype resource reconciler, but cluster apply cannot call it blindly and then claim cluster atomicity | +| Public docs/tests | Config, policy, server, and query behavior are already documented and tested | Every behavior change below has user docs and test fallout | + +## Compatibility Stance + + + +1. `cluster.yaml` is a new target-state file, not `omnigraph.yaml` v2. +2. Existing `omnigraph.yaml` keeps working for CLI, server boot, aliases, + graph locators, bearer-token env lookup, and the current stored-query + registry. +3. Initial cluster commands are explicit: `omnigraph cluster validate`, + `omnigraph cluster plan`, `omnigraph cluster apply`, `omnigraph cluster + status`, `omnigraph cluster refresh`, and `omnigraph cluster import`. +4. Cluster config is one shared folder, resolved from the command's cluster + root or explicit path. It is not merged from global + project + active + context layers. +5. The per-operator connection layer selects the cluster root and actor + identity. It is not committed into `cluster.yaml`. +6. `mcp.expose` remains supported in current `omnigraph.yaml` until the + per-query policy replacement ships. + +## Terraform-Aligned Schema Validation + + + +Every field in target-state `cluster.yaml` must be **honored or rejected**: + +- If a field is part of the declared resource schema, it must affect + validation, plan, apply, state, or status. +- If a field is misspelled, placed under the wrong resource kind, or reserved + for a future phase, `cluster validate` / `cluster plan` must fail with a + typed diagnostic. +- Compatibility warnings are allowed only in an explicit migration window for + old schema versions. They are not allowed in the target schema. +- Free-form extension areas must be named as such, for example `labels`, + `metadata`, `vars`, or `provider_options`; accidental unknown keys are never + treated as extension data. + +Examples: + +```yaml +graphs: + knowledge: + schema: ./knowledge.pg + lables: { team: platform } # invalid: typo, use `labels` + +pipelines: + github_sync: + source: { kind: github, token: ${GITHUB_TOKEN} } + into: + - { graph: engineering, map: ./github.map.yaml } + retry_magic: true # invalid unless `retry_magic` is in schema +``` + +```yaml +graphs: + knowledge: + schema: ./knowledge.pg + labels: { team: platform } # valid free-form metadata bucket + provider_options: + lance: + compaction_window: daily # valid only if this extension is declared +``` + +## Typed Resource And Provider Addresses + + + + +A locator is a typed address to another declared thing. **Internally β€” in plan and +state β€” every reference is a typed address** (axiom 9). At the config *surface* a +field may accept **bare shorthand when its schema fixes the referent kind** (a +policy `applies_to:` list is graph refs; a pipeline `into.graph` is a graph id) β€” +the parser normalizes it to the typed address before planning. A value whose +*kind* is ambiguous or wrong (a `source:` that could be a connector type, an +instance, or a provider) has no safe normalization and must be a typed +`provider.*` address or an explicit inline block. + +Target address forms: + +```text +graph. +schema. +query.. +policy. +ui.dashboard. +pipeline. +provider.storage. +provider.source. +provider.embedding. +``` + +Bad shape β€” the value's **kind is ambiguous or wrong**, not merely bare: + +```yaml +pipelines: + github_sync: + source: github # AMBIGUOUS kind: connector type, instance, or provider? + # β†’ provider.source. or inline { kind: github, ... } +policies: + base_rbac: + applies_to: [query.knowledge.find_experts] # WRONG kind: a query address in a graph-ref field +``` + +OK shorthand (kind fixed by the field β†’ normalized): + +```yaml +policies: + base_rbac: + applies_to: [knowledge, engineering] # bare names in a graph-ref field β†’ graph.knowledge, graph.engineering +``` + +Target shape: + +```yaml +providers: + storage: + prod_graphs: + kind: s3 + bucket: company + prefix: prod + source: + github_org: + kind: github + token: ${GITHUB_TOKEN} + +graphs: + knowledge: + storage: provider.storage.prod_graphs + path: graphs/knowledge.omni + schema: ./knowledge.pg + engineering: + storage: provider.storage.prod_graphs + path: graphs/engineering.omni + schema: ./engineering.pg + +policies: + base_rbac: + file: ./base_rbac.policy.yaml + applies_to: + - graph.knowledge + - graph.engineering + +pipelines: + github_sync: + source: provider.source.github_org + into: + - { graph: graph.engineering, map: ./github_to_engineering.map.yaml } + - { graph: graph.knowledge, map: ./github_to_people.map.yaml } +``` + + + +Validation rules: + +- A field that expects a graph address accepts `graph.`, not + `query..` or an arbitrary string. +- A field that expects a query address accepts `query..`, and the + planner validates both the graph and the query symbol. +- A field that expects a source provider accepts `provider.source.`, not + `provider.storage.`. +- A field that expects storage accepts `provider.storage.` or an explicit + storage block, not a server URL or source connector. + +- A field whose schema **fixes the kind** accepts bare shorthand (e.g. `knowledge` + in a graph-ref field) and normalizes it to the typed address; a kind-ambiguous + or wrong-kind value is rejected with a typed diagnostic. +- Plan and state always store the **normalized typed address**, regardless of + whether the surface used shorthand. + +## Target Components + +Preferred split: + +| Component | Responsibility | Depends on | +|---|---|---| +| `omnigraph-cluster` crate | Cluster spec types, path resolution, resource graph, plan model, state backend traits, apply orchestration | `omnigraph-config` only for shared simple config types if needed; avoid server deps | +| `omnigraph` engine additions | Graph lifecycle primitives, schema-apply integration, recovery hooks for graph moves during cluster apply; optional future cluster manifest publisher if JSON state is not enough | Lance, existing graph manifest/recovery | +| `omnigraph-cli` | `cluster *` commands, plan rendering, approval collection, state lock UX | `omnigraph-cluster`, engine | +| `omnigraph-server` | Optional boot from cluster state, registry reload, status endpoints, policy-filtered query catalog | `omnigraph-cluster`, engine, policy | +| `omnigraph-policy` | Cluster/server actions, per-query list/invoke scope, approval policy predicates | none above server | +| `omnigraph-queries` | Registry without exposure side-channel; dependency metadata for downstream validation | compiler/config | +| `omnigraph-api-types` | New status/plan/apply response types if cluster HTTP endpoints ship | serde only | + +If the first implementation avoids a new crate, keep the same boundary in +modules. The important constraint is that cluster spec parsing must not drag +HTTP/server code into compiler or engine crates. + +## Resource Model + +Resource identity is stable and typed: + +```text +ClusterRoot +ResourceKey = // +ResourceAddress = . | .. +ProviderAddress = provider.. + +graph/cluster/knowledge +schema/graph:knowledge/main +query/graph:knowledge/find_experts +policy/cluster/base_rbac +ui/cluster/dashboard.overview +pipeline/cluster/github_sync +alias/cluster/experts +embedding/cluster/default +``` + + + +Resource records carry: + +| Field | Meaning | +|---|---| +| `kind` | Graph, Schema, Query, PolicyBundle, UiSpec, Binding, Alias, EmbeddingConfig, Pipeline | +| `scope` | Cluster or graph id | +| `name` | Stable resource name inside scope | +| `fingerprint` | Content hash of the normalized spec and all referenced files | +| `dependencies` | Resource keys this resource references | +| `observed` | Applied graph manifest version, policy digest, query digest, schedule id, etc. | +| `status` | `Pending`, `Planned`, `Applying`, `Applied`, `Drifted`, `Blocked`, `Error` | +| `conditions` | Typed details such as `ActualAppliedStatePending`, `NeedsApproval`, `DependencyMissing`, `PartialPipelineRun` | + +The planner builds a dependency graph from these records and uses it for both +validation and blast-radius reporting. + +## Terraform-Style Validate / Plan / Apply + +The cluster workflow deliberately mirrors Terraform's safe sequence: + +```text +cluster validate # parse + schema-check desired config, no state mutation +cluster plan # diff desired config against state, with optional refresh +cluster apply # apply an accepted fresh plan and update state +cluster status # read state-backed deployed reality +cluster refresh # repair/import observations from actual cluster state +``` + +Implementation rollout follows the same safety posture: ship parser/validate +first, then read-only plan, then state backend and lock, then apply. + +The plan is a structured artifact, not just terminal text. It must include: + +| Plan field | Why it exists | +|---|---| +| `desired_revision` | Git commit / config digest being evaluated | +| `resource_digests` | Exact digest of every schema, query, policy, UI, pipeline, and map file | +| `dependencies` | Edges such as query -> graph/schema, dashboard -> query, pipeline -> source provider + graph | +| `state_observations` | Applied revision, resource fingerprints, graph manifest versions, status conditions, and drift | +| `changes` | Create/update/delete/replace/refresh-only operations | +| `blast_radius` | Downstream resources to revalidate or affected behavior to surface | +| `approvals_required` | Irreversible/data-loss or compatibility-narrowing gates | + +`cluster apply` must reject a stale plan when state, resource digests, or +observed graph versions no longer match the plan base. The operator or agent +must re-plan or explicitly refresh first. + +## Cluster Storage Layout + +Target Phase-1 cluster-root layout: + +```text +/ + __cluster/ + state.json + lock.json + status/ + .json + approvals/ + .json + recoveries/ + .json + recovery/ + .json + resources/ + query///.gq + policy//.yaml + ui//.dashboard.yaml + pipeline//.pipeline.yaml + graphs/ + .omni/ +``` + + +The exact filenames can change, but the shape cannot: + +- There is one cluster-control namespace under the cluster root. +- Graph data remains in ordinary OmniGraph graph roots. +- State is a locked/CAS-updated JSON document, not a Lance dataset. +- Status, approval, and recovery ledgers are append-only or per-resource JSON + records until table semantics are proven necessary. +- Resource payloads are content-addressed by digest so apply can be idempotent. +- Cluster state is not inferred from the operator's working tree. +- A Lance-backed control-plane store is a future backend option only if + row-level queryability/history or tighter publish fencing justifies it. + +## State Backend Protocol + +### Cluster-Hosted JSON State + +When `state.backend: cluster`, the baseline backend stores JSON documents under +`/__cluster/` and protects `state.json` with object-store lock/CAS. +It is cluster-hosted, but it is still a separate state write from graph Lance +manifest movement. + +Apply protocol: + +1. Acquire the cluster state lock. +2. Read current `state.json` and backend CAS token / object generation. +3. Validate plan base still matches state. +4. Write a cluster recovery sidecar before any graph manifest or non-idempotent + resource can move. +5. Write content-addressed resource payloads and perform any required graph + manifest movements. +6. CAS-update `state.json` with the new applied revision, resource + fingerprints, observed graph versions, status references, and approval / + recovery references. +7. If step 6 fails after actual resources moved, do not acknowledge success. + Surface `ActualAppliedStatePending` and require `refresh` / `import` repair. +8. Delete the sidecar and release the lock only after the state outcome is + recorded. + +### External State + + + +When `state.backend` points outside the cluster root, the same JSON state shape +lives in an external store. It is locked and CAS-updated, but it is not atomic +with Lance or OmniGraph manifests. + +Apply protocol: + +1. Acquire the external state lock. +2. Read state and CAS token. +3. Validate plan base still matches state. +4. Write a cluster recovery sidecar. +5. Perform the cluster resource changes. +6. CAS-update external state with the new applied revision, statuses, and the + observed graph manifest / resource versions it records. +7. If step 6 fails, do not acknowledge success. Surface + `ActualAppliedStatePending` and require `refresh` / `import` repair. +8. Release the external lock only after the state outcome is recorded. + +This mode can be strongly coordinated, but it must never be documented as one +atomic commit across both stores. + +### Future Lance-Backed State + +A Lance-backed state/status/approval/recovery store is deliberately not the +baseline. It becomes attractive only if JSON files become a real liability: +large status sets need structured filtering, approval/recovery history needs +table scans, or cluster apply needs a manifest publisher that can fence state +and graph-version pins together. Until then, Lance datasets add bootstrapping, +schema migration, and control-plane recovery surface without enough benefit. + +## Cluster Manifest Publisher + +The cluster publisher is a possible later layer above today's graph publisher. +It does not replace Lance or the per-graph `__manifest` table, and it is not +required for Phase-1 JSON state / read-only plan. + +Required semantics: + +| Requirement | Detail | +|---|---| +| Expected-version CAS | Every resource in an apply group supplies its expected current version/fingerprint | +| Resource changes | Register/update/tombstone resource payloads and graph version pins | +| Graph-head fencing | If a graph schema/lifecycle operation moves a graph manifest, the cluster manifest records the exact graph manifest version | +| Sidecar coverage | Any graph or cluster resource that can move before cluster publish must be recoverable all-or-nothing | +| Deterministic publish order | Sidecars and apply groups process in stable order | +| Loud partials | If a group cannot be rolled back or forward in-process, status records the condition before more apply work proceeds | + +The risky case is nested publish: + +```text +schema apply moves graph:knowledge manifest +cluster apply has not yet published query/policy/state records +process crashes +``` + +That is not safe unless the cluster sidecar records enough information to roll +the graph movement forward into the cluster manifest or roll it back using the +same recovery discipline as current graph recovery. + +## Plan Model + +Plan output is a durable, replay-checked proposal, not just pretty text: + +```text +Plan { + plan_id, + desired_revision, + base_state_revision, + base_state_cas, + changes[], + apply_groups[], + approvals_required[], + blast_radius, + diagnostics[] +} +``` + +Each change records: + +| Field | Meaning | +|---|---| +| `resource` | Stable `ResourceKey` | +| `operation` | Create, Update, Delete, Replace, RefreshOnly | +| `reversibility` | Reversible, Recoverable, CompatibilityNarrowing, IrreversibleDataLoss | +| `effect` | ConfigOnly, Catalog, GraphDefinition, GraphDataRewrite, DataPlaneSchedule | +| `downstream` | Resources that must be revalidated or will observe changed behavior | +| `approval` | None, HumanRequired, PolicyRequired, AlreadySatisfied | + +`apply` must re-read state and reject stale plans unless an explicit +`--refresh` / `--replan` path recomputes the plan. + +## Downstream Dependency Rules + +These are the concrete "what requires downstream" rules. + +| Changed resource | Must revalidate / recompute downstream | Blocking failures | +|---|---|---| +| Graph create/delete/rename | Policies, queries, aliases, dashboards, pipelines, bindings, server registry, state graph set | Dangling graph references; duplicate URI; invalid `GraphId`; graph delete without irreversible approval | +| Schema | Stored queries, pipeline maps, UI bindings/query outputs, embedding/index config, data-impact preview, policy predicates once row/type pushdown exists | Unsupported migration; query breakage; missing target type/property; hard drop without approval | +| Stored query | Aliases, UI bindings, policy list/invoke grants, MCP/tool catalog compatibility, typed params | Query file parse/type errors; registry key != `query `; removed query still referenced | +| Policy bundle | Query catalog visibility, graph/server action authorization, approval gates, bootstrap permissions | Invalid Cedar/YAML; server-scoped action in graph policy; per-query list/invoke gap unhandled | +| UI/dashboard | Query bindings, graph refs, output field expectations, policy visibility for referenced queries | Binding to missing graph/query/param/output | +| Alias | CLI command resolution, graph/query refs, shared-vs-personal boundary | Dangling graph/query; mutation alias pointing at read-only context | +| Embedding config | Schema `@embed` columns, model dimension, index rebuild/reconcile, env refs | Dimension mismatch; missing env ref; unsupported model/provider | +| Pipeline definition | Target graph schemas, mapping files, env refs, scheduler/runtime state, per-target run ledger | Missing target graph/type/property; overwrite mode without approval; source secret missing | +| Binding | Referenced source/surface pair, dependency order, visibility policy | Missing source or target; incompatible params | +| State backend config | Lock implementation, import/refresh protocol, apply acknowledgements | Backend missing CAS/lock; state CAS failure after graph/resource movement | + +## Blast Radius Matrix + +| Area | Required downstream change | Blast radius | Notes | +|---|---|---|---| +| Config parsing | Add strict `cluster.yaml` parser, path/env-ref resolver, resource fingerprints, no layered merge | High | Separate from `OmnigraphConfig`; existing config tests still need backcompat coverage | +| CLI | Add `cluster validate/plan/apply/status/refresh/import`, plan rendering, approval flags, actor threading | High | Must not change existing command selection or `omnigraph use` behavior | +| State backend | Add JSON state document, status/approval/recovery records, lock/CAS, and import/refresh repair | High | Must not silently succeed after state CAS failure | +| Optional cluster publisher | Add a cluster manifest plus table-backed state/status store only if stronger all-or-nothing apply is required | Very high | Touches core atomicity and recovery invariants | +| Recovery | Add cluster sidecars and failpoint coverage for graph-move-before-state-publish gaps | Very high | Any missed sidecar is a correctness bug | +| Graph lifecycle | First-class graph resource create/delete/rename or stable-id story | High | Current server add/remove is intentionally not exposed | +| Schema apply integration | Make schema apply cluster-aware or wrap it with cluster recovery | High | Existing schema apply cannot be treated as cluster atomic by assertion | +| Query registry | Remove target-state exposure flag, add dependency metadata, keep `mcp.expose` bridge | Medium/high | Catalog behavior is observable public API | +| Policy | Add cluster plan/apply/admin actions and per-query list/invoke scope | High | Needs docs, tests, Cedar schema migration, and compatibility with coarse `invoke_query` | +| Server registry | Boot from cluster state, eventually reload/reconcile graph handles, expose statuses | High | Affects routing, OpenAPI, auth, and workload admission | +| API types/OpenAPI | Plan/status/apply DTOs if HTTP management endpoints ship | Medium/high | OpenAPI drift must be regenerated | +| UI specs | New renderer/spec validator/binding checker | High | New product surface, not currently implemented | +| Pipelines | New scheduler/runtime/connector/mapping/idempotency/run ledger | Very high | Second data-plane seam; large product and correctness surface | +| Embeddings | Cluster-level defaults, env refs, model/dimension validation, index interaction | Medium | Existing embedding code is mostly offline/client-side | +| Docs | User docs for cluster config, policy, server, CLI; dev docs for invariants/testing | High | Public contract changes | +| Tests | New cluster suites plus extensions to config/server/policy/recovery/schema/query tests | High | Needs boundary-matched coverage | + +## Reversibility And Approval Tiers + +| Tier | Examples | Gate | +|---|---|---| +| Display-only | Dashboard layout, non-breaking alias addition | No approval beyond policy | +| Catalog behavior | Add query, hide/list query via policy, add policy grant | Policy check; no data-loss approval | +| Compatibility narrowing | Future validated enum narrowing, query param removal, policy removal that revokes access | Explicit compatibility warning; may require human approval by policy | +| Recoverable definition rewrite | Soft schema drop, graph schema rename, index rebuild | Plan warning; no data-loss approval unless policy requires | +| Irreversible data loss | Graph delete, hard schema drop, cleanup-triggered prior-version reclamation, overwriting pipeline target | Human approval artifact recorded in audit ledger | + +Future enum narrowing belongs in `CompatibilityNarrowing` unless the migration +also drops/coerces data or triggers cleanup. That distinction matters for plan +wording and for policy predicates. + +## Rollout Phases + + + +### Phase 0: Documentation And Parser Skeleton + +- Add cluster spec types and strict parser behind an unused feature/module. +- Implement `cluster validate --config ` with no state backend. +- Validate file paths, env refs, duplicate resource keys, and dependency graph. +- No behavior change to `omnigraph.yaml`, server boot, or query exposure. + +### Phase 1: Read-Only Planning + +- Add `cluster plan` against a mock/imported state snapshot. +- Produce plan JSON and human output. +- Reuse existing schema migration planner for schema resources. +- Validate stored queries against desired schema. +- Compute downstream dependencies and blast radius. +- Still no apply. + +### Phase 2: State Backend And Lock + +- Add `state.backend: cluster` JSON storage and lock/CAS. +- Add external backend trait only if lock + CAS semantics are explicit. +- Add `cluster status`, `refresh`, and `import`. +- Persist `AppliedRevision`, `ResourceStatus`, and audit references in JSON. + +### Phase 3: Config-Only Apply + +- Apply query, policy, UI, alias, embedding, and pipeline definition resources + that do not move graph manifests. +- Publish by writing content-addressed resource payloads and CAS-updating + `state.json`. +- Keep server boot from `omnigraph.yaml`; cluster state is inspectable but not + yet serving traffic. + +### Phase 4: Graph And Schema Apply + +- Add graph create/delete as cluster resources. +- Make schema apply cluster-aware, with sidecar coverage for graph manifest + movements before JSON state publish. +- Gate irreversible data-loss operations with approval artifacts. +- Consider a cluster manifest publisher only if the JSON sidecar + repair path + is not strong enough for the accepted safety contract. + +### Phase 5: Server Reads Cluster Catalog + +- Allow server startup from cluster state. +- Add status and catalog endpoints as needed. +- Keep the current `omnigraph.yaml` startup path as compatibility mode. +- Regenerate OpenAPI for any HTTP surface. + +### Phase 6: Policy-Owned Query Exposure + +- Add per-query policy scope for list/invoke. +- Filter `GET /queries` by actor. +- Keep coarse `invoke_query` as a broad allow rule for compatibility until + docs and migrations say it can be narrowed. +- Deprecate and later remove `mcp.expose` from target-state cluster config. + +### Phase 7: Pipeline Runtime + +- Add scheduler/worker/runtime. +- Add source connector contracts, mapping validation, idempotency keys, + per-target run status, and retry behavior. +- Treat fan-out execution as data-plane writes unless explicitly staged through + branch/merge. + +## Test Ownership + +Tests must prove the Terraform-style workflow, not just individual parsers. +The minimum behavior contract: + +```text +validate catches bad config +plan is deterministic and complete +apply only applies a fresh accepted plan +state changes are locked and durable +drift and partial convergence are visible, not silent +``` + +| Change | Existing coverage to extend | New coverage likely needed | +|---|---|---| +| Cluster parser | `omnigraph-config` inline config tests for strictness/path resolution | `omnigraph-cluster` parser/dependency tests | +| Plan dependency graph | Schema planner tests, query registry tests | Golden plan JSON for cross-resource downstream impacts | +| State lock/backend | Existing schema apply lock tests as model | JSON state CAS/lock race tests | +| Optional cluster manifest publisher | `crates/omnigraph/src/db/manifest/tests.rs` | Cluster publisher CAS, expected-version, deterministic order tests if that backend ships | +| Cluster recovery | `recovery.rs`, `failpoints.rs` | Phase B -> state publish failpoints, external state CAS failure tests | +| Schema cluster apply | `schema_apply.rs`, failpoints schema apply cases | Nested graph/cluster recovery tests | +| Query exposure policy | `omnigraph-policy` invoke_query tests, server query catalog tests | Per-query list/invoke allow/deny and no-probing tests | +| Server cluster boot | `omnigraph-server/tests/server.rs`, `openapi.rs` | Boot from cluster state, registry reload/status tests | +| CLI cluster commands | `omnigraph-cli/tests/cli.rs`, `system_local.rs` | `cluster validate/plan/apply/status` system tests | +| Pipelines | None today | New runtime/mapping/idempotency/run-ledger suites | + +Workflow-specific tests: + +| Workflow area | Required assertions | +|---|---| +| Parser / validate | Unknown fields, wrong-kind typed addresses, missing providers, inline secret values, dangling graph/query/pipeline refs, and future-phase fields fail with typed diagnostics | +| Plan goldens | Given config + imported/fake state, plan JSON contains stable resource digests, dependency edges, state observations, proposed changes, blast radius, and approval gates in deterministic order | +| Fresh-plan apply | Changing config digest, state revision, resource digest, or observed graph manifest version after planning makes `cluster apply` reject and require re-plan/refresh | +| State lock / CAS | Concurrent applies against the same backend cannot both succeed; loser gets a typed lock/CAS conflict | +| Recovery / partial apply | Fail after graph/resource movement but before cluster state publish; assert recovery or status surfaces `ActualAppliedStatePending`/sidecar state and never returns success | +| Server/runtime phase | Before cluster state drives routing or registry reload, tests are hermetic: no real home dir, no real global config, no real credentials, no ignored remote tests | +| Pipeline phase | Fan-out run records per-target status, commit ids, retryability, and idempotency keys; no aggregate success unless every target succeeded | + +Hard gates: + +- Do not ship `cluster apply` until `cluster validate` and read-only + `cluster plan` have hermetic tests. +- Do not ship graph/schema-moving apply until failpoint recovery tests prove the + Phase B -> state publish gap is covered. + +For docs-only changes, `scripts/check-agents-md.sh` is enough. For +implementation phases, run the boundary tests above before widening to +`cargo test --workspace --locked`. + +## User-Visible Documentation Fallout + +The following public docs must change when the corresponding phase ships: + +| Phase | User docs | +|---|---| +| Parser/validate | New `docs/user/cluster-config.md`; CLI reference for `cluster validate` | +| Plan/apply | CLI reference, transactions, policy, errors | +| State backend | Storage, deployment, constants, maintenance | +| Server cluster boot | Server, deployment, OpenAPI | +| Policy query exposure | Policy, server, query language / stored-query registry docs | +| Pipelines | New pipeline user guide, deployment, audit, errors | +| Embeddings config | Embeddings, indexes | + +Do not ship a user-visible command, flag, env var, endpoint, or config key +without updating the corresponding user doc in the same PR. + +## Known High-Risk Design Decisions + +1. **Cluster root identity.** Decide whether `metadata.name` is a label or + identity. Prefer root-derived stable identity plus display name to avoid a + rename breaking resource identity. +2. **Graph storage derivation.** The high-level sample omits graph storage. + Implementation should derive graph roots under `ClusterRoot/graphs/.omni` + by default and treat external graph roots as a separate, explicit feature. +3. **Nested apply.** Schema apply and graph lifecycle cannot move a graph + manifest outside cluster sidecar coverage. +4. **External state.** Must expose pending repair instead of returning success + when graph/resource movement succeeds and external state CAS fails. +5. **Per-query policy.** Catalog filtering must avoid probing leaks: callers + without list/invoke permission should not distinguish hidden from missing. +6. **Pipeline fan-out.** Do not promise atomic multi-graph ingestion unless the + runtime uses a real branch/merge or equivalent protocol for every target. +7. **Drift correction.** Reconciler-initiated deletes are the same data-loss + class as human-requested deletes. + +## Exit Criteria For A Real RFC + +Before implementation begins beyond parser/validate, the RFC must answer: + +1. Exact JSON state/status/approval/recovery schemas and object-store paths. +2. Exact sidecar JSON schema and recovery decision matrix. +3. State backend interface and supported lock/CAS implementations. +4. Cluster apply group syntax and dependency ordering rules. +5. Plan JSON schema, including blast-radius and approval fields. +6. Bootstrap authority and first-actor story. +7. Server startup and migration path from `omnigraph.yaml`. +8. Per-query policy schema and compatibility bridge for `mcp.expose`. +9. Pipeline runtime owner, status schema, and idempotency contract. diff --git a/docs/dev/cluster-config-specs.md b/docs/dev/cluster-config-specs.md new file mode 100644 index 0000000..8aa63cb --- /dev/null +++ b/docs/dev/cluster-config-specs.md @@ -0,0 +1,422 @@ +# Cluster Config Spec β€” Declarative, As-Code, Agent-Operated + +**Status:** Draft / thinking-in-progress +**Type:** Architecture direction +**Date:** 2026-06-07 +**Relationship:** generalizes today's `omnigraph.yaml` graph/query/policy configuration surface ([CLI reference](../user/cli-reference.md), [server docs](../user/server.md)) into a future cluster control plane. The distilled rules are in [cluster-axioms.md](cluster-axioms.md); detailed downstream implementation spec and blast-radius assessment in [cluster-config-implementation-spec.md](cluster-config-implementation-spec.md). This is a proposed architecture, not an implemented RFC. + +> **Implementation status.** The examples below describe the full target schema. +> Stage 2B only accepts the read-only subset documented in +> [cluster-config.md](../user/cluster-config.md). Future-phase fields such as +> `env_file`, `apply`, `providers`, `pipelines`, `embeddings`, `ui`, `aliases`, +> and `bindings` are intentionally rejected with typed diagnostics until their +> reconciler semantics are implemented. + +> **Revision 2026-06-07 β€” full commitment to the Terraform paradigm.** Three changes from the earlier draft: (1) **state is an authoritative, locked ledger in a backend** (server-hosted *or* a separate cloud store), not "a mostly-rebuildable projection"; (2) `plan` is framed as the **CLI diff between local config and state**; (3) **ETL pipelines** (external data sources) are a first-class config asset β€” a second seam, alongside schema, where a definition triggers a data-plane effect. The full set of config assets (incl. **aliases**, **embeddings**) is enumerated below. + +--- + +## The problem (the Sarah/Bob test) + +Two operators, Sarah and Bob, administer the same OmniGraph deployment. Sarah adds new queries, changes a schema, adds a dashboard, updates policies, and wires in a new data feed. + +**How does Bob find out?** + +Today he can't β€” not cleanly. Sarah's changes land in many different places via many different mechanisms: + +- schema β†’ the schema-apply path, accepted state in `_schema.pg`, `_schema.ir.json`, `__schema_state.json`, and table versions in the graph manifest +- queries β†’ `.gq` files passed per request or resolved through CLI query roots / aliases; not durable cluster state +- policies β†’ `policy.file` in `omnigraph.yaml`, pointing at Cedar/YAML files that are usually GitOps'd externally +- aliases β†’ CLI sugar in each operator's `omnigraph.yaml` +- external data β†’ ad-hoc `load`/`ingest` scripts, cron jobs, glue code that lives nowhere durable +- UI β†’ undefined + +There is no single diff that spans them, no single change record attributed to Sarah, no one place Bob (or Bob's agent) reads to answer "what is this deployment, and what changed?" The state is **fragmented**, and fragmentation is hostile to the one thing an agent must do: reason over the system *as a whole*. + +A design passes only if it answers the Sarah/Bob test directly. + +--- + +## Thesis + +The unit of declarative state is the **cluster** (the deployment), described by **a single config, as code, in version control**, operated by an **agent** through a plan/apply/reconcile loop against an authoritative state ledger. + +Every surface is a declarative as-code artifact β€” schema (`.pg`), queries (`.gq`), policies (`.yaml`), UI (`.yaml`), aliases, **ETL pipelines**, and embeddings config. The UI is not a separately-deployed application; it is a declarative spec, a first-class resource reconciled exactly like the others. + +Three pillars, none optional: + +1. **DECLARATIVE** β€” you describe the desired end state, not the steps. The reconciler computes the steps. +2. **AS CODE** β€” the config is declarative text in a repo, version-controlled. This is the **source of truth for *intent***. +3. **OPERATED BY AGENT** β€” an agent authors config changes and drives reconciliation as an authenticated actor, with policy and approval gates. No human state-management burden. + +This is **Terraform's model, taken literally**: config (as code) is desired truth; **state is an authoritative, locked ledger** of what has been applied β€” held in a backend (the cluster, or a separate cloud store); `plan` diffs config against state; `apply` converges reality to config and updates state β€” applied at **cluster** scope, with OmniGraph as its own data-aware provider and an agent as the controller. + +--- + +## Why as-code (the recursion argument) + +"As code" is not branding. It is the structural property that makes a self-describing system well-founded. + +Consider the rejected alternative: model the cluster's definition *as a graph* (a meta-graph whose nodes are graphs/policies/queries/UI). To describe a graph you need a schema. The meta-graph's schema is either: + +- **hardcoded** β†’ the base case is *code* (you smuggled code in at the bottom anyway), or +- **another graph** β†’ infinite regress, no base case. + +Graph-describing-graph never terminates. **Code is the base case.** A declarative config needs no meta-describer because it is parsed by the engine's compiled code β€” not described by more user-space data. + +> **Declarative-as-code terminates. Declarative-as-data (a graph of graphs) recurses.** + +This is also why **config** must live **outside** the running system: reviewable (PRs), reproducible (clone + apply), diffable as text, and editable by an agent β€” without depending on the running system to describe its own intent. + +Corollary on direction: change flows **code β†’ cluster, never the reverse.** You do not edit the running system and call that intent. (State, separately, *records* what the cluster currently is β€” see the next section β€” but it is never where you express what it *should* be.) + +--- + +## Why per-cluster, not per-graph + +The definition Sarah changed does not *belong* to any single graph: + +1. **Policies cross-cut graphs.** "Member can't delete on any graph," "who may list/create/delete graphs" β€” cluster facts. No graph could own them. +2. **"Which graphs exist" has no home in a per-graph model.** The set of graphs is state *above* any graph. +3. **Queries, UI, pipelines, and aliases span graphs.** The MCP/tool catalog an agent discovers is the *cluster's* surface; a dashboard renders multiple graphs; a pipeline may fan out into several. +4. **Cross-graph apply groups.** Sarah may add a graph *and* wire it into the UI *and* grant policy access *and* attach a feed as one logical change β€” only the cluster can express, plan, and eventually fence that as one apply group. +5. **Operators operate clusters.** Bob is Sarah's peer on a *deployment*, not a graph. The collaboration unit is the cluster. + +The graph is a *resource within* the cluster, not the unit of operation. + +The mirror question β€” *why not per-fleet?* β€” is the same one this section used against per-graph, one level up. A fleet of clusters may eventually want its own declarative spec describing which clusters exist. That recursion is real but **out of scope here**: this proposal stops at the cluster because the cluster is the unit two operators collaborate over. Fleet is the next scope up, named and deferred, not denied. + +--- + +## The model: config / state / reconcile (the Terraform model, literally) + +| Layer | What it is | Source of truth for… | Who manages it | +|---|---|---|---| +| **Config** (as code, a folder of files) | Desired state of the whole cluster β€” graphs, schemas, policies, queries, UI, bindings, aliases, embeddings, ETL pipelines | **Intent** ("what it should be") | Operators/agents, in version control | +| **State** (a locked ledger in a backend) | The authoritative record of what has been applied β€” applied revision, per-resource fingerprints, observed graph/table versions, audit-record references, resource conditions | **Deployed reality** ("what is") | The reconciler; humans don't hand-edit it | +| **Actual cluster** | The realized *definition* of the running graphs β€” schema/policies/queries/UI/pipelines as actually in force | β€” (reality itself) | The engine; `apply` converges it to config | + +**`plan`** = `diff(config, state)` β†’ proposed change set (optionally refreshed against the actual cluster). +**`apply`** = acquire the state lock β†’ converge actual β†’ config β†’ **update state** β†’ release lock. Apply does **not** acknowledge success until the state update succeeds; if actual moved but the state write failed, the next `plan` / `refresh` must surface the non-success state and repair or import it before more work proceeds. + +### State is an authoritative, locked ledger β€” not a throwaway projection + +This is the 2026-06-07 revision. State is treated exactly as Terraform treats `tfstate`: + +- **Authoritative.** State is the trusted record of what is deployed. `plan` diffs config against **state** (fast, deterministic), not against a full live scan of the cluster on every command. "What exists" is answered from state. +- **In a backend.** State lives in a configurable backend: the **cluster's own object-store backend**, or a **separate cloud store** (e.g. a different bucket/account) β€” the operator's choice, mirroring Terraform's local/S3/remote backends. The config declares which. +- **JSON first.** The baseline state format is Terraform-style JSON documents (`state.json` plus status/approval/recovery JSON records) protected by backend lock/CAS. Lance control-plane datasets are a possible later backend only if row-level history, queryability, or tighter publish fencing justifies the added machinery. +- **Atomicity depends on backend and publish scope.** A JSON state backend, even when stored under the cluster root, is a separate CAS step from graph Lance manifest moves. If actual resources move but the state write fails, apply must surface `ActualAppliedStatePending` (or equivalent) and require refresh/import repair instead of pretending one atomic commit covered every object. A future Lance-backed state backend or cluster manifest publisher may tighten this, but that is not the Phase-1 assumption. +- **Locked.** `plan`/`apply` acquire a **state lock** before touching state, so two operators (or two agents) cannot converge concurrently and corrupt the ledger. This generalizes the existing `__schema_apply_lock__` from schema scope to cluster scope. +- **Reconstructable, but not casually rebuilt.** OmniGraph's edge over opaque-cloud Terraform: the running cluster is self-describing (manifests, commit logs), so a lost state ledger can be **imported / refreshed** from the live cluster. That is a *resilience* property β€” not licence to treat state as disposable. State is protected and backed up like any source of truth. +- **One slice is never reconstructable.** Who *approved* an irreversible apply cannot be re-derived from a manifest scan. That approval/audit record lives in the **durable audit ledger** (baseline: append-only JSON records in the state backend; future: a Lance table only if needed). State *references* it by id; it never *is* it. + +**The control plane reconciles definition, not data.** The reconcile loop converges the cluster's *definition* β€” schema, policies, queries, UI, bindings, aliases, pipelines, and the set of graphs. It does **not** converge **data**: rows, edges, and vectors are data-plane content, mutated by `load`/`mutate` and by **pipeline execution**, versioned by the commit DAG, and they sit entirely outside the reconcile loop. (`load`/`mutate` never appear in `cluster.yaml`.) **Two** definition kinds *trigger* a data-plane effect without owning data β€” schema and ETL pipelines (see "ETL pipelines" below). + +### Cluster resource model + +Minimum vocabulary: + +- **ClusterRoot** β€” the object-store prefix / control namespace for one deployment. +- **DesiredRevision** β€” git commit, `cluster.yaml` digest, and per-resource digests. +- **ResourceKind** β€” `Graph`, `Schema`, `Query`, `PolicyBundle`, `UiSpec`, `Binding`, `Alias`, `EmbeddingConfig`, **`Pipeline`** (ETL), and future cluster-scoped resources. +- **ResourceAddress** β€” normalized typed references between resources, such as `graph.knowledge`, `query.knowledge.find_experts`, `policy.base_rbac`, and `pipeline.github_sync`; illustrative YAML may use shorthand, but plan/state store the typed form. +- **ProviderAddress** β€” typed references to provider instances, such as `provider.storage.prod_graphs`, `provider.source.github_org`, and `provider.embedding.default`; provider addresses keep storage, external sources, and embedding providers from being inferred from ambiguous strings. +- **StateBackend** β€” where the JSON state ledger is stored: `cluster` (this deployment's own backend) or an external store (a separate bucket/account). +- **StateLock** β€” the cluster-scope lock acquired before plan/apply. +- **AppliedRevision** β€” the durable, locked record (the heart of state) of which desired revision is applied, with audit-record references, resource fingerprints, and graph/table version observations. +- **ResourceStatus** β€” `Pending | Planned | Applying | Applied | Drifted | Blocked | Error`, with typed conditions and observed actual state. +- **ApplyGroup** β€” the explicit atomicity unit. Default is one independent resource per group; cross-resource references force planner-derived groups, and user-declared groups may opt into larger atomicity only for resources the active backend protocol can fence or repair. Baseline JSON state supports small, explicit groups; larger all-or-nothing groups require a future cluster publisher or equivalent proof. + +--- + +## State: backend, lock, and the config ↔ state diff + +The CLI is the operator's window onto the gap between config and state. + +The Terraform-aligned workflow is: + +```text +cluster validate # parse + schema-check desired config, no state mutation +cluster plan # diff desired config against state, with optional refresh +cluster apply # apply an accepted fresh plan and update state +cluster status # read what state says is deployed now +cluster refresh # update/import state observations from actual cluster state +``` + +`plan` is the central artifact. It records the desired revision, resource +digests for every referenced file, dependency edges between resources, observed +state fingerprints / graph manifest versions, proposed changes, and approval +gates. The human output below is a rendering of that structured plan, not the +only representation. + +``` + $ omnigraph cluster plan + config ./ β†’ diff against state (backend: cluster Β· lock: acquired) + + ~ schema knowledge hard-drop Person.legacy_id ⚠ prior versions reclaimed β€” needs approval + + query knowledge.find_experts (new stored query) + - query knowledge.orphan_pages (removed) + ~ policy base_rbac grant invoke find_experts β†’ members (this is what EXPOSES the new query) + + pipeline saas_sync notion β†’ knowledge, hourly + ~ ui dashboards.overview add panel "experts" + + alias experts + ───────────────────────────────────────────────────────────────────── + 6 changes Β· 1 requires approval (hard schema drop on knowledge) Β· run `apply` to converge +``` + + +That output **is** the answer to the Sarah/Bob test: one diff, spanning every surface, attributed to a git commit and concrete resource digests, with data-impact peeked (axiom-6 schema seam), dependency fallout visible, observed state compared, and approval gates surfaced *before* anything moves. Drift (someone poked the live cluster out-of-band) shows up here too β€” `plan` reconciles state against the actual cluster and flags resources whose observed version no longer matches the ledger. + + +`apply` then: acquire **state lock** β†’ execute the change set (ordered/grouped per the planner) β†’ **CAS-update the JSON state ledger** with the new applied revision/status observations β†’ release the lock. For config-only resources, content-addressed payload writes can happen before the state CAS because state is the publish point. For graph/schema moves, the graph manifest may move before the state CAS; a crash or CAS failure there leaves a loud repair/import condition and no success acknowledgement, not a silently successful atomic apply. A future cluster manifest publisher can tighten this gap, but the baseline protocol does not assume it. + +--- + +## ETL pipelines (the second data-plane seam) + +External data β€” from another database, an API, a file drop, a stream β€” is a first-class config asset, not glue code that lives nowhere. + +A **Pipeline** is declared in config: a **source** (e.g. `notion`, `github`, `slack`, `gdrive`, `postgres`, `http`, `s3-files`, `kafka`), an optional **schedule/trigger**, and **one or more target graphs**, each with its own **mapping/transform** (external records β†’ graph types & properties). A single feed can **fan out across graphs** β€” e.g. a GitHub sync that populates both the `engineering` graph and the people/teams in `knowledge`. It is reconciled like any resource β€” `apply` creates / updates / deletes / (re)schedules the pipeline *definition*. This is the canonical "company brain" move: the deployment's graphs are continuously assembled from the SaaS tools the org already uses. + +The crucial boundary (axiom 6, axiom 13): the pipeline **definition** is control-plane and reconciled; the pipeline's **execution** β€” actually pulling rows and writing them β€” is a **data-plane effect** that produces ordinary `load`/`mutate` commits *outside* the reconcile loop. The reconciler converges the pipeline; the rows it ingests are never reconciled state (just as a cron *definition* is config but its output is not). This makes ETL the **second seam** where a definition triggers a data-plane effect β€” schema being the first (a migration conforms existing rows; ETL ingests new ones). + +Consequences that fall out of the existing model: + +- **`plan` previews the pipeline, not the data.** "pipeline `saas_sync`: notion β†’ `knowledge`, hourly" is a definition diff; it does not scan the source (data-volume-independent), the same way schema `plan` previews impact only at the bounded, opt-in data peek. +- **Source credentials come from the `.env` file** (axiom 10): `token: ${NOTION_TOKEN}` β€” resolved from the gitignored `.env` file per deployment, never inline. +- **Reversibility gradient applies** (axiom 8): a pipeline that *appends* is reversible-ish; one configured to *overwrite* a target is a data-loss path and hits the irreversible-op gate. +- **Referential integrity is plan-time** (axiom 9): a pipeline whose `into:` names a graph/type the same revision removes is a fail-closed `plan` error. +- **Fan-out is statusful, not magically atomic.** A pipeline execution that writes to several graphs is a set of ordinary per-target graph writes unless the pipeline explicitly stages through a branch/merge protocol that can fence those targets. A failed run may therefore leave `engineering=Applied`, `knowledge=Error` (for example), and the pipeline run ledger must expose per-target status, commit ids, retryability, and idempotency keys. Control-plane `apply` only converges the definition/schedule; it never means every future data-plane target has ingested successfully. + +--- + +## Config assets β€” the full set + +Everything below is **shared cluster config** (in the folder, version-controlled, secret-free) unless marked per-operator. The rule of thumb: if two operators must agree on it, it's config; if it's how *you personally* reach or view the cluster, it's per-operator. + +| Asset | In config? | Notes | +|---|---|---| +| **Graphs** (the set that exists) | βœ… config | the named graphs; their existence is cluster state | +| **Schema** (`.pg`, **one per graph**) | βœ… config | also encodes indexes (`@index`/`@unique`/vector), constraints, and search (`@embed`) β€” so indexes & search are reconciled *via* schema | +| **Stored queries** (`.gq`, **per graph**) | βœ… config | a `.gq` file declares **many** named queries; the registry declares which exist (name β†’ file, key must match the `query ` symbol). **Target design:** exposure β€” who may list/invoke each β€” is a policy decision, not a registry flag. **Current compatibility bridge:** shipped `omnigraph.yaml` still has `queries..mcp.expose`, and the HTTP catalog is not Cedar-filtered per query yet. Aliases & bindings reference a query by name | +| **Policy bundles** (`.yaml`) | βœ… config | YAML (not Cedar files); **shared across graphs** via `applies_to: [cluster \| ]` (many-to-many; fix 2026-06-08 unified the old `scope:`/`graphs:` split). Gates actions **and query exposure** (who may list/invoke each stored query) | +| **UI specs / dashboards** (`.yaml`) | βœ… config | first-class resources; a dashboard **reads from several graphs** (`graphs: [...]`) | +| **Bindings** | βœ… config | wiring between resources (query ⇄ UI surface) | +| **Aliases** | βœ… config* | CLI shortcut to a stored query: `{ command, query: <.gq file>, name: , args, format }` β€” `query` is the **file**, `name` the **query symbol** in it. See note | +| **Embeddings config** | βœ… config | model + dimension + which fields embed; the **API key comes from the `.env` file** (`${…}`) | +| **ETL pipelines** | βœ… config | source β†’ transform β†’ **one or more target graphs**; source credentials come from the `.env` file | +| **Apply settings** | βœ… config | `apply.default_grain`, grouping/ordering hints | +| **State backend + lock** | βœ… config | where the ledger lives, whether to lock | +| **Secrets (`.env` file)** | βœ… ref'd by config; values **gitignored** | a separate `.env` of secret values, referenced as `${NAME}`; never committed (OmniGraph's standard env-file convention) | +| **Connection** (which cluster URI) | ❌ per-operator | how *you* reach the cluster | +| **Operator token** | ❌ per-operator (secret) | each operator's own credential to reach the cluster | +| **CLI prefs** (output format, table layout, active graph/branch selection) | ❌ per-operator | personal ergonomics, not shared truth | + +\* **Aliases β€” the one with a split.** A shared alias that names a cluster resource (a stored query, a dashboard) is config β€” it's a vocabulary the whole team relies on, and it belongs in the spec (often it *is* just the stored-query catalog entry, since that already carries name + params + tool metadata). A *purely personal* shortcut (your own command abbreviations) stays in the per-operator layer. When in doubt: if it should survive `git clone` and be the same for Bob as for Sarah, it's config. + +--- + +## The synthesis (beyond vanilla Terraform) + +Embracing Terraform does not mean stopping at Terraform. Three extensions make this specifically right for OmniGraph and the agentic future: + +1. **OmniGraph is its own data-aware provider, and `plan` can peek across the data boundary.** A Terraform provider CRUDs resources blind to your data. Here, the control-plane resource is the schema **definition** (declarative, reconciled); converging it *triggers* a data-plane **effect** β€” currently soft/hard drops, rewrites, and index creation, with future validated migrations such as enum narrowing or `String`β†’`enum` conversion once the planner grows that tier. The leverage is that `plan`, before applying the definition change, can *peek* at bounded data-plane consequence and report it β€” **"hard-dropping this property requires approval and will make prior versions unreachable after cleanup"** or, in the future, **"narrowing this enum will fail on 37 rows"** β€” which Terraform structurally cannot do. This is deliberate and bounded: a data peek makes that `plan` cost scale with data volume, so it is **opt-in / bounded** (sampled or skippable for large tables), and it never makes the control plane the owner of data. Schema and ETL pipelines are the **two** seams where the control plane reaches into the data plane; everywhere else `plan` is data-volume-independent. + +2. **JSON state first, explicit partials, optional stronger fencing later.** Terraform apply is not transactional β€” partial applies are a real failure mode. Lance commits are per dataset, and today's OmniGraph manifest atomicity is graph-scoped: one graph commit flips the relevant sub-table versions together, protected by expected table versions and recovery sidecars. The first cluster-control backend should match Terraform's shape: a locked JSON state document plus append-only JSON status/approval/recovery records. That keeps Phase 1 inspectable and narrow. Cluster-level all-or-nothing apply is a later capability only if we add a **cluster manifest publisher** or Lance-backed state backend that fences graph *version pins*, query catalogs, policy bundles, UI specs, pipeline definitions, recovery sidecars, and state as one commit protocol. Until that exists, apply must surface partial convergence as `ResourceStatus`, not pretend it was atomic. + +3. **Agent-as-controller fuses Terraform with Kubernetes.** Terraform contributes the as-code config (truth outside the system, recursion-terminating) and the locked state ledger. Kubernetes contributes *continuous* reconciliation (controllers watch, not apply-on-demand). The agent is both author and controller: it reads a config change, runs the data-aware plan, evaluates blast radius against the reversibility gradient, **auto-applies the reversible parts only when policy permits, and escalates irreversible / data-loss gates to a human approval artifact recorded in the audit ledger and referenced by state.** + +> Terraform's as-code config + locked state Γ— Kubernetes' continuous reconciliation Γ— the agent as the controller that bridges them β€” on OmniGraph's data-aware, atomic substrate. + +--- + +## Concrete shape (illustrative) + +The config is **a set of files in one folder** (flat, Terraform-style β€” the extension carries the type): + +``` + company-brain/ + β”œβ”€β”€ cluster.yaml # the spec (graphs, policies, ui, bindings, aliases, pipelines, state, vars ref) + β”œβ”€β”€ .env # SECRET VALUES β€” gitignored, never committed + β”œβ”€β”€ knowledge.pg Β· engineering.pg # schemas (one per graph) (.pg) + β”œβ”€β”€ knowledge.gq Β· engineering.gq # query files β€” each holds MANY queries (.gq) + β”œβ”€β”€ cluster_admin.policy.yaml Β· base_rbac.policy.yaml Β· knowledge_pii.policy.yaml # shared policy bundles + β”œβ”€β”€ overview.dashboard.yaml # cross-graph UI spec (.dashboard.yaml) + └── notion_to_knowledge.map.yaml Β· github_to_engineering.map.yaml Β· github_to_people.map.yaml # pipeline maps +``` + +Secrets live in a gitignored `.env` file (OmniGraph's standard env-file convention); the config references them as `${NAME}`: + +```bash +# .env β€” secret values; gitignored; never committed. Referenced in cluster.yaml as ${NAME}. +NOTION_TOKEN=… +GITHUB_TOKEN=… +EMBEDDING_API_KEY=… +``` + +Resource relationships (so the wiring is unambiguous): + +``` + cluster ──has many──► graph ──has one──► schema + └────has──► query file(s) (.gq) ──each declares MANY──► query { … } symbols + registry entry key = the query symbol ──points to──► its .gq file (queries: { : { file } }) + (registry says a query EXISTS; it carries NO expose flag) + policy bundle ──applies to──► { cluster | one or MANY graphs } (SHARED, many-to-many) + └──governs query EXPOSURE──► who may LIST / INVOKE each stored query (no `expose:` in the registry) + alias (command, query = .gq FILE, name = symbol, args, format) ──selects one query from that file + binding names a query by registry name (graph.queryName) ──► resolved to (file, symbol) + dashboard ──reads from──► one or MANY graphs + pipeline ──writes into──► one or MANY graphs + secrets ──live in──► a separate gitignored `.env` file; config uses ${NAME} +``` + +```yaml +# cluster.yaml β€” desired state of the whole deployment (config = source of truth for INTENT) +version: 1 +metadata: + name: company-brain + +state: # the authoritative ledger's backend (Terraform-style) + backend: cluster # "cluster" = this deployment's own store; or s3://… (a separate store) + lock: true # acquire a state lock before plan/apply + +env_file: ./.env # secret VALUES live in a gitignored .env file; referenced below as ${NAME} + +apply: + default_grain: resource # references may force groups; explicit groups request more atomicity + +graphs: # the cluster's graphs β€” each is ONE schema + a set of named queries + knowledge: # people Β· teams Β· docs Β· decisions Β· projects + schema: ./knowledge.pg # desired schema; reconciler runs (and plan previews) the migration + queries: # the graph's stored (named) queries; KEY must match a `query ` in the file + find_experts: { file: ./knowledge.gq } # ─┐ `query find_experts` and `query related_docs` + related_docs: { file: ./knowledge.gq } # β”€β”˜ both live in knowledge.gq. Who may LIST/INVOKE β†’ policy (not here) + engineering: # repos Β· services Β· incidents Β· PRs + schema: ./engineering.pg + queries: + service_owners: { file: ./engineering.gq } + open_incidents: { file: ./engineering.gq } + +policies: # policy BUNDLES (YAML) β€” SHARED across graphs (many-to-many). + # Policy ALSO governs query EXPOSURE: who may list/invoke each stored query. + # Fix (2026-06-08): unified the binding field on `applies_to:` (was a + # `scope:` + `graphs:` split) β€” one field, takes `cluster` or graph refs; + # bare graph names are shorthand for `graph.` (see impl-spec typed addresses). + cluster_admin: # cluster-scoped: graph_list, create/delete, management + file: ./cluster_admin.policy.yaml + applies_to: [cluster] + base_rbac: # read/write + which roles may invoke which queries, across both graphs + file: ./base_rbac.policy.yaml + applies_to: [knowledge, engineering] + knowledge_pii: # an extra bundle, only for knowledge + file: ./knowledge_pii.policy.yaml + applies_to: [knowledge] + +pipelines: # ETL β€” ONE pipeline may write into SEVERAL graphs (definition only) + saas_sync: # the "company brain" move: assemble graphs from the SaaS tools + source: { kind: notion, token: ${NOTION_TOKEN} } # secret via ${NAME}, never inline + schedule: "0 * * * *" # hourly; execution is a data-plane effect, not reconciled state + into: # fans out across graphs + - { graph: knowledge, map: ./notion_to_knowledge.map.yaml } + github_sync: + source: { kind: github, token: ${GITHUB_TOKEN} } + schedule: "*/15 * * * *" + into: + - { graph: engineering, map: ./github_to_engineering.map.yaml } + - { graph: knowledge, map: ./github_to_people.map.yaml } # same feed enriches a SECOND graph + +embeddings: # semantic search over docs/decisions; key via the `.env` file + model: gemini-embedding-2 + dimension: 3072 + api_key: ${EMBEDDING_API_KEY} + +ui: # dashboards read from SEVERAL graphs + dashboards: + overview: + file: ./overview.dashboard.yaml + graphs: [knowledge, engineering] # cross-graph + +aliases: # CLI shortcuts. ⚠ an alias's `query:` is the .gq FILE PATH; + # `name:` selects the query SYMBOL inside it (a file declares many). + experts: { command: query, graph: knowledge, query: ./knowledge.gq, name: find_experts, args: [topic], format: table } + incidents: { command: query, graph: engineering, query: ./engineering.gq, name: open_incidents, format: table } + +bindings: # wiring between resources + - query: knowledge.find_experts + surface: ui.dashboards.overview +``` + + +What this is *not*: it is **not** a graph, and it carries **no credentials** β€” only secret *references* (`${…}`). It is parsed by the engine (the base case), describes the desired cluster, and is the thing two operators diff and review. + +The **state ledger** lives in the configured backend (the cluster, or a separate cloud store), versioned, CAS-updated, schema-versioned, locked during apply, agent-managed β€” the authoritative record of what is deployed. The baseline backend is JSON, so even cluster-hosted state is published through a state CAS and repaired explicitly if graph/resource movement happened first. A future cluster publisher can tighten that boundary, but it is not assumed by the high-level spec. + +--- + +## Boundaries that hold (orthogonal correctness, not Terraform-bias) + +1. **Secrets live in a `.env` file, never inline in config.** The committed config is what the cluster *is* (shared, reviewable, as code) and carries **no secret values** β€” only `${NAME}` references. The values (embedding API key, pipeline source credentials, per-deployment settings) live in a separate **`.env` file** β€” which is **gitignored and never committed**, and supplied per deployment. Separately, an *operator's own token* (how they personally reach the cluster) belongs to the per-operator connection layer, not the cluster config or its `.env` file. + +2. **The reversibility gradient gates apply β€” including drift correction.** Dropping a graph, hard-dropping schema data, or an overwriting pipeline is irreversible data loss; a future validated enum narrowing is a compatibility-narrowing migration unless it also drops or coerces stored values; recoloring a dashboard is not. Unified config, unified plan β€” but **tiered gates inside apply**, keyed to physics, not to who operates it. The gate applies to **drift correction too**: converging actualβ†’config can mean *dropping* something added out-of-band β€” a data-loss path that hits the same gate. A reconciler "just fixing drift" is never an exception. + +3. **Agents are actors, not ambient authority.** The reconciler runs with a resolved actor or service account, subject to Cedar policy. If it applies on behalf of a human, the durable audit ledger carries both the controller actor and the approving human / approval artifact, and state references that ledger entry. Client-supplied actor identity is never trusted. + +4. **Status is explicit when apply is not atomic.** A unified plan does not imply a unified commit. If an apply group partially converges, the cluster must expose `ResourceStatus` and typed conditions until reconciliation finishes or rolls back. Silent partial success is forbidden. + +5. **State integrity is protected.** State is locked during apply and stored durably in its backend. The baseline state backend is JSON plus lock/CAS, so state update failures surface a repair/import condition before success is acknowledged. A lost ledger is recoverable (import/refresh from the self-describing cluster), but state is never treated as disposable. + +--- + +## Relationship to current config + +This is not green field, but it is also not today's `omnigraph.yaml`. The current file is a shared convenience for CLI and server startup: named graph targets, server defaults, query roots, aliases, embeddings model, auth env-file lookup, and `policy.file`. It is **not** the cluster's source of truth, it has no separate state ledger, and parts of it are intentionally per-operator. + +This proposal: + +- **splits** per-operator connection/credential/preference config from shared cluster config, +- **adds** `cluster.yaml` + a flat config folder as the full declarative cluster config (graphs, schemas, query catalog, policy bundles, UI specs, bindings, **aliases**, **embeddings**, **ETL pipelines**), +- **adds** the **JSON state ledger** (authoritative, locked, in a backend) and the `cluster plan`/`apply` loop, +- **adds** the reconciler (with OmniGraph as its own data-aware provider), while treating a cluster manifest publisher as a later option rather than the baseline, +- **lets an agent drive** plan/apply/continuous-reconcile. + +The connection/credential/preference layer remains per operator: it points at a cluster, resolves that operator's identity, and holds personal ergonomics. The cluster config stays shared, secret-free, and reviewable; the state ledger stays authoritative and locked. + +Implementation gate: the Terraform-style workflow must be testable in order. +`cluster validate` must catch bad config before any apply path exists; +read-only `cluster plan` must have deterministic structured-plan tests before +state mutation ships; and graph/schema-moving apply must have recovery tests for +the gap between graph/resource movement and JSON state publish. Otherwise the +control plane can look declarative while still hiding drift or partial success. + +--- + +## Open questions + +1. **Cluster state layout.** What exact JSON documents / object-store paths hold `AppliedRevision`, `ResourceStatus`, approval records, recovery records, sidecars, and resource content for query/policy/UI/pipeline specs? What evidence would justify a future Lance-backed state backend? +2. **State backend options.** Beyond "cluster" and "a separate bucket," what backends are first-class (a different account, a remote control service)? How is the backend itself bootstrapped and its lock implemented (object-store CAS vs an external lock service)? +3. **State import / refresh.** The exact actual-state scan that reconstructs a conservative `AppliedRevision` when the ledger is lost, and which fields become `Unknown`. +4. **Apply grain syntax.** Apply defaults to per-resource `ApplyGroup`; cross-resource references force planner-derived groups; user-declared groups opt into more atomicity. What's the YAML, and which combinations can the publisher actually fence? +5. **Pipeline runtime.** Where do pipelines *execute* (in the server? a worker? an external scheduler?), how are runs observed in `ResourceStatus`, and how does a failed/partial run reconcile vs. retry? +6. **Continuous reconciliation trigger.** Watch-and-converge (k8s-style) vs. apply-on-config-change. The agent-as-controller model leans toward continuous. +7. **Tenant partitioning (cloud).** A cluster may host multiple tenants; config/state is then tenant-partitioned, consistent with the reserved `GraphKey { tenant_id, graph_id }`. Tenant resolved from the token, never the config. +8. **Bootstrap β€” config, state, *and* authority.** How a cluster comes into existence from an initial config (`init` seeds; cluster owns; git mirrors for CI/DR), the first state write, and the chicken-and-egg of the very first apply (which needs an actor before any cluster exists to resolve policy against β€” so the bootstrap actor is necessarily out-of-band and privileged). Security-sensitive; needs an explicit story. +9. **Alias scoping.** Where exactly the shared/personal alias line falls, and whether shared aliases are just stored-query catalog entries. +10. **UI render and safety model.** Generic engine-side renderer vs. thin client, allowed components, query-binding validation, policy propagation, sandboxing, version compatibility. +11. **Cluster identity vs. `metadata.name`.** Is `metadata.name` a label or stable identity? If identity, renaming loses it β€” the stable-ID-across-rename gap already in `invariants.md`. Decide whether identity keys on `name` or on `ClusterRoot`, and reuse the existing known-gap framing. +12. **Resource dependency ordering.** Explicit dependency DAG (Terraform) vs. eventual convergence with retries (k8s). The most consequential unmade fork: it decides whether `plan` can promise an apply *order* before any data moves. +13. **Query exposure in policy (supersedes `mcp.expose`).** *Today* the stored-query registry carries a per-query `mcp.expose` flag and invocation is gated with the coarse `invoke_query` Cedar action β€” with **per-query authorization a documented gap** (the catalog isn't Cedar-filtered per query yet). This design **folds exposure fully into policy and drops the flag**: a stored query's visibility (catalog membership) and invocability are both policy decisions, so the catalog `GET /queries` returns each actor's policy-permitted set. The open work is the exact policy predicates for *list* vs *invoke* per query, and retiring `mcp.expose`. + +--- + +## Prior art + +- **Terraform** β€” declarative infra *as code*; config is desired truth, **state is an authoritative ledger in a backend**, **state locking** serializes applies, `plan` diffs config↔state, providers do the CRUD. The core model adopted here, taken literally. +- **Kubernetes** β€” one cluster store, many resource types under one API; controllers reconcile continuously; cluster-level RBAC. The continuous-reconciliation half of the synthesis. +- **dbt / Airflow / Dagster** β€” declarative, as-code data pipelines with lineage. Prior art for the **ETL-pipeline-as-config** asset (the second data-plane seam). +- **OmniGraph's own schema-apply** β€” already a faithful plan/apply/state/drift loop for the `schema` resource type, with `__schema_apply_lock__` as the lock seed; the reconciler this generalizes. diff --git a/docs/dev/codeowners.md b/docs/dev/codeowners.md index 9a7fb50..50c4dc7 100644 --- a/docs/dev/codeowners.md +++ b/docs/dev/codeowners.md @@ -4,24 +4,45 @@ This setup gives every role change a reviewable PR and a permanent in-repository audit trail (`git log .github/codeowners-roles.yml`). -## Current roles +## Who owns what -| Role | Members | Scope | +The tables below are **generated** from `.github/codeowners-roles.yml` by `.github/scripts/render-codeowners.py` (the same render that produces `.github/CODEOWNERS`). They are the always-current "who owns what at this commit" view β€” don't edit them by hand; edit the yml and re-render. + + + +**Path β†’ owners** (GitHub applies *last match wins*; the `*` catch-all is listed first and is overridden by the specific patterns below it): + +| Path | Owners | Role(s) | |---|---|---| -| `engineering` | `@ragnorc` | All code under `crates/**`, repository infrastructure, default for unmapped paths | -| `docs` | `@ragnorc` | `docs/**`, README.md, AGENTS.md, CLAUDE.md, SECURITY.md | +| `*` | @ragnorc @aaltshuler | engineering | +| `crates/**` | @ragnorc @aaltshuler | engineering | +| `docs/**` | @ragnorc | docs | +| `README.md` | @ragnorc | docs | +| `AGENTS.md` | @ragnorc | docs | +| `CLAUDE.md` | @ragnorc | docs | +| `SECURITY.md` | @ragnorc | docs | -GitHub treats multiple owners in a CODEOWNERS line as **"any one of them satisfies the review requirement"**. To require N distinct approvers on a specific path, layer a CI check on top (not currently configured). +**Roles**: + +| Role | Members | Description | +|---|---|---| +| `engineering` | @ragnorc @aaltshuler | All production code under crates/**. Engine, CLI, server, compiler. | +| `docs` | @ragnorc | Documentation under docs/**, plus repo-level docs (README.md, AGENTS.md, CLAUDE.md symlink, SECURITY.md). | + + + +GitHub treats multiple owners on a CODEOWNERS line as **"any one of them satisfies the review requirement"**. To require N distinct approvers on a specific path, layer a CI check on top (not currently configured). ## How to change role membership or path mappings 1. Edit `.github/codeowners-roles.yml`. -2. Run `python3 .github/scripts/render-codeowners.py` (requires PyYAML; `pip install pyyaml`). -3. Commit both files in the same PR. +2. Open a PR. **CI re-renders for you**: the `CODEOWNERS` workflow regenerates `.github/CODEOWNERS` and the ownership tables above and auto-commits them back to your PR branch on same-repository PRs β€” you don't have to run the script locally (though you can: `python3 .github/scripts/render-codeowners.py`, requires PyYAML). + +On a fork (where CI can't push back), the workflow instead fails with the diff so you can run the script and commit it yourself. CI fails the PR if: -- `CODEOWNERS` was edited without a corresponding yml change, or -- The yml was changed but the rendered `CODEOWNERS` doesn't match. +- a fork PR left a generated artifact out of sync, or +- `CODEOWNERS` was edited without a corresponding yml change (the `CODEOWNERS not hand-edited` check). ## How to add a new role diff --git a/docs/dev/index.md b/docs/dev/index.md index e2c770e..4118bdd 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -51,6 +51,18 @@ constraints. User-facing behavior should still be documented through | Install and deployment packaging | [install.md](../user/install.md), [deployment.md](../user/deployment.md) | | Release history | [releases/](../releases/) | +## Contribution & Governance + +| Area | Read | +|---|---| +| How to contribute (external) | [CONTRIBUTING.md](../../CONTRIBUTING.md) | +| Governance model, roles, decision authority | [GOVERNANCE.md](../../GOVERNANCE.md) | +| Public contribution RFC track | [rfcs/](../rfcs/) | + +The `docs/rfcs/` track is the **public, externally-authorable** RFC process. The +maintainer/internal RFCs below (`rfc-00N-*.md`) are a separate, team-owned +track; don't conflate the two. + ## Active Implementation Plans Working documents for in-flight feature work. Removed when the work lands. @@ -61,6 +73,7 @@ Working documents for in-flight feature work. Removed when the work lands. | Inline + stored queries, request/response envelope, MCP (MR-656 / MR-976 / MR-969) | [rfc-001-queries-envelope-mcp.md](rfc-001-queries-envelope-mcp.md) | | Config & CLI architecture β€” global-first layered config, typed locators, multi-server targeting, auth model (MR-973 / MR-974 / MR-981) | [rfc-002-config-cli-architecture.md](rfc-002-config-cli-architecture.md) | | MCP server surface β€” full tool parity, stored queries, modular auth (MR-969 / MR-956 / MR-974) | [rfc-003-mcp-server-surface.md](rfc-003-mcp-server-surface.md) | +| Future cluster control plane β€” declarative as-code config, JSON state ledger, reconciler | [cluster-config-specs.md](cluster-config-specs.md), [cluster-axioms.md](cluster-axioms.md), [cluster-config-implementation-spec.md](cluster-config-implementation-spec.md) | ## Boundary diff --git a/docs/dev/invariants.md b/docs/dev/invariants.md index 5ee4f17..4baff5e 100644 --- a/docs/dev/invariants.md +++ b/docs/dev/invariants.md @@ -101,7 +101,7 @@ Use it this way: | Deletes | Inline-commit residual; delete-only queries allowed, mixed insert/update/delete rejected by D2 | [query-language.md](../user/query-language.md), [writes.md](writes.md) | | Branch delete | Manifest is the single authority, flipped atomically first; per-table forks + commit-graph branch are derived state, reclaimed best-effort (`force_delete_branch`) with the `cleanup` reconciler as the guaranteed backstop. Reusing a name whose reclaim failed before `cleanup` surfaces an actionable error | [branches-commits.md](../user/branches-commits.md), [maintenance.md](../user/maintenance.md) | | Schema validation | Type checks, required fields, defaults, edge endpoint checks, and edge cardinality are enforced on write paths | [schema-language.md](../user/schema-language.md), [execution.md](execution.md) | -| Unique constraints | Intra-batch and write-path checks exist; full cross-version uniqueness is still a gap | [schema-language.md](../user/schema-language.md) | +| Unique constraints | Intra-batch and write-path checks exist; intake and branch-merge derive the composite key through one shared function (`loader::composite_unique_key`, a separator-free `Vec` tuple) and fail loudly on an un-keyable column type rather than silently exempting it; full cross-version uniqueness against already-committed rows is still a gap | [schema-language.md](../user/schema-language.md) | | Storage trait | `TableStorage` exists as the sealed staged-write surface; full call-site migration and capability/stat surfaces are incomplete | [writes.md](writes.md), [architecture.md](architecture.md) | | Index lifecycle | `ensure_indices` is explicit today; reconciler-based convergence is roadmap | [indexes.md](../user/indexes.md), [maintenance.md](../user/maintenance.md) | | Traversal IDs | Runtime still builds `TypeIndex`; Lance stable row-id based graph IDs are roadmap | [architecture.md](architecture.md), [query-language.md](../user/query-language.md) | @@ -139,6 +139,20 @@ them explicit. Remove the skip when the upstream Lance fix lands β€” the `lance_surface_guards.rs::compact_files_still_fails_on_blob_columns` guard turns red on that bump to force it. +- **Manifestβ†’commit-graph publish atomicity:** a graph commit advances + `__manifest` (the visibility authority) and then appends `_graph_commits` as + two separate writes (`commit_updates_with_actor_with_expected`, failpoint + `graph_publish.before_commit_append`). A crash between them leaves the manifest + at version N with no commit-graph row for N. Live reads and durability are + unaffected β€” the live version resolves via the manifest + (`GraphCoordinator::version()`), not the commit-graph head β€” and the open-time + recovery sweep does NOT repair it (`lance_head == manifest_pinned` classifies + `NoMovement`; a recovery sidecar would not change this). Impact is bounded to + commit history: `commit list` misses N, time-travel by commit id to N fails, + and merge-base loses a node (a likely-benign off-by-one re-merge). This affects + every publish, not a specific maintenance command. Eventual fix: make the + commit graph reconcilable from the manifest (or the two writes atomic) β€” not a + recovery-sidecar concern. - **Planner capability/stat surfaces:** cost-aware planning, complete capability advertisement, and explain-with-cost are roadmap. Do not describe them as implemented. diff --git a/docs/dev/testing.md b/docs/dev/testing.md index febc455..a7a8d69 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -8,6 +8,7 @@ This file is the always-on map of the test surface. **Consult it before every ta |---|---|---| | `omnigraph` (engine) | `crates/omnigraph/tests/` | Integration tests (21 files), fixture-driven, share `tests/helpers/mod.rs` | | `omnigraph-cli` | `crates/omnigraph-cli/tests/` | `cli.rs` (unit-ish), `system_local.rs`, `system_remote.rs`, share `tests/support/mod.rs` | +| `omnigraph-cluster` | mostly in-source `#[cfg(test)] mod tests` | Cluster config parser, local JSON state diff, state CAS/lock handling, read-only validate/plan/status plus explicit refresh/import graph observations | | `omnigraph-server` | `crates/omnigraph-server/tests/` | `server.rs` (HTTP-level), `openapi.rs` (OpenAPI drift / regeneration) | | `omnigraph-compiler` | mostly in-source `#[cfg(test)] mod tests` | Parser, type-checker, IR lowering, lint | @@ -20,7 +21,7 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `end_to_end.rs` | Full init β†’ load β†’ query/mutate flow | | `branching.rs` | Branch create / list / delete, lazy fork | | `merge_truth_table.rs` | Merge-pair truth table (MR-786): all 9Γ—9 `(left_op, right_op)` cells from `{noop, addNode, removeNode, addEdge, removeEdge, setProperty, dropProperty, addLabel, removeLabel}`. Adding a new op to `OpVariant` forces a compile error in `build_case` until the new row + column are dispositioned. 36 executable cells run through real `branch_merge` with a structured oracle (`MergeOutcome` / `MergeConflictKind` + graph-state assert); 45 cells involving `dropProperty`/`addLabel`/`removeLabel` are recorded as `Unsupported` until the mutation grammar grows. | -| `writes.rs` | Direct-publish writes: cancellation, concurrent-writer CAS, multi-statement atomicity, MR-794 staged-write rewire (Dβ‚‚ rejection, insert+update coalesce, multi-append coalesce, partial-failure recovery, load RI/cardinality recovery) | +| `writes.rs` | Direct-publish writes: cancellation, non-strict insert/merge rebase under the per-table queue, strict stale-write conflicts, multi-statement atomicity, MR-794 staged-write rewire (Dβ‚‚ rejection, insert+update coalesce, multi-append coalesce, partial-failure recovery, load RI/cardinality recovery) | | `staged_writes.rs` | TableStore staged-write primitives (`stage_append`, `stage_merge_insert`, `commit_staged`, `scan_with_staged`, `count_rows_with_staged`) β€” primitive-level only; engine code uses the in-memory `MutationStaging` accumulator instead | | `lifecycle.rs` | Graph lifecycle, schema state | | `point_in_time.rs` | Snapshots, time travel (`snapshot_at_version`, `entity_at`) | @@ -34,10 +35,10 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `s3_storage.rs` | S3-backed graph (skipped unless `OMNIGRAPH_S3_TEST_BUCKET` is set) | | `lance_version_columns.rs` | Per-row `_row_last_updated_at_version` behavior | | `validators.rs` | Schema constraint enforcement (enum, range, unique, cardinality) across JSONL, insert, update paths | -| `maintenance.rs` | `optimize` (compaction) + `cleanup` (version GC): empty/idempotent/no-op edges, policy validation, head preservation | -| `failpoints.rs` | Failure-injection coverage (gated on `failpoints` feature). Includes the four per-writer Phase B β†’ recovery integration tests (`recovery_rolls_forward_after_finalize_publisher_failure`, `schema_apply_phase_b_failure_recovered_on_next_open`, `branch_merge_phase_b_failure_recovered_on_next_open`, `ensure_indices_phase_b_failure_recovered_on_next_open`). | +| `maintenance.rs` | `optimize` (compaction), `repair` (explicit uncovered-drift publish), and `cleanup` (version GC): empty/idempotent/no-op edges, policy validation, head preservation; `optimize` publishes its own compaction (`optimize_publishes_compaction_to_manifest_so_schema_apply_succeeds`), skips pre-existing uncovered drift (`optimize_skips_preexisting_manifest_head_drift`), and refuses to run while a `__recovery` sidecar is pending (`optimize_defers_when_recovery_sidecar_is_pending`); `repair` previews/heals verified maintenance drift, refuses raw semantic drift without `--force`, and forced repair publishes only by explicit operator choice | +| `failpoints.rs` | Failure-injection coverage (gated on `failpoints` feature). Includes the five per-writer Phase B β†’ recovery integration tests (`recovery_rolls_forward_after_finalize_publisher_failure`, `schema_apply_phase_b_failure_recovered_on_next_open`, `branch_merge_phase_b_failure_recovered_on_next_open`, `ensure_indices_phase_b_failure_recovered_on_next_open`, `optimize_phase_b_failure_recovered_on_next_open`). | | `recovery.rs` | Open-time recovery sweep β€” sidecar I/O, classifier dispatch (NoMovement / RolledPastExpected / UnexpectedAtP1 / UnexpectedMultistep / InvariantViolation), all-or-nothing decision, roll-forward via `ManifestBatchPublisher::publish`, roll-back via `Dataset::restore`, audit row in `_graph_commit_recoveries.lance`, `OpenMode::ReadOnly` skip path | -| `composite_flow.rs` | Compositional/narrative end-to-end stories β€” multi-step flows that compose mechanics covered by other test files. Catches integration regressions where individual operations all pass their unit tests but their composition breaks (sequential merges, post-merge main writes, time-travel through merge DAG, reopen consistency over multi-merge histories). | +| `composite_flow.rs` | Compositional/narrative end-to-end stories β€” multi-step flows that compose mechanics covered by other test files. Catches integration regressions where individual operations all pass their unit tests but their composition breaks (sequential merges, post-merge main writes, time-travel through merge DAG, reopen consistency over multi-merge histories, post-optimize and post-cleanup strict writes). | **Config & CLI (RFC-002 V1a + V1-remainder schema reshape)** β€” these live outside the engine `tests/` dir: the version-gated config-strictness, the v1 diff --git a/docs/dev/writes.md b/docs/dev/writes.md index 974f7a6..d2c7c7e 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -14,8 +14,11 @@ publisher's row-level CAS on `__manifest` is the single fence. - No `RunRecord`, no `_graph_runs.lance`, no `_graph_run_actors.lance`. - No `omnigraph run *` CLI subcommands and no `/runs/*` HTTP endpoints. -- No `__run__` staging branches. (Legacy on-disk artifacts from - pre-MR-771 repos are inert; MR-770 sweeps them in production.) +- No `__run__` staging branches; `__run__*` is no longer a reserved + name. The branch-name guard was removed in MR-770, and any stale + `__run__*` branch on an upgraded graph is swept off `__manifest` by the + v2β†’v3 internal-schema migration on first read-write open. (The inert + `_graph_runs.lance` bytes remain until a `delete_prefix` primitive lands.) - Cancelled mutation futures leave **no graph-level state** β€” only orphaned Lance fragments, which the existing `omnigraph cleanup` pipe reclaims. @@ -154,10 +157,14 @@ are left at `Lance HEAD = manifest_pinned + 1`. **Recovery protocol** (lifecycle of every staged-write writer β€” `MutationStaging::finalize`, `schema_apply::apply_schema_with_lock`, -`branch_merge_on_current_target`, `ensure_indices_for_branch`): +`branch_merge_on_current_target`, `ensure_indices_for_branch`, +`optimize_all_tables`): 1. **Phase A**: writer writes a sidecar JSON to - `__recovery/{ulid}.json` BEFORE its first `commit_staged`. The + `__recovery/{ulid}.json` BEFORE its first HEAD-advancing commit + (`commit_staged`, or `compact_files` for `optimize_all_tables`, + which advances the Lance HEAD via a reserve-fragments + rewrite + commit rather than a staged write). The sidecar names every `(table_key, table_path, expected_version, post_commit_pin)` it intends to commit + the writer kind + actor_id. @@ -192,8 +199,13 @@ recovery sweep in `crates/omnigraph/src/db/manifest/recovery.rs`: otherwise full open-time recovery rolls them back and refresh-time recovery leaves them for the next read-write open. - Otherwise **roll back**: per-table `Dataset::restore` to the - manifest-pinned table version for that branch. Rollback records the - actual restore target in the audit row's `to_version`. + manifest-pinned table version, then a single `ManifestBatchPublisher::publish` + of the restored HEAD β€” symmetric with roll-forward, so `manifest == HEAD` + after recovery (no residual drift). This convergence is what lets a + failed-then-retried schema apply succeed instead of failing one version higher + each iteration. The audit row's `to_version` records the logical + rolled-back-to version (`manifest_pinned`); the manifest is published at the + restore commit (`manifest_pinned + 1`, same content). - After a successful roll-forward or roll-back, an audit row is recorded β€” `_graph_commits.lance` carries a commit tagged `actor_id = "omnigraph:recovery"`, and a sibling @@ -245,9 +257,14 @@ list`. ## Migration code -`db/manifest/migrations.rs` does not change. Active deletion of -`_graph_runs.lance` belongs in MR-770 (the production sweep) β€” this PR -stops *creating* run state but does not destroy legacy bytes on disk. +`db/manifest/migrations.rs` carries the v2β†’v3 internal-schema step (MR-770): +a one-time sweep that deletes legacy `__run__*` staging branches off +`__manifest`. It runs in `Omnigraph::open(ReadWrite)` (via +`manifest::migrate_on_open`, before the coordinator reads branch state) and +again on the publisher's write path; both are idempotent once the stamp is at +v3. Deleting the inert `_graph_runs.lance` / `_graph_run_actors.lance` dataset +*bytes* is still deferred β€” it needs a `StorageAdapter::delete_prefix` +primitive β€” but those bytes are invisible to graph-level state. ## Mid-query partial failure: closed by MR-794 diff --git a/docs/releases/v0.6.1.md b/docs/releases/v0.6.1.md index aafe1af..0acc34b 100644 --- a/docs/releases/v0.6.1.md +++ b/docs/releases/v0.6.1.md @@ -7,6 +7,7 @@ v0.6.1 focuses on operational polish after v0.6.0: stored-query registries, safe - **Stored-query registries.** `omnigraph.yaml` can declare curated `queries:` blocks per graph. Servers load and type-check them at startup, `omnigraph queries validate` checks them offline, `omnigraph queries list` shows exposed queries and typed params, `GET /queries` exposes a typed catalog, and `POST /queries/{name}` invokes a stored query without accepting ad hoc `.gq` source from the client. - **Stored-query policy gate.** New Cedar action `invoke_query` gates the stored-query invocation surface. Stored mutations are double-gated: `invoke_query` to reach the stored query and `change` for the actual write. - **Safer branch deletion.** `branch_delete` now treats the manifest as the authority, flips branch visibility atomically, and reclaims per-table/commit-graph forks as derived state. If best-effort reclaim is interrupted, `cleanup` reconciles orphaned forks; reusing a branch name before cleanup reports an actionable error. +- **Legacy `__run__` cleanup (MR-770).** Removed the last functional remnant of the Run state machine (retired in v0.4.0): the `__run__` branch-name guard. A new v2β†’v3 `__manifest` internal-schema migration sweeps any stale `__run__*` staging branches on the first read-write open, so `__run__*` is no longer a reserved branch name. This closes the "unpromoted `__run__` branches block reads" condition behind the zombie-run cascade incident; the inert `_graph_runs.lance` row cleanup is tracked separately (it needs a `delete_prefix` primitive). - **Blob-safe optimize.** `omnigraph optimize` skips tables with `Blob` properties instead of failing the whole sweep on Lance's blob-v2 compaction decode bug. Skips are visible in human output, `--json` as `skipped`, `TableOptimizeStats.skipped`, and logs; non-blob tables still compact normally. - **Deployment improvements.** The container entrypoint now composes `OMNIGRAPH_TARGET_URI` with `OMNIGRAPH_CONFIG`, so operators can keep the graph URI in env while loading policy/query config from a mounted file. The local RustFS bootstrap pins RustFS beta.3 and allows the current insecure local-dev default credentials. - **Windows release support.** Tagged and edge releases now publish Windows x86_64 archives containing `omnigraph.exe` and `omnigraph-server.exe`, with a PowerShell installer and Windows install docs. @@ -17,6 +18,7 @@ v0.6.1 focuses on operational polish after v0.6.0: stored-query registries, safe - A graph selected by name (`--target` or `server.graph`) now uses `graphs..policy` and `graphs..queries`. Top-level `policy` / `queries` blocks are only for anonymous bare-URI single-graph mode; using them with a named graph now fails loudly with migration guidance. - `mcp.expose` defaults to `true` for stored-query registry entries. Set `mcp: { expose: false }` for service-only queries that should not appear in the catalog. - `invoke_query` is graph-scoped, not branch-scoped. Branch/snapshot access remains enforced by the inner `read` / `change` gate. +- **Legacy `__run__` migration.** Graphs created before v0.4.0 are migrated automatically on the first **read-write** open by a v0.6.1 binary (one-time `__manifest` stamp v2β†’v3 sweep of stale `__run__*` branches). No action required. Two caveats: (1) a graph opened **read-only** still lists any stale `__run__*` branch until its first read-write open, since the migration is write-path-only like all manifest migrations β€” long-lived read-only deployments should be opened read-write once after upgrading; (2) the inert `_graph_runs.lance` / `_graph_run_actors.lance` dataset bytes are left in place until a future `delete_prefix` primitive (they are invisible to graph-level state). - Blob tables are not compacted until the upstream Lance fix lands, so fragment count and deleted-row space on blob tables are not reclaimed by `optimize`. Reads, writes, and query results are unaffected; no on-disk migration is required. - `TableOptimizeStats` is now `#[non_exhaustive]` and gains a `skipped: Option` field (so does the new `SkipReason` enum). This is a source-level change only for downstream code that built this returned result struct by literal β€” rare, since it is produced by `optimize` and consumed by reading its fields; field access is unaffected, and `#[non_exhaustive]` keeps future additions non-breaking. diff --git a/docs/releases/v0.6.2.md b/docs/releases/v0.6.2.md new file mode 100644 index 0000000..2504813 --- /dev/null +++ b/docs/releases/v0.6.2.md @@ -0,0 +1,55 @@ +# Omnigraph v0.6.2 + +v0.6.2 is a maintenance-safety release on top of v0.6.1. It tightens the +`optimize` / recovery boundary, adds an explicit repair path for uncovered +manifest/head drift, accepts pretty-printed JSON load input, and updates the +project governance and release automation around those fixes. + +## Highlights + +- **Explicit `omnigraph repair`.** New `repair` CLI support previews uncovered + manifest/head drift by default and reports each table's classification, + action, manifest version, Lance HEAD version, Lance operations, and any + classification error. `--confirm` publishes verified maintenance-only drift; + `--force --confirm` can publish suspicious or unverifiable drift after + operator review. +- **Optimize skips uncovered drift.** `omnigraph optimize` now refuses to + interpret Lance HEAD movement that is ahead of `__manifest` without a recovery + sidecar. Those tables are reported as `skipped: DriftNeedsRepair` and left + untouched until `omnigraph repair` classifies them. +- **Optimize publishes compaction.** Successful compaction now publishes the + compacted Lance version back through the graph manifest and is covered by an + `Optimize` recovery sidecar. A crash after Lance compaction but before + manifest publish converges through the normal recovery sweep instead of + leaving hidden drift. +- **Recovery roll-back convergence.** Recovery roll-back now aligns the + manifest-visible version after restoring a table, closing the residual where + Lance HEAD and `__manifest` could stay out of sync after recovery. +- **Pretty-printed JSON load input.** `load` accepts multi-line JSON objects in + addition to one-object-per-line JSONL, so formatted fixture or export files no + longer need to be minified before import. + +## Operational Notes + +- `repair` requires a clean recovery state. Pending `__recovery` sidecars still + belong to automatic open-time recovery; reopen the graph first, then run + repair if drift remains. +- `repair --confirm` only auto-publishes drift made of Lance maintenance + operations (`Rewrite` and `ReserveFragments`). Semantic operations such as + append, delete, update, and merge are refused unless the operator uses + `--force --confirm`. +- `optimize` remains non-destructive. It still skips blob-bearing tables while + OmniGraph is pinned to the Lance version with the blob-v2 compaction issue. +- No manual on-disk migration is required. Existing graphs open under v0.6.2; + the internal manifest schema stamp remains v3. + +## Docs, Governance, And CI + +- Added issue, discussion, RFC, and pull-request templates plus governance docs + for the external contribution path. +- Regenerated CODEOWNERS tables and adjusted branch-protection docs so code + owners can bypass required PR review where repository rules allow it. +- Trimmed Windows release builds out of per-PR CI and kept Windows packaging on + tag releases. +- Made Homebrew audit diagnostic-only in the release workflow so a flaky audit + cannot block publishing an otherwise valid formula update. diff --git a/docs/rfcs/0000-template.md b/docs/rfcs/0000-template.md new file mode 100644 index 0000000..48f4bda --- /dev/null +++ b/docs/rfcs/0000-template.md @@ -0,0 +1,54 @@ +# RFC NNNN: + +| | | +|---|---| +| **Status** | Proposed | +| **Author(s)** | <your name / handle> | +| **Discussion** | <link to the originating Discussion, if any> | +| **Implementation** | <issue/PR links, filled in as work lands> | + +> Status is maintained by maintainers: `Proposed` while the PR is open, +> `Accepted` on merge, `Declined` on close, `Superseded by NNNN` later. + +## Summary + +One paragraph: what this changes, in plain terms. + +## Motivation + +What problem does this solve, and why is it worth the ongoing cost? Tie it to a +concrete need (a Discussion, a recurring issue, a user request). Per the +project's first principle, argue the *long-run liability*, not just the +short-term convenience. + +## Guide-level explanation + +Explain the change as you'd teach it to a user or contributor: new commands, +syntax, API shapes, behavior. Examples first. + +## Reference-level design + +The precise design: data structures, IR/AST/planner changes, storage/format +impact, migration path, error behavior. Enough that a reviewer can find the +holes. + +## Invariants & deny-list check + +Which Hard Invariants in [../dev/invariants.md](../dev/invariants.md) does this +touch? Does it brush against any deny-list item β€” and if so, why is this the +justified exception? State explicitly that no invariant is weakened, or which +Known Gap moves. + +## Drawbacks & alternatives + +What does this cost, what did you reject, and why. "Do nothing" is a valid +alternative to weigh. + +## Reversibility + +Is this reversible? On-disk/wire/format and substrate choices are near-permanent +and demand more evidence; a CLI flag or doc is cheap to undo. Say which this is. + +## Unresolved questions + +What's deliberately left open for review to settle. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md new file mode 100644 index 0000000..99cdd76 --- /dev/null +++ b/docs/rfcs/README.md @@ -0,0 +1,66 @@ +# RFCs + +Substantial changes to OmniGraph β€” new user-facing surface, format or protocol +changes, anything irreversible or cross-cutting β€” go through a lightweight RFC +so the design is agreed *as reviewable code* before implementation starts. This +is the public RFC track, open to **anyone, including external contributors**. + +This complements the always-on review bar in +[../dev/invariants.md](../dev/invariants.md): the invariants say *what every +change must respect*; an RFC says *why this particular change is worth making and +how*. + +> **Two tracks, don't conflate them.** This `docs/rfcs/` directory is the +> **public contribution** track (anyone authors; maintainers accept). The +> maintainer-internal RFCs under `docs/dev/rfc-00N-*.md` are a separate, +> team-owned track for in-flight internal work. If you're an outside +> contributor, you're in the right place here. + +## When you need one + +- **RFC required:** new query/schema/CLI/HTTP surface; on-disk or wire-format + changes; a new substrate dependency; anything the deny-list in + [../dev/invariants.md](../dev/invariants.md) flags; anything irreversible + ("reversibility shapes evidence demand"). +- **RFC not required:** bug fixes for an `accepted` issue, and the trivial + fast-lane (typos, docs, deps) β€” see [../../CONTRIBUTING.md](../../CONTRIBUTING.md). + +If you're unsure, start a [Discussion](../../../discussions); a maintainer will +tell you whether it needs an RFC. + +## Lifecycle + +``` +Discussion (incubate, get rough consensus) + β”‚ graduate + β–Ό +RFC pull request β†’ adds docs/rfcs/NNNN-title.md (Status: Proposed) + β”‚ +maintainer review ──▢ changes requested / declined (PR closed, with rationale) + β”‚ + β–Ό +merged == Accepted (the merged file is the durable decision record) + β”‚ + β–Ό +Implementation PR(s) reference the accepted RFC +``` + +- **Author:** anyone. **Acceptance:** a maintainer decision, performed by + merging the RFC PR. Declining is closing it with rationale. +- The merged RFC *is* the accepted record β€” there is no separate sign-off step. +- Later reversals don't edit history: supersede with a new RFC that links back + and flip the old one's `Status` to `Superseded`. + +## Numbering & naming + +- File: `docs/rfcs/NNNN-kebab-title.md`, where `NNNN` is the next free + zero-padded integer (`0001`, `0002`, …). `0000-template.md` is reserved. +- Pick the number when you open the PR; if it collides with another in-flight + RFC, the second to merge bumps theirs. + +## Status values + +`Proposed` (open PR) Β· `Accepted` (merged) Β· `Declined` (closed) Β· +`Superseded by NNNN` Β· `Implemented` (set once the work lands, optional). + +Copy [0000-template.md](0000-template.md) to start. diff --git a/docs/user/audit.md b/docs/user/audit.md index e8abe5b..ab028ac 100644 --- a/docs/user/audit.md +++ b/docs/user/audit.md @@ -4,4 +4,4 @@ - `_as` variants of every write API let callers override the actor: `mutate_as`, `ingest_as`, `branch_merge_as`, `apply_schema_as`, etc. - Actor IDs are persisted on `GraphCommit.actor_id` with split storage in `_graph_commit_actors.lance` (the commit graph is split into `_graph_commits.lance` for the linkage and `_graph_commit_actors.lance` for the actor map). - HTTP server uses the bearer-token actor automatically; CLI uses the local user / explicit env (no implicit actor). -- Pre-v0.4.0 graphs also stored actor IDs on `RunRecord.actor_id` in `_graph_runs.lance` / `_graph_run_actors.lance`. The Run state machine was removed in MR-771; those files are inert post-v0.4.0 and reclaimed by MR-770's production sweep. +- Pre-v0.4.0 graphs also stored actor IDs on `RunRecord.actor_id` in `_graph_runs.lance` / `_graph_run_actors.lance`. The Run state machine was removed in MR-771; those files are inert post-v0.4.0. The v2β†’v3 manifest migration sweeps any stale `__run__*` branches on first write-open (MR-770); the inert dataset bytes remain until a `delete_prefix` primitive lands. diff --git a/docs/user/branches-commits.md b/docs/user/branches-commits.md index c1894f9..a4044cb 100644 --- a/docs/user/branches-commits.md +++ b/docs/user/branches-commits.md @@ -9,8 +9,8 @@ Lance supports branching at the dataset level: a branch is a named lineage of ve OmniGraph builds *graph branches* on top by branching every sub-table coherently: - `branch_create(name)` / `branch_create_from(target, name)` β€” disallowed name `main`; fails if branch exists; ensures the schema-apply lock is idle. Atomic and authority-first like `branch_delete`: it flips the `__manifest` branch (authority), then creates the derived commit-graph branch, force-dropping any orphaned commit-graph ref left by an incomplete prior delete (the manifest branch is fresh, so a same-named commit-graph branch is provably a zombie). If commit-graph creation fails, the manifest branch is rolled back so the name never half-exists. -- `branch_list()` β€” returns public branches, **filters internal** `__run__…` and `__schema_apply_lock__` prefixes. -- `branch_delete(name)` β€” refuses if there are descendants or active runs on the branch. The manifest is the single authority for branch existence: deletion flips the `__manifest` branch ref first (one atomic op), after which the branch is gone from every snapshot. The owned per-table forks and the commit-graph branch are derived state, reclaimed best-effort with `force_delete_branch` after the flip. A failure during that reclaim (transient object-store error) does not fail the call or block the authority flip; the leftover forks are unreachable orphans that the [`cleanup`](maintenance.md) reconciler converges. One consequence: if a delete's best-effort reclaim fails, reusing that branch name before the next `cleanup` surfaces a clear error pointing at `cleanup` (the stale fork would otherwise collide on first write). +- `branch_list()` β€” returns public branches, **filters the internal** `__schema_apply_lock__` branch. +- `branch_delete(name)` β€” refuses if there are descendants on the branch, or if it is the current branch. The manifest is the single authority for branch existence: deletion flips the `__manifest` branch ref first (one atomic op), after which the branch is gone from every snapshot. The owned per-table forks and the commit-graph branch are derived state, reclaimed best-effort with `force_delete_branch` after the flip. A failure during that reclaim (transient object-store error) does not fail the call or block the authority flip; the leftover forks are unreachable orphans that the [`cleanup`](maintenance.md) reconciler converges. One consequence: if a delete's best-effort reclaim fails, reusing that branch name before the next `cleanup` surfaces a clear error pointing at `cleanup` (the stale fork would otherwise collide on first write). - **Lazy forking**: a branch only forks a sub-table when that sub-table is first mutated on it. Pure-read branches share fragments with their source. A fork collision is classified by the manifest authority, not by Lance branch versions: if the live manifest already records the fork on the active branch, a concurrent first-write won and the caller gets a retryable "refresh and retry"; if the manifest does not, a physical branch there is an orphan and the caller is pointed at `cleanup`. - `sync_branch(branch)` β€” re-binds the in-memory handle to the latest head of the branch. @@ -51,13 +51,13 @@ Notes: ## L2 β€” Internal system branches -Filtered from `branch_list()` but visible to internals: +Internal or legacy branch refs: -- `__schema_apply_lock__` β€” serializes schema migrations. -- `__run__<run-id>` β€” legacy from the pre-v0.4.0 Run state machine (removed in MR-771). The branch-name guard predicate `is_internal_run_branch` is kept as defense-in-depth so users cannot create a branch matching the legacy prefix; the filter will be removed once production legacy branches are swept (MR-770). +- `__schema_apply_lock__` β€” serializes schema migrations; filtered from `branch_list()` but visible to internals. +- `__run__<run-id>` β€” legacy from the pre-v0.4.0 Run state machine (removed in MR-771). These are swept off `__manifest` on the first read-write open by the v2β†’v3 internal-schema migration (MR-770), and `__run__*` is no longer a reserved name. Known limitation: a pre-v0.4.0 graph opened **read-only** still surfaces any stale `__run__*` branch in `branch_list()` until its first read-write open (the migration is write-path-only, like all manifest migrations). ## L2 β€” Recovery audit trail -The four migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`) protect their multi-table commits with a sidecar at `__recovery/{ulid}.json` written before Phase B and deleted after Phase C. The next `Omnigraph::open` (gated on `OpenMode::ReadWrite`) runs the recovery sweep in `crates/omnigraph/src/db/manifest/recovery.rs`: classify per-table state, decide all-or-nothing per sidecar, roll forward / back, record an audit row. +The five migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`, `optimize_all_tables`) protect their multi-table commits with a sidecar at `__recovery/{ulid}.json` written before Phase B and deleted after Phase C. The next `Omnigraph::open` (gated on `OpenMode::ReadWrite`) runs the recovery sweep in `crates/omnigraph/src/db/manifest/recovery.rs`: classify per-table state, decide all-or-nothing per sidecar, roll forward / back, record an audit row. Audit rows live in `_graph_commit_recoveries.lance` (sibling to `_graph_commits.lance`) and reference the commit graph by `graph_commit_id`. The linked recovery commit is identified by that same `graph_commit_id`, and `actor_id="omnigraph:recovery"` is stored in `_graph_commit_actors.lance` (joined by `graph_commit_id`) β€” `_graph_commits.lance` itself does not carry the `actor_id` column. To find recoveries for a specific original actor: `omnigraph commit list --filter actor=omnigraph:recovery`, then join to `_graph_commit_recoveries.lance` by `graph_commit_id` to read `recovery_for_actor`. Schema: see `crates/omnigraph/src/db/recovery_audit.rs`. diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 115ac86..a1fc238 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -2,7 +2,7 @@ A reference for the `omnigraph` binary's command surface and `omnigraph.yaml` schema. For a quick-start guide, see [cli.md](cli.md). -17 top-level command families, 40+ subcommands. All commands accept either a positional `URI`, `--uri`, or a `--graph <name>` resolved against `omnigraph.yaml`. +Top-level command families and subcommands. Graph-targeting commands accept either a positional `URI`, `--uri`, or a `--graph <name>` resolved against `omnigraph.yaml`; `cluster` commands use `--config <dir>`. ## Top-level commands @@ -17,11 +17,12 @@ A reference for the `omnigraph` binary's command surface and `omnigraph.yaml` sc | `export` | dump to JSONL on stdout (`--type T`, `--table K` filters) | | `branch create \| list \| delete \| merge` | branching ops | | `commit list \| show` | inspect commit graph | -| `run list \| show \| publish \| abort` | transactional run ops | | `schema plan \| apply \| show (alias: get)` | migrations | | `lint` (alias: `check`) | offline / graph-backed query validation. Replaces `query lint` / `query check`, which are kept as deprecated argv-level shims that print a one-line warning and rewrite to `omnigraph lint` | | `queries validate \| list` | operate on the server-side stored-query registry (the `queries:` block). `validate` type-checks every stored query against the live schema offline (opens the selected graph; exits non-zero on any breakage), catching schema drift without restarting the server; `list` prints the selected registry's query names, MCP exposure, and typed params. For per-graph registries, pass `--graph <graph>` or set `defaults.graph`; with no graph selection, `list` shows only top-level `queries:`. Distinct from `lint`, which validates a single `.gq` file | -| `optimize` | non-destructive Lance compaction (skips tables with `Blob` columns; `--json` reports a `skipped` field) | +| `cluster validate \| plan \| status \| refresh \| import` | cluster-control preview. `validate` checks a local `cluster.yaml` folder and referenced schema/query/policy files; `plan` diffs it against local JSON state at `__cluster/state.json`; `status` reads the state ledger; `refresh`/`import` explicitly update local JSON state from read-only graph observations. No apply, graph-resource mutation, server change, or `plan --refresh` occurs in Stage 2B | +| `optimize` | non-destructive Lance compaction (skips tables with `Blob` columns or uncovered drift; `--json` reports `skipped`) | +| `repair [--confirm] [--force]` | preview or explicitly publish uncovered manifest/head drift. `--confirm` heals verified maintenance drift and exits non-zero if suspicious/unverifiable drift is refused; `--force --confirm` publishes suspicious/unverifiable drift after operator review | | `cleanup --keep N --older-than 7d --confirm` | destructive version GC | | `embed` | offline JSONL embedding pipeline | | `policy validate \| test \| explain` | Cedar tooling. Selects `defaults.graph`, else `serve.graphs`, else top-level `policy.file` | @@ -123,6 +124,28 @@ directory of the layer that defined them. `config view --resolved <graph>` prints the final embedded/remote locator. The server does **not** layer the global config β€” it reads only the project/`--config` manifest. +## Cluster config preview + +```bash +omnigraph cluster validate --config ./company-brain +omnigraph cluster plan --config ./company-brain --json +omnigraph cluster status --config ./company-brain --json +omnigraph cluster refresh --config ./company-brain --json +omnigraph cluster import --config ./company-brain --json +``` + +`--config` is a directory containing `cluster.yaml`; it defaults to `.`. +Stage 2B accepts graphs, schemas, stored queries, and policy bundle file +references. `cluster plan` reads local JSON state from +`<config-dir>/__cluster/state.json`; a missing file means empty state. Plan, +refresh, and import acquire `__cluster/lock.json` by default and release it +before returning. `cluster status` reads state only and reports any existing +lock. `refresh` requires an existing `state.json`; `import` creates one only +when it is missing. Both observe declared graphs read-only at +`<config-dir>/graphs/<graph-id>.omni`. External state backends, apply, +`plan --refresh`, pipelines, UI specs, embeddings, aliases, and bindings are +reserved for later stages. See [cluster-config.md](cluster-config.md). + ## Output formats (`query` command, alias: `read`) - `json` β€” pretty-printed object with metadata + rows diff --git a/docs/user/cluster-config.md b/docs/user/cluster-config.md new file mode 100644 index 0000000..77954bd --- /dev/null +++ b/docs/user/cluster-config.md @@ -0,0 +1,152 @@ +# Cluster Config + +**Status:** Stage 2B state-observation preview. + +Cluster config is the future control-plane configuration surface for a whole +OmniGraph deployment. In this stage, OmniGraph can validate a local +`cluster.yaml` folder, produce a deterministic read-only plan, inspect the +local JSON state ledger, and explicitly refresh/import graph observations into +that ledger. It does not apply desired changes, start servers, or write graph +resources. + +## Commands + +```bash +omnigraph cluster validate --config ./company-brain +omnigraph cluster plan --config ./company-brain --json +omnigraph cluster status --config ./company-brain --json +omnigraph cluster refresh --config ./company-brain --json +omnigraph cluster import --config ./company-brain --json +``` + +`--config` points at a directory, not a file. The directory must contain +`cluster.yaml`. When omitted, it defaults to the current directory. + +## Supported `cluster.yaml` + +Stage 2B accepts only the read-only resource subset: + +```yaml +version: 1 +metadata: + name: company-brain + +state: + backend: cluster + lock: true + +graphs: + knowledge: + schema: ./knowledge.pg + queries: + find_experts: + file: ./knowledge.gq + +policies: + base: + file: ./base.policy.yaml + applies_to: [knowledge] +``` + +`metadata.name` is a display label. `state.backend` may be omitted or set to +`cluster`; external state backends are reserved for a later stage. `state.lock` +defaults to `true`. When enabled, `cluster plan`, `cluster refresh`, and +`cluster import` briefly acquire `<config-dir>/__cluster/lock.json`, then remove +it before returning. `cluster status` never acquires the lock; it only reports +whether one is present. + +## Validation + +`cluster validate` checks: + +- `cluster.yaml` syntax and supported fields +- duplicate YAML keys +- schema, query, and policy file existence +- schema parsing and catalog construction +- stored-query parsing and query-name matching +- stored-query type-checking against the desired schema +- policy `applies_to` graph references + +Fields reserved for later phases, such as `pipelines`, `embeddings`, `ui`, +`aliases`, and `bindings`, fail with a typed diagnostic instead of being +silently ignored. + +## Planning + +`cluster plan` first performs validation, then reads local JSON state from: + +```text +<config-dir>/__cluster/state.json +``` + +If the file is missing, the state is treated as empty and every desired +resource is planned as a create. If present, the file must use this shape: + +```json +{ + "version": 1, + "state_revision": 0, + "applied_revision": { + "config_digest": "...", + "resources": { + "graph.knowledge": { "digest": "..." }, + "schema.knowledge": { "digest": "..." }, + "query.knowledge.find_experts": { "digest": "..." }, + "policy.base": { "digest": "..." } + } + }, + "resource_statuses": { + "graph.knowledge": { + "status": "applied", + "conditions": [], + "message": "optional status detail" + } + }, + "approval_records": {}, + "recovery_records": {}, + "observations": {} +} +``` + +`state_revision`, `resource_statuses`, `approval_records`, `recovery_records`, +and `observations` are optional so older Stage 1 state fixtures keep working. +Missing `state_revision` is treated as `0`. Resource status values are +`pending`, `planned`, `applying`, `applied`, `drifted`, `blocked`, or `error`. + +Plan output compares desired resource digests against state resource digests +and reports `create`, `update`, and `delete` changes. It also reports the state +CAS (`sha256:<digest>`) and state revision. `state_observations.locked` means an +existing lock file was observed; a successful `plan` instead reports +`lock_acquired: true` and an `acquired_lock_id`, then releases the lock before +returning. The command never writes `state.json` and does not scan live graphs. +Use explicit `cluster refresh` / `cluster import` when the state ledger should +be updated from live observations. Apply and live drift scans during plan are +later-stage work. + +## Status + +`cluster status` reads the same local JSON state ledger and prints what the +ledger says is deployed. It does not validate referenced schema/query/policy +files and does not inspect live graphs. Missing `state.json` succeeds with a +warning; invalid state JSON or an unsupported state version fails. + +## Refresh And Import + +`cluster refresh` updates an existing `state.json` from actual observations. +`cluster import` creates the first `state.json` when the ledger is missing. +Both commands open declared graphs read-only at: + +```text +<config-dir>/graphs/<graph-id>.omni +``` + +They observe only branch `main`, recording graph existence, manifest version, +live schema digest, desired schema digest, and schema-match status under +`observations["graph.<id>"]`. Missing graph roots are recorded as drift and +remove the graph/schema digests from state so a later `plan` proposes creates. +Invalid graph roots are recorded as errors; `refresh` persists the error +observation and exits non-zero, while `import` exits non-zero without creating +initial state. + +Refresh/import do not observe query or policy resources yet. Existing query and +policy state digests are preserved on refresh and are not invented on import. diff --git a/docs/user/constants.md b/docs/user/constants.md index 8f13555..f523042 100644 --- a/docs/user/constants.md +++ b/docs/user/constants.md @@ -4,15 +4,19 @@ |---|---|---| | `MANIFEST_DIR` | `__manifest` | `db/manifest/layout.rs` | | Commit graph dir | `_graph_commits.lance` | `db/commit_graph.rs` | -| Run registry dir (legacy, removed MR-771) | `_graph_runs.lance` | inert post-v0.4.0; reclaimed by MR-770 | -| Run branch prefix (legacy, removed MR-771) | `__run__` | filtered by `is_internal_run_branch` defense-in-depth | +| Run registry dir (legacy, removed MR-771) | `_graph_runs.lance` | inert post-v0.4.0; bytes remain until a `delete_prefix` primitive lands | +| Run branch prefix (legacy, removed MR-771/MR-770) | `__run__` | swept off `__manifest` by the v2β†’v3 migration; no longer a reserved name | | Schema apply lock | `__schema_apply_lock__` | `db/mod.rs` | | Manifest publisher retry budget | `PUBLISHER_RETRY_BUDGET = 5` | `db/manifest/publisher.rs` | -| Internal manifest schema version | `INTERNAL_MANIFEST_SCHEMA_VERSION = 2` | `db/manifest/migrations.rs` | +| Internal manifest schema version | `INTERNAL_MANIFEST_SCHEMA_VERSION = 3` | `db/manifest/migrations.rs` | | Merge stage batch | `MERGE_STAGE_BATCH_ROWS = 8192` | `exec/merge.rs` | | Maintenance concurrency | `OMNIGRAPH_MAINTENANCE_CONCURRENCY=8` | `db/omnigraph/optimize.rs` | | Lance blob compaction support | `LANCE_SUPPORTS_BLOB_COMPACTION = false` | `db/omnigraph/optimize.rs` | | Graph index cache size | `8` (LRU) | `runtime_cache.rs` | +| Expand indexed-path frontier ceiling | `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER=1024` | `exec/query.rs` | +| Expand indexed-path hop ceiling | `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS=6` | `exec/query.rs` | +| Expand CSR-build cost factor | `CSR_BUILD_FACTOR = 1.5` | `exec/query.rs` | +| Expand mode override | `OMNIGRAPH_TRAVERSAL_MODE` (`indexed`\|`csr`; unset = cost-based auto) | `exec/query.rs` | | Default body limit | `1 MB` | `omnigraph-server/lib.rs` | | Ingest body limit | `32 MB` | `omnigraph-server/lib.rs` | | Engine embed model | `gemini-embedding-2-preview` | `omnigraph/embedding.rs` | @@ -21,3 +25,16 @@ | Embed retries | `4` | both clients | | Embed retry backoff | `200 ms` | both clients | | LANCE memory pool default | `1 GB` (raised in v0.3.0) | runtime | + +**Expand traversal dispatch.** With `OMNIGRAPH_TRAVERSAL_MODE` unset, the engine +chooses the indexed (per-hop BTREE) vs CSR (whole-graph in-memory) path with a +cost model over cheap manifest counts (frontier size, |E|, source-vertex count, +hops) plus the index-coverage signal: the indexed path is preferred when its +frontier-relative work beats building the CSR (β‰ˆ when `hops Γ— frontier` is a +small fraction of the source-vertex set), and CSR is preferred for dense/deep +traversals or when the BTREE coverage is degraded and a full scan would be paid +per hop. The two ceilings bound the **initial dispatch** frontier/hops (beyond +them CSR is always used); they are not a hard per-hop bound β€” the cost model +*estimates* total indexed work as ~`hops Γ— frontier Γ— fanout`, so dense fan-out is +priced toward CSR rather than capped mid-traversal. The override flag forces a path (the `auto` result is identical either way; +only the path differs). diff --git a/docs/user/index.md b/docs/user/index.md index 1b93efa..6cf6ade 100644 --- a/docs/user/index.md +++ b/docs/user/index.md @@ -13,6 +13,7 @@ of MRs, internal recovery mechanics, or contributor-only invariants. | Install OmniGraph | [install.md](install.md) | | Run the CLI locally | [cli.md](cli.md) | | Look up every CLI flag and config field | [cli-reference.md](cli-reference.md) | +| Validate and plan cluster config | [cluster-config.md](cluster-config.md) | | Write schemas | [schema-language.md](schema-language.md) | | Read schema-lint diagnostic codes | [schema-lint.md](schema-lint.md) | | Write queries and mutations | [query-language.md](query-language.md) | diff --git a/docs/user/indexes.md b/docs/user/indexes.md index ce6c728..df898c4 100644 --- a/docs/user/indexes.md +++ b/docs/user/indexes.md @@ -21,6 +21,6 @@ This is OmniGraph-specific (not Lance): - `TypeIndex`: dense `u32 ↔ String id` mapping per node type. - `CsrIndex`: Compressed Sparse Row representation of edges per edge type β€” `offsets[i]..offsets[i+1]` slices into `targets`. -- `GraphIndex { type_indices, csr (out), csc (in) }` β€” built on demand from a snapshot's edge tables. +- `GraphIndex { type_indices, csr (out), csc (in) }` β€” built on demand from a snapshot's edge tables, **lazily**: only when an `Expand` the planner routes to the CSR path (dense / large frontier) or an `AntiJoin` actually needs it. - Cached in `RuntimeCache::graph_indices` (LRU, max 8 entries, keyed by snapshot id + edge table versions). -- Built only when an `Expand` or `AntiJoin` IR op is present in the lowered query, so pure scans skip it. +- Selective `Expand`s resolve neighbors from the persisted `src`/`dst` BTREE instead (one indexed scan per hop) and never trigger the CSR build; see [query-language](query-language.md) β†’ Expand. Pure scans, and queries served entirely by the indexed traversal path, skip it. diff --git a/docs/user/maintenance.md b/docs/user/maintenance.md index 3628fa0..e69bba3 100644 --- a/docs/user/maintenance.md +++ b/docs/user/maintenance.md @@ -1,15 +1,26 @@ -# Maintenance: Optimize & Cleanup +# Maintenance: Optimize, Repair & Cleanup -`db/omnigraph/optimize.rs`. +`db/omnigraph/optimize.rs` and `db/omnigraph/repair.rs`. ## `optimize_all_tables(db)` β€” non-destructive -- Lance `compact_files()` on every node + edge table on `main`. -- Rewrites small fragments into fewer large ones; old fragments remain reachable via older manifests. +- Lance `compact_files()` on every node + edge table on `main`, then **publishes the compacted version to the `__manifest`** so the manifest's `table_version` tracks the compacted Lance HEAD. Reads pin the manifest version, so without this publish compaction would be invisible to readers *and* would break the HEAD-vs-manifest precondition of the next schema apply / strict update/delete ("stale view … refresh and retry"). The publish advances the graph version (a system-attributed commit) only for tables that actually compacted. +- Rewrites small fragments into fewer large ones; old fragments remain reachable via older manifests until `cleanup` runs. +- Each table's compactβ†’publish runs under its per-`(table, main)` write queue (serializing with concurrent mutations β€” compaction is a Lance `Rewrite` op that retryable-conflicts with a concurrent merge/update/delete on overlapping fragments). The Lance-HEAD-before-manifest-publish gap is covered by a `SidecarKind::Optimize` recovery sidecar (loose-match): a crash in that window rolls the compacted version forward on the next `Omnigraph::open` (compaction is content-preserving, so roll-forward is always safe). +- **Requires a recovered graph.** `optimize` refuses (errors) when an unresolved recovery sidecar is present under `__recovery` β€” operating on an unrecovered graph could publish a partial write the open-time recovery sweep would roll back. Reopen the graph to run the recovery sweep, then re-run `optimize`. +- **Uncovered drift is skipped, not interpreted.** If a table's Lance HEAD is ahead of the version recorded in `__manifest` and no recovery sidecar covers that movement, `optimize` reports `skipped: Some(DriftNeedsRepair)` with the manifest/head versions and leaves the table untouched. Run `omnigraph repair` to classify and explicitly publish that drift. - Bounded by `OMNIGRAPH_MAINTENANCE_CONCURRENCY` (default 8). -- Returns `[TableOptimizeStats { table_key, fragments_removed, fragments_added, committed, skipped }]`. +- Returns `[TableOptimizeStats { table_key, fragments_removed, fragments_added, committed, skipped, manifest_version, lance_head_version }]`. - **Blob tables are skipped.** A table that declares any `Blob` property is not compacted: it is reported with `skipped: Some(BlobColumnsUnsupportedByLance)` (and logged via `tracing::warn`) instead of compacted, and the rest of the sweep proceeds normally. The current Lance `compact_files` mis-decodes blob-v2 columns under its forced `BlobHandling::AllBinary` read; **reads and writes are unaffected** β€” only compaction is. This is gated by `LANCE_SUPPORTS_BLOB_COMPACTION` (`db/omnigraph/optimize.rs`) and removed when the upstream Lance fix lands (see [docs/dev/lance.md](../dev/lance.md)). Consequence: fragment count and deleted-row space on blob tables are not reclaimed until then; query results are never affected. +## `repair_all_tables(db, options)` β€” explicit + +- Handles **uncovered manifest/head drift**: a table's Lance HEAD is ahead of the manifest pin and no recovery sidecar records the writer intent. +- Preview by default. `omnigraph repair --json <uri>` reports each table's `classification`, `action`, manifest/head versions, Lance operation names, and any classification error. `--confirm` publishes only verified maintenance drift; if any suspicious or unverifiable table is refused, the CLI prints the per-table output and exits non-zero. `--force --confirm` also publishes suspicious or unverifiable drift after operator review. +- Classifies drift by reading Lance transactions from `manifest_version + 1` through `lance_head_version`. Only `ReserveFragments` and `Rewrite` are verified maintenance. Semantic operations such as `Append`, `Delete`, `Update`, `Merge`, or missing transaction history are not auto-healed. +- Publishes repair by advancing `__manifest` to the existing Lance HEAD; it does **not** rewrite Lance data. If the publish succeeds, normal reads and strict writes use the repaired version. If it fails, no new data-side partial state was created. +- Requires a clean recovery state. Pending `__recovery` sidecars still belong to automatic sidecar recovery, not manual repair. + ## `cleanup_all_tables(db, options)` β€” destructive - Lance `cleanup_old_versions()` per table. diff --git a/docs/user/query-language.md b/docs/user/query-language.md index 6c7516f..acdc45d 100644 --- a/docs/user/query-language.md +++ b/docs/user/query-language.md @@ -55,6 +55,8 @@ Used inside MATCH or as expressions inside RETURN/ORDER: - `order { <expr> [asc|desc], … }` β€” supports plain expressions and `nearest(...)`. - `limit <integer>` β€” required when there is a `nearest(...)` ordering. +- **Total, deterministic order.** Rows with equal user-sort keys are broken by the bound entities' key columns (`<var>.id`, ascending) appended as a final tie-break, so the result is a *total* order β€” reproducible across runs, and `order … limit N` returns a deterministic top-N even when ties straddle the cutoff. (Aggregate results have no entity-key columns; their group rows are already distinct on the projected group keys.) +- **NULL placement** is *nulls-first ascending, nulls-last descending* (i.e. `nulls_first = !descending`): a NULL sorts as if smaller than any value. ## Mutation statements @@ -79,7 +81,7 @@ Reason: under the staged-write rewire (MR-794), inserts and updates accumulate i Pipeline operations: - `NodeScan { variable, type_name, filters }` -- `Expand { src_var, dst_var, edge_type, direction (Out|In), dst_type, min_hops, max_hops, dst_filters }` β€” destination filters are pushed *into* the expand so Lance scalar pushdown can prune. +- `Expand { src_var, dst_var, edge_type, direction (Out|In), dst_type, min_hops, max_hops, dst_filters }` β€” destination filters are pushed *into* the expand so Lance scalar pushdown can prune. Executed one of two ways, chosen per-expand by a cost model over cheap manifest counts (frontier size, |E|, source-vertex count, hops) plus index coverage: selective traversals (small frontier relative to the source set) resolve neighbors from the persisted `src`/`dst` BTREE (one indexed scan per hop); dense / deep / large-frontier traversals β€” or those whose BTREE coverage is degraded so a full scan would be paid per hop β€” use the in-memory CSR adjacency index. Both produce identical results. The `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER` / `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS` ceilings bound the *initial dispatch* frontier/hops (beyond them CSR is always used); the cost model estimates total indexed work as ~`hops Γ— frontier Γ— fanout` and prices dense fan-out toward CSR β€” they are not a hard per-hop bound. `OMNIGRAPH_TRAVERSAL_MODE=indexed|csr` forces a mode (see [constants](constants.md)). - `Filter { left, op, right }` - `AntiJoin { outer_var, inner: Vec<IROp> }` β€” for `not { … }` diff --git a/docs/user/storage.md b/docs/user/storage.md index c22d4d6..2c57a92 100644 --- a/docs/user/storage.md +++ b/docs/user/storage.md @@ -22,7 +22,7 @@ OmniGraph is **not** a single Lance dataset; it is a *graph* of datasets coordin - `edges/{fnv1a64-hex(edge_type_name)}` β€” one Lance dataset per edge type - `__manifest/` β€” the catalog of all sub-tables and their published versions - `_graph_commits.lance` / `_graph_commit_actors.lance` β€” the commit graph and its actor map - - (legacy `_graph_runs.lance` / `_graph_run_actors.lance` from pre-v0.4.0 graphs are inert; the run state machine was removed in MR-771 and these files are cleaned up via MR-770's production sweep) + - (legacy `_graph_runs.lance` / `_graph_run_actors.lance` from pre-v0.4.0 graphs are inert; the run state machine was removed in MR-771. The v2β†’v3 manifest migration sweeps stale `__run__*` branches on first write-open; the inert dataset bytes themselves remain until a `delete_prefix` storage primitive lands) - **Manifest row schema** (`object_id, object_type, location, metadata, base_objects, table_key, table_version, table_branch, row_count`): - `object_type` ∈ `table | table_version | table_tombstone` - `table_key` ∈ `node:<TypeName> | edge:<EdgeName>` @@ -47,6 +47,7 @@ Adding a new on-disk shape change is one constant bump (`INTERNAL_MANIFEST_SCHEM |---|---| | v1 (implicit, pre-stamp) | `__manifest.object_id` had no PK annotation; publisher had no row-level CAS protection. | | v2 | `__manifest.object_id` carries `lance-schema:unenforced-primary-key=true`; row-level CAS engaged. Stamped as `omnigraph:internal_schema_version=2`. | +| v3 | One-time sweep of legacy `__run__*` staging branches (pre-v0.4.0 Run state machine, removed MR-771) off `__manifest`. Runs at `Omnigraph::open(ReadWrite)` and on publish. Stamped as `omnigraph:internal_schema_version=3`. | ## On-disk layout @@ -91,9 +92,9 @@ flowchart TB - **Graph root** is one directory (or S3 prefix). Everything below is part of one OmniGraph graph. - **`__manifest/`** is a Lance dataset whose rows describe which sub-table version is published at which graph-branch. Reading a snapshot starts here. - **`nodes/`** and **`edges/`** are sibling directories holding one Lance dataset per declared type. Names are `fnv1a64-hex` of the type name to keep paths fixed-length and case-safe. -- **`_graph_commits.lance`** is an L2 dataset that records the graph-level commit DAG, with a paired `_graph_commit_actors.lance` for the actor map. (Pre-v0.4.0 graphs also have inert `_graph_runs.lance` / `_graph_run_actors.lance` from the removed Run state machine; MR-770 sweeps these in production.) +- **`_graph_commits.lance`** is an L2 dataset that records the graph-level commit DAG, with a paired `_graph_commit_actors.lance` for the actor map. (Pre-v0.4.0 graphs also have inert `_graph_runs.lance` / `_graph_run_actors.lance` from the removed Run state machine; the v2β†’v3 migration sweeps their stale `__run__*` branches, and the dataset bytes are reclaimed once `delete_prefix` lands.) - **`_graph_commit_recoveries.lance`** β€” one row per recovery sweep action. Joined to `_graph_commits.lance` by `graph_commit_id`; the linked commit row carries `actor_id=omnigraph:recovery`. Operators correlate recoveries with the original mutations they rolled forward / back via this join. See `crates/omnigraph/src/db/recovery_audit.rs`. -- **`__recovery/{ulid}.json`** β€” transient sidecar files written by the four migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`) before Phase B begins, deleted after Phase C succeeds. A sidecar persisting after process exit means the writer crashed in the Phase B β†’ Phase C window; the next `Omnigraph::open` recovery sweep processes it. Steady-state directory is empty. See `crates/omnigraph/src/db/manifest/recovery.rs`. +- **`__recovery/{ulid}.json`** β€” transient sidecar files written by the five migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`, `optimize_all_tables`) before Phase B begins, deleted after Phase C succeeds. A sidecar persisting after process exit means the writer crashed in the Phase B β†’ Phase C window; the next `Omnigraph::open` recovery sweep processes it. Steady-state directory is empty. See `crates/omnigraph/src/db/manifest/recovery.rs`. - **`_refs/branches/{name}.json`** is graph-level branch metadata β€” pointers from a branch name to the manifest version it heads. - **Inside each Lance dataset** (orange): the standard Lance directory layout. `_versions/{n}.manifest` records every commit; `data/` holds the actual Arrow fragments; `_indices/{uuid}/` holds index segments with their own `fragment_bitmap` for partial coverage; `_refs/` holds Lance-native per-dataset branches and tags. diff --git a/openapi.json b/openapi.json index aced64d..335c0bc 100644 --- a/openapi.json +++ b/openapi.json @@ -7,7 +7,7 @@ "name": "MIT", "identifier": "MIT" }, - "version": "0.6.1" + "version": "0.6.2" }, "paths": { "/branches": { diff --git a/scripts/check-agents-md.sh b/scripts/check-agents-md.sh index abc6469..02a177a 100755 --- a/scripts/check-agents-md.sh +++ b/scripts/check-agents-md.sh @@ -34,10 +34,15 @@ PY canonical=() while IFS= read -r line; do canonical+=("$line") -done < <(find docs -type f -name '*.md' ! -path 'docs/releases/*' ! -path 'docs/internal/*' | sort) +done < <(find docs -type f -name '*.md' ! -path 'docs/releases/*' ! -path 'docs/internal/*' ! -path 'docs/rfcs/*' | sort) if [[ -d docs/releases ]]; then canonical+=("docs/releases/") fi +# RFCs are a growing collection (like releases): represent the directory, not +# every per-RFC file. The dir must be linked from an audience index. +if [[ -d docs/rfcs ]]; then + canonical+=("docs/rfcs/") +fi linked=() for index_file in "${index_files[@]}"; do