diff --git a/.github/branch-protection.json b/.github/branch-protection.json index c039e32..aa1ab19 100644 --- a/.github/branch-protection.json +++ b/.github/branch-protection.json @@ -5,7 +5,6 @@ "contexts": [ "Classify Changes", "Check AGENTS.md Links", - "Test Workspace", "Test omnigraph-server --features aws", "CODEOWNERS matches source", "CODEOWNERS not hand-edited" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56ef3e3..fca08da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,6 +128,23 @@ jobs: test: name: Test Workspace needs: classify_changes + # PR latency: the full workspace + failpoints build/test is the slowest + # gate (~15min warm, up to the 75min ceiling cold) and dominated PR + # turnaround. It now runs only on push to `main` (post-merge), on tags, + # and on manual `workflow_dispatch` — NOT on pull_request. Trade-off + # accepted deliberately: a regression is caught on the `main` run after + # merge rather than before it, so `main` can briefly go red. Mitigations: + # (1) `Test Workspace` is removed from required PR checks in + # `.github/branch-protection.json` (a required check that never + # reports would leave every PR permanently pending); + # (2) run the full suite locally before merging risky changes + # (`cargo test --workspace --locked`), or trigger this workflow via + # the Actions "Run workflow" button (workflow_dispatch) on your branch; + # (3) openapi.json is no longer auto-regenerated on PRs (that step lived + # here) — regenerate it locally for server/API changes + # (`OMNIGRAPH_UPDATE_OPENAPI=1 cargo test -p omnigraph-server --test openapi`) + # or the strict drift check fails the post-merge `main` run. + if: github.event_name != 'pull_request' runs-on: ubuntu-latest # 75, not 45: a cold rust-cache (every Cargo.lock change) costs a full # workspace + failpoints-feature build on a 2-core runner, which now @@ -274,6 +291,9 @@ jobs: rustfs_integration: name: RustFS S3 Integration + # `needs: test` means this is push-/dispatch-only too: on pull_request the + # `test` job is skipped, so this dependent is skipped with it. S3 + # integration runs post-merge on `main`, alongside the workspace suite. needs: - classify_changes - test diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 9484b98..4fac941 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -1,6 +1,6 @@ name: Publish to crates.io -# Publishes the four workspace crates to crates.io in dependency order. +# Publishes the publishable workspace crates to crates.io in dependency order. # # Triggers: # - push of any v* tag (future releases auto-publish alongside release.yml) @@ -115,10 +115,14 @@ jobs: # Order matters: each crate must precede anything that depends on it. # omnigraph-compiler and omnigraph-policy have no internal deps; - # omnigraph-engine depends on both; server depends on engine + the - # two leaf crates; cli depends on everything. + # omnigraph-engine depends on both; omnigraph-api-types and + # omnigraph-cluster depend on engine (+ compiler); server depends on + # engine + api-types + cluster + the two leaf crates; cli depends on + # everything. publish_if_new omnigraph-compiler publish_if_new omnigraph-policy publish_if_new omnigraph-engine + publish_if_new omnigraph-api-types + publish_if_new omnigraph-cluster publish_if_new omnigraph-server publish_if_new omnigraph-cli diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a265c40..4b9456a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,17 +1,34 @@ name: Release +# Build per-platform binaries in a matrix, then publish the GitHub release ONCE +# from a single job. The matrix used to call `softprops/action-gh-release` +# concurrently — three jobs racing to create/finalize the same release, which +# exhausted the action's finalize retries and dropped whole platforms' assets. +# The matrix now only uploads workflow artifacts; `publish_release` is the sole +# writer of the release (no race). +# +# Triggers: +# - push of a v* tag (normal release) +# - workflow_dispatch with an explicit `tag` (re-publish a past tag without +# re-cutting it; resolves the same `${{ inputs.tag || github.ref_name }}`) + on: push: tags: - "v*" workflow_dispatch: + inputs: + tag: + description: "Tag to (re)publish (e.g. v0.7.0). Required for manual dispatches." + required: true + type: string jobs: build_release: name: Build ${{ matrix.asset_name }} runs-on: ${{ matrix.runner }} permissions: - contents: write + contents: read strategy: fail-fast: false matrix: @@ -27,6 +44,8 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v5.0.1 + with: + ref: ${{ inputs.tag || github.ref_name }} - name: Install Linux dependencies if: runner.os == 'Linux' @@ -81,20 +100,46 @@ jobs: throw "Windows release archive is missing expected binaries" } - - name: Publish GitHub release assets + # Upload artifacts only — the single `publish_release` job attaches them to + # the release, so no two jobs ever write the release concurrently. + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.asset_name }} + path: | + ${{ matrix.asset_name }}.* + if-no-files-found: error + retention-days: 1 + + publish_release: + name: Publish GitHub release + needs: build_release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download all build artifacts + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Publish release (single writer — no matrix race) uses: softprops/action-gh-release@v2.5.0 with: - files: | - ${{ matrix.asset_name }}.* + tag_name: ${{ inputs.tag || github.ref_name }} + files: dist/** + overwrite_files: true update_homebrew_tap: name: Update Homebrew tap - needs: build_release + needs: publish_release runs-on: ubuntu-latest permissions: contents: read env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} steps: - name: Skip if HOMEBREW_TAP_TOKEN is not configured if: env.HOMEBREW_TAP_TOKEN == '' @@ -105,6 +150,8 @@ jobs: - name: Checkout source if: env.HOMEBREW_TAP_SKIP != '1' uses: actions/checkout@v5.0.1 + with: + ref: ${{ env.RELEASE_TAG }} - name: Checkout Homebrew tap if: env.HOMEBREW_TAP_SKIP != '1' @@ -119,7 +166,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - ./scripts/update-homebrew-formula.sh "${GITHUB_REF_NAME}" homebrew-tap/Formula/omnigraph.rb + ./scripts/update-homebrew-formula.sh "${RELEASE_TAG}" homebrew-tap/Formula/omnigraph.rb # Diagnostic only: brew is not on PATH on the ubuntu runner by default, so # set it up explicitly. Both this setup and the audit below are best-effort @@ -158,22 +205,26 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add Formula/omnigraph.rb - git commit -m "Update Omnigraph formula to ${GITHUB_REF_NAME}" + git commit -m "Update Omnigraph formula to ${RELEASE_TAG}" git push origin HEAD:main smoke_windows_installer: name: Smoke Windows installer - needs: build_release - if: startsWith(github.ref, 'refs/tags/v') + needs: publish_release + if: ${{ inputs.tag != '' || startsWith(github.ref, 'refs/tags/v') }} runs-on: windows-latest permissions: contents: read + env: + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} steps: - name: Checkout source uses: actions/checkout@v5.0.1 + with: + ref: ${{ env.RELEASE_TAG }} - name: Install from tagged release - run: ./scripts/install.ps1 -Version "$env:GITHUB_REF_NAME" -InstallDir "$env:RUNNER_TEMP/omnigraph-bin" + run: ./scripts/install.ps1 -Version "$env:RELEASE_TAG" -InstallDir "$env:RUNNER_TEMP/omnigraph-bin" - name: Smoke installed binaries run: | diff --git a/AGENTS.md b/AGENTS.md index d9e0c45..378de88 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,8 +17,8 @@ Tools that support `@`-imports (Claude Code) auto-include all three files via th `CLAUDE.md` is a symlink to this file — there is exactly one source of truth. Edit `AGENTS.md`. **Version surveyed:** 0.7.0 -**Workspace crates:** `omnigraph-compiler`, `omnigraph` (engine), `omnigraph-policy`, `omnigraph-cluster`, `omnigraph-cli`, `omnigraph-server` -**Storage substrate:** Lance 6.x (columnar, versioned, branchable) +**Workspace crates:** `omnigraph-compiler`, `omnigraph` (engine), `omnigraph-policy`, `omnigraph-api-types` (shared HTTP wire DTOs), `omnigraph-cluster`, `omnigraph-cli`, `omnigraph-server` +**Storage substrate:** Lance 7.x (columnar, versioned, branchable) **License:** MIT **Toolchain:** Rust stable, edition 2024 @@ -33,8 +33,8 @@ OmniGraph is a typed property-graph engine built as a coordination layer over ma - **Multi-modal querying**: vector ANN (`nearest`), full-text (`search`/`fuzzy`/`match_text`/`bm25`), Reciprocal Rank Fusion (`rrf`), and graph traversal (`Expand`, anti-join `not { … }`) in one runtime. - **Branches and commits across the whole graph**: Git-style — every successful publish appends to a commit DAG; merges are three-way at the row level. - **Atomic per-query writes**: `mutate_as` and `load` accumulate insert/update batches into an in-memory `MutationStaging.pending` per touched table; one `stage_*` + `commit_staged` per table runs at end-of-query, then `ManifestBatchPublisher::publish` commits the manifest atomically with per-table `expected_table_versions` CAS. A mid-query failure leaves Lance HEAD untouched on staged tables — no drift, no run state machine, no staging branches. Deletes still inline-commit; D₂ at parse time prevents inserts/updates and deletes from coexisting in one query. -- **HTTP server**: Axum + utoipa OpenAPI, bearer auth (SHA-256 hashed, optional AWS Secrets Manager). Cedar policy enforcement is engine-wide — every `_as` writer calls `Omnigraph::enforce(action, scope, actor)`, so HTTP, CLI, and embedded SDK consumers all hit the same gate. **Two modes** (v0.6.0+): single-graph (legacy flat routes) and multi-graph (`/graphs/{graph_id}/...` cluster routes + read-only `GET /graphs` enumeration). Per-graph + server-level Cedar policies. Multi-graph mode boots from a cluster directory (`--cluster `, RFC-005) or the legacy `omnigraph.yaml` `graphs:` map. Runtime add/remove (`POST /graphs`, `DELETE /graphs/{id}`) is not exposed — operators run `cluster apply` (or edit the legacy file) and restart. -- **CLI** with two-surface config (RFC-008): the team-owned cluster directory (`cluster.yaml`) plus the per-operator `~/.omnigraph/config.yaml` (servers, credentials, actor, aliases). The legacy combined `omnigraph.yaml` still loads with per-key deprecation warnings — `config migrate` proposes the split, `OMNIGRAPH_NO_LEGACY_CONFIG=1` enforces strict mode. **Never extend `omnigraph.yaml`.** Multi-format output (json/jsonl/csv/kv/table). +- **HTTP server**: Axum + utoipa OpenAPI, bearer auth (SHA-256 hashed, optional AWS Secrets Manager). Cedar policy enforcement is engine-wide — every `_as` writer calls `Omnigraph::enforce(action, scope, actor)`, so HTTP, CLI, and embedded SDK consumers all hit the same gate. **Cluster-only boot** (RFC-011): the server always boots from a cluster directory (`--cluster `, RFC-005) and serves N graphs (N ≥ 1) under multi-graph routes (`/graphs/{graph_id}/...` + read-only `GET /graphs` enumeration); there are no single-graph flat routes and no positional-URI boot. Per-graph + server-level Cedar policies. Runtime add/remove (`POST /graphs`, `DELETE /graphs/{id}`) is not exposed — operators run `cluster apply` and restart. +- **CLI** with two-surface config (RFC-007/008): the team-owned cluster directory (`cluster.yaml`) plus the per-operator `~/.omnigraph/config.yaml` (servers, clusters, credentials, actor, profiles, aliases, defaults). Graphs are addressed via `--store`/`--server`/`--cluster`/`--profile`/operator defaults (RFC-011). Multi-format output (json/jsonl/csv/kv/table). Throughout the docs, capabilities are split into **L1 — Inherited from Lance** vs **L2 — Added by OmniGraph**. @@ -53,7 +53,7 @@ CLI (omnigraph) HTTP Server (omnigraph-server, Axum) omnigraph (engine) ── ManifestCoordinator, CommitGraph, RunRegistry, GraphIndex (CSR/CSC), exec │ ▼ - Lance 6.x ── columnar Arrow, fragments, per-dataset versions/branches, indexes + Lance 7.x ── columnar Arrow, fragments, per-dataset versions/branches, indexes │ ▼ Object store (file / s3 / RustFS / MinIO / S3-compat) @@ -73,32 +73,38 @@ Full diagram and concurrency model: [docs/dev/architecture.md](docs/dev/architec | **Lance docs index — fetch upstream Lance docs by problem domain** | **[docs/dev/lance.md](docs/dev/lance.md)** | | **Test coverage map — what's covered, what helpers to reuse, before-every-task checklist** | **[docs/dev/testing.md](docs/dev/testing.md)** | | Architecture, L1/L2 framing, concurrency model | [docs/dev/architecture.md](docs/dev/architecture.md) | -| Storage layout, `__manifest` schema, URI schemes, S3 env vars | [docs/user/storage.md](docs/user/storage.md) | -| `.pg` schema language, types, constraints, annotations, migration planning | [docs/user/schema-language.md](docs/user/schema-language.md) | -| Schema-lint codes (`OG-XXX-NNN`), families, severity, suppression | [docs/user/schema-lint.md](docs/user/schema-lint.md) | -| `.gq` query language, MATCH/RETURN/ORDER, search funcs, mutations, IR ops, lint codes | [docs/user/query-language.md](docs/user/query-language.md) | -| Indexes (BTREE / inverted / vector / graph topology) | [docs/user/indexes.md](docs/user/indexes.md) | -| Embeddings (compiler + engine clients, env vars, `@embed`) | [docs/user/embeddings.md](docs/user/embeddings.md) | -| Branches, commit graph, snapshots, system branches | [docs/user/branches-commits.md](docs/user/branches-commits.md) | -| Transactions and atomicity (per-query atomic; branches as multi-query transactions) | [docs/user/transactions.md](docs/user/transactions.md) | +| Storage layout, `__manifest` schema, URI schemes, S3 env vars | [docs/user/concepts/storage.md](docs/user/concepts/storage.md) | +| `.pg` schema language, types, constraints, annotations, migration planning | [docs/user/schema/index.md](docs/user/schema/index.md) | +| Schema-lint codes (`OG-XXX-NNN`), families, severity, suppression | [docs/user/schema/lint.md](docs/user/schema/lint.md) | +| `.gq` query language, MATCH/RETURN/ORDER, IR ops, lint codes | [docs/user/queries/index.md](docs/user/queries/index.md) | +| Mutations — insert/update/delete, D2, atomicity | [docs/user/mutations/index.md](docs/user/mutations/index.md) | +| Search funcs (`nearest`/`bm25`/`rrf`), hybrid ranking | [docs/user/search/index.md](docs/user/search/index.md) | +| Indexes (BTREE / inverted / vector / graph topology) | [docs/user/search/indexes.md](docs/user/search/indexes.md) | +| Embeddings (engine client, env vars, `@embed`) | [docs/user/search/embeddings.md](docs/user/search/embeddings.md) | +| Concepts — what OmniGraph is, L1/L2 framing | [docs/user/concepts/index.md](docs/user/concepts/index.md) | +| Quickstart — init → load → query → branch | [docs/user/quickstart.md](docs/user/quickstart.md) | +| Branches, commit graph, system branches | [docs/user/branching/index.md](docs/user/branching/index.md) | +| Snapshots & time travel | [docs/user/branching/time-travel.md](docs/user/branching/time-travel.md) | +| Three-way merge and conflict kinds (user-facing) | [docs/user/branching/merge.md](docs/user/branching/merge.md) | +| Transactions and atomicity (per-query atomic; branches as multi-query transactions) | [docs/user/branching/transactions.md](docs/user/branching/transactions.md) | | Direct-publish write path (staging, D2, recovery sidecars; the former Run state machine) | [docs/dev/writes.md](docs/dev/writes.md) | | Three-way merge and conflict kinds | [docs/dev/merge.md](docs/dev/merge.md) | -| Diff / change feed (`diff_between`, `diff_commits`) | [docs/user/changes.md](docs/user/changes.md) | +| Diff / change feed (`diff_between`, `diff_commits`) | [docs/user/branching/changes.md](docs/user/branching/changes.md) | | Query execution, mutation execution, bulk loader, `load` vs `ingest` | [docs/dev/execution.md](docs/dev/execution.md) | -| `optimize` (compaction) and `cleanup` (version GC) | [docs/user/maintenance.md](docs/user/maintenance.md) | -| Cluster operator guide (deploy/manage clusters, approvals, recovery, serving) | [docs/user/cluster.md](docs/user/cluster.md) | -| Cedar policy actions, scopes, CLI | [docs/user/policy.md](docs/user/policy.md) | -| HTTP server endpoints, auth, error model, body limits | [docs/user/server.md](docs/user/server.md) | -| CLI quick-start | [docs/user/cli.md](docs/user/cli.md) | -| CLI command surface and config schemas (`~/.omnigraph/config.yaml`, legacy `omnigraph.yaml`) | [docs/user/cli-reference.md](docs/user/cli-reference.md) | -| Audit / actor tracking | [docs/user/audit.md](docs/user/audit.md) | -| Error taxonomy and result serialization | [docs/user/errors.md](docs/user/errors.md) | +| `optimize` (compaction) and `cleanup` (version GC) | [docs/user/operations/maintenance.md](docs/user/operations/maintenance.md) | +| Cluster operator guide (deploy/manage clusters, approvals, recovery, serving) | [docs/user/clusters/index.md](docs/user/clusters/index.md) | +| Cedar policy actions, scopes, CLI | [docs/user/operations/policy.md](docs/user/operations/policy.md) | +| HTTP server endpoints, auth, error model, body limits | [docs/user/operations/server.md](docs/user/operations/server.md) | +| CLI quick-start | [docs/user/cli/index.md](docs/user/cli/index.md) | +| CLI command surface and config schema (`~/.omnigraph/config.yaml`) | [docs/user/cli/reference.md](docs/user/cli/reference.md) | +| Audit / actor tracking | [docs/user/operations/audit.md](docs/user/operations/audit.md) | +| Error taxonomy and result serialization | [docs/user/operations/errors.md](docs/user/operations/errors.md) | | Install (binary / Homebrew / source / channels) | [docs/user/install.md](docs/user/install.md) | -| Deployment (binary / container / RustFS bootstrap / auth / build variants) | [docs/user/deployment.md](docs/user/deployment.md) | +| Deployment (binary / container / S3-local testing / auth / build variants) | [docs/user/deployment.md](docs/user/deployment.md) | | CI / release workflows | [docs/dev/ci.md](docs/dev/ci.md) | | Code ownership (CODEOWNERS source of truth, roles, regeneration) | [docs/dev/codeowners.md](docs/dev/codeowners.md) | | Branch protection policy (declarative, applied via `scripts/apply-branch-protection.sh`) | [docs/dev/branch-protection.md](docs/dev/branch-protection.md) | -| Constants & tunables cheat sheet | [docs/user/constants.md](docs/user/constants.md) | +| Constants & tunables cheat sheet | [docs/user/reference/constants.md](docs/user/reference/constants.md) | | Per-version release notes | [docs/releases/](docs/releases/) | --- @@ -138,6 +144,7 @@ These are architectural rules that need to be in scope on every change. They're 4. **Bearer-token plaintext never persists in process memory.** Tokens are hashed at startup; auth uses constant-time comparison; the actor id is server-resolved from the hash match and must not be settable by the client. 5. **Reads always see the current index state for the branch they're reading.** Indexes track the branch head, not historical snapshots. If you change index lifecycle, preserve this guarantee. 6. **Stable type IDs survive renames.** Schema migration relies on identity that's stable across rename — don't mint new IDs on rename. +7. **Logical contract over physical state.** Physical state (index coverage, fragment layout, compaction versions, staged writes) is derived and rebuildable; it must never fail a logical operation. Check preconditions against logical state and let reconciliation converge the physical state idempotently — genuine logical conflicts still fail loudly. This is the rule rules 1–6 instantiate; full statement and applications in [docs/dev/invariants.md](docs/dev/invariants.md). ### Deny-list (fast-pass review filter — full reasoning in [docs/dev/invariants.md](docs/dev/invariants.md)) @@ -173,7 +180,7 @@ Rust stable workspace (edition 2024). `protoc` is a build dependency (`brew inst cargo build --workspace --locked # build everything cargo test --workspace --locked # the canonical CI gate (matches CI exactly) cargo run -p omnigraph-cli -- # run the `omnigraph` CLI from source -cargo run -p omnigraph-server -- --bind 0.0.0.0:8080 # run the server from source +cargo run -p omnigraph-server -- --cluster --bind 0.0.0.0:8080 # run the server from source # Run one crate / one test file / one test fn cargo test -p omnigraph-engine --test traversal # one integration-test file (see docs/dev/testing.md) @@ -185,7 +192,7 @@ cargo test -p omnigraph-engine --features failpoints --test failpoints # fault cargo build -p omnigraph-server --features aws # AWS Secrets Manager bearer-token source ``` -S3-backed tests (`s3_storage`, and the S3 paths in server/CLI system tests) **skip** unless `OMNIGRAPH_S3_TEST_BUCKET` + `AWS_*` (incl. `AWS_ENDPOINT_URL_S3` for non-AWS) are set; CI runs them against containerized RustFS. `scripts/local-rustfs-bootstrap.sh` stands up a local S3 environment. +S3-backed tests (`s3_storage`, and the S3 paths in server/CLI system tests) **skip** unless `OMNIGRAPH_S3_TEST_BUCKET` + `AWS_*` (incl. `AWS_ENDPOINT_URL_S3` for non-AWS) are set; CI runs them against containerized RustFS. To run RustFS/MinIO yourself, see [docs/user/deployment.md](docs/user/deployment.md) → *Testing against S3 locally*. CI does **not** run `clippy` or `rustfmt` as gates — but `cargo test --workspace --locked` is the exact gate, so run it before pushing. Two non-test CI checks: `scripts/check-agents-md.sh` (doc cross-link integrity — run it after moving/renaming docs) and OpenAPI drift (`crates/omnigraph-server/tests/openapi.rs` regenerates `openapi.json`; set `OMNIGRAPH_UPDATE_OPENAPI=1` to update the checked-in copy when a server/API change is intentional). @@ -203,9 +210,9 @@ omnigraph load --data ./seed.jsonl --mode overwrite s3://my-bucket/graph.omni # Load a review batch onto its own branch (--from forks it if missing) omnigraph load --branch review/2026-04-25 --from main --mode merge --data ./batch.jsonl s3://my-bucket/graph.omni -# Run a hybrid (vector + BM25) query -omnigraph read --query ./queries.gq --name find_similar \ - --params '{"q":"trends in AI safety"}' --format table s3://my-bucket/graph.omni +# Run a hybrid (vector + BM25) query — ad-hoc .gq against a store (positional = query name) +omnigraph query --query ./queries.gq find_similar \ + --params '{"q":"trends in AI safety"}' --format table --store s3://my-bucket/graph.omni # Plan + apply schema migration omnigraph schema plan --schema ./next.pg s3://my-bucket/graph.omni @@ -225,10 +232,10 @@ omnigraph cleanup --keep 10 --older-than 7d --confirm s3://my-bucket/graph.omni # Stand up the HTTP server (token from env) OMNIGRAPH_SERVER_BEARER_TOKEN=xxxx \ - omnigraph-server s3://my-bucket/graph.omni --bind 0.0.0.0:8080 + omnigraph-server --cluster s3://my-bucket/cluster --bind 0.0.0.0:8080 # Cedar policy explain -omnigraph policy explain --actor act-alice --action change --branch main +omnigraph policy explain --cluster ./company-brain --graph knowledge --actor act-alice --action change --branch main ``` --- @@ -241,10 +248,10 @@ omnigraph policy explain --actor act-alice --action change --branch main | Per-dataset versioning + time travel | ✅ | `snapshot_at_version`, `entity_at`, snapshot-pinned reads across many tables | | Per-dataset branches | ✅ | **Graph-level** branches (atomic across all sub-tables), lazy fork, system branch filtering | | Atomic single-dataset commits | ✅ | **Multi-table publish via three layers**, NOT a single Lance primitive: (1) per-table Lance `commit_staged` for the data write, (2) `__manifest` row-level CAS via `ManifestBatchPublisher` for cross-table ordering, (3) the open-time recovery sweep for the residual gap between (1) and (2). All three layers ship; the five migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`, `optimize_all_tables`) write a `__recovery/{ulid}.json` sidecar before Phase B and delete it after Phase C. The next `Omnigraph::open` (gated on `OpenMode::ReadWrite`) runs the sweep in `db/manifest/recovery.rs`: classify, decide all-or-nothing per sidecar, roll forward via single `ManifestBatchPublisher::publish` or roll back via `Dataset::restore` followed by a manifest publish of the restored version (so both directions converge to `manifest == HEAD` — no residual drift), and record an audit row in `_graph_commit_recoveries.lance` (queryable via `omnigraph commit list --filter actor=omnigraph:recovery`). The write entry points (`load_as`, `mutate_as`, `apply_schema_as`, `branch_merge_as`) and `refresh` additionally run an in-process roll-forward-only heal (serialized against live writers via the per-table write queues), so a long-lived server converges on its next write without restart; only rollback-eligible sidecars still defer to the next read-write open (a future background reconciler's goal). Engine writes route through a sealed `TableStorage` trait (`db.storage()`) exposing only `stage_*` + `commit_staged` + reads; the inline-commit residuals (`delete_where`, `create_vector_index`) are split onto a separate sealed `InlineCommitResidual` trait reached via `db.storage_inline_residual()` (MR-854), so the default surface cannot couple a write with a HEAD advance — §1 holds by construction. `delete_where` and `create_vector_index` stay inline until upstream Lance ships a public two-phase API ([#6658](https://github.com/lance-format/lance/issues/6658), [#6666](https://github.com/lance-format/lance/issues/6666)); `LoadMode::Overwrite` uses Lance `Overwrite` staged transactions. | -| Compaction (`compact_files`) | ✅ | `omnigraph optimize` orchestrates over all node/edge tables, bounded concurrency; **publishes each compacted table's new version to `__manifest`** (so the manifest tracks the Lance HEAD — required for reads to observe compaction and for schema apply / strict writes to pass their HEAD-vs-manifest precondition), under the per-`(table, main)` write queue with `SidecarKind::Optimize` recovery coverage; **refuses on an unrecovered graph** (errors if a `__recovery` sidecar is pending); **skips uncovered HEAD > manifest drift** with `DriftNeedsRepair` instead of interpreting it; **skips blob-bearing tables** (reported via `TableOptimizeStats.skipped`, not silent), gated on `LANCE_SUPPORTS_BLOB_COMPACTION` until the upstream blob-v2 compaction-decode bug is fixed (see [docs/dev/invariants.md](docs/dev/invariants.md) Known Gaps) | +| Compaction (`compact_files`) + reindex (`optimize_indices`) | ✅ | `omnigraph optimize` orchestrates over all node/edge tables, bounded concurrency; per table runs `compact_files` **then Lance `optimize_indices`** (folds appended/rewritten fragments back into existing indexes — incremental merge, not retrain) and **publishes the resulting version to `__manifest`** (so the manifest tracks the Lance HEAD — required for reads to observe the work and for schema apply / strict writes to pass their HEAD-vs-manifest precondition), under the per-`(table, main)` write queue with `SidecarKind::Optimize` recovery coverage spanning both ops; **commits even with no compaction work if index coverage is stale**; **refuses on an unrecovered graph**; **skips uncovered HEAD > manifest drift** with `DriftNeedsRepair`; **skips blob-bearing tables** (reported via `TableOptimizeStats.skipped`, not silent; reindex is skipped for them too today), gated on `LANCE_SUPPORTS_BLOB_COMPACTION` until the upstream blob-v2 compaction-decode bug is fixed (see [docs/dev/invariants.md](docs/dev/invariants.md) Known Gaps) | | Repair uncovered drift | — | `omnigraph repair` explicitly classifies uncovered table `HEAD > manifest` drift: verified maintenance drift (`ReserveFragments`/`Rewrite`) can be published with `--confirm`; suspicious or unverifiable drift requires `--force --confirm`. Sidecar-covered crash residuals still recover automatically on open. | | Cleanup (`cleanup_old_versions`) | ✅ | `omnigraph cleanup` with `--keep` / `--older-than` policy | -| BTREE / inverted (FTS) / vector indexes | ✅ | `ensure_indices` builds them on every relevant column; idempotent; lazy across branches | +| BTREE / inverted (FTS) / vector indexes | ✅ | `@index`/`@key` declares intent; the physical index is derived state that never fails a logical op. Built per column through one chokepoint (`build_indices_on_dataset_for_catalog`, type-dispatched by `node_prop_index_kind`: enum + orderable scalar → BTREE, free-text String → FTS, Vector → vector); idempotent; lazy across branches. **Schema apply builds nothing** (records intent only); `load`/`mutate` build inline but **defer an untrainable Vector column** (no trainable vectors yet) as *pending* rather than aborting. `ensure_indices`/`optimize` is the reconciler that materializes declared-but-missing indexes and restores coverage of appended/rewritten fragments (`optimize_indices`), reporting still-pending columns (see Compaction row). | | `merge_insert` upsert | ✅ | `LoadMode::Merge`, mutation `update`/`insert`/`delete` lowering | | Vector search | ✅ | `nearest()` query op; embedding pipeline (Gemini / OpenAI clients); `@embed` in schema | | Full-text search | ✅ | `search/fuzzy/match_text/bm25` query ops | @@ -257,11 +264,12 @@ omnigraph policy explain --actor act-alice --action change --branch main | Per-query atomic writes | — | In-memory `MutationStaging.pending` accumulator + `stage_*` / `commit_staged` per touched table at end-of-query + publisher CAS via `commit_with_expected` (single manifest commit per `mutate_as` / `load`); D₂ parse-time rule keeps inserts/updates and deletes from mixing | | Three-way row-level merge | — | `OrderedTableCursor` + `StagedTableWriter`, structured `MergeConflictKind` | | Change feeds | — | `diff_between` / `diff_commits` with manifest fast path + ID streaming | -| Cedar policy | — | Per-graph actions plus server-scoped actions (see [docs/user/policy.md](docs/user/policy.md) for the current list), branch / target_branch / protected scopes, validate/test/explain CLI. **Engine-wide enforcement** (MR-722): every `_as` writer (`apply_schema_as`, `mutate_as`, `load_as` — the deprecated `ingest_as` shims route through it — `branch_create_as` / `branch_create_from_as`, `branch_delete_as`, `branch_merge_as`) calls `Omnigraph::enforce(action, scope, actor)` — HTTP, CLI, embedded SDK all hit the same gate. | -| HTTP server | — | Axum, OpenAPI via utoipa, bearer auth (SHA-256, AWS Secrets Manager option), `authorize_request` at the HTTP boundary (resolves bearer→actor, applies admission control), NDJSON streaming export, **multi-graph mode (v0.6.0+) with cluster routes + read-only `GET /graphs` enumeration + per-graph + server-level Cedar policies. Multi-graph boots from a cluster directory (`--cluster`) or the legacy `omnigraph.yaml`; add/remove graphs via `cluster apply` (or by editing the legacy file) and restarting.** | -| CLI with config | — | two-surface config (team `cluster.yaml` dir + per-operator `~/.omnigraph/config.yaml`; legacy `omnigraph.yaml` deprecated per RFC-008), aliases, multi-format output (json/jsonl/csv/kv/table) | +| Cedar policy | — | Per-graph actions plus server-scoped actions (see [docs/user/operations/policy.md](docs/user/operations/policy.md) for the current list), branch / target_branch / protected scopes, validate/test/explain CLI. **Engine-wide enforcement** (MR-722): every `_as` writer (`apply_schema_as`, `mutate_as`, `load_as` — the deprecated `ingest_as` shims route through it — `branch_create_as` / `branch_create_from_as`, `branch_delete_as`, `branch_merge_as`) calls `Omnigraph::enforce(action, scope, actor)` — HTTP, CLI, embedded SDK all hit the same gate. | +| HTTP server | — | Axum, OpenAPI via utoipa, bearer auth (SHA-256, AWS Secrets Manager option), `authorize_request` at the HTTP boundary (resolves bearer→actor, applies admission control), NDJSON streaming export, **cluster-only boot (RFC-011): always `--cluster `, serving N graphs (N ≥ 1) under multi-graph routes + read-only `GET /graphs` enumeration + per-graph + server-level Cedar policies. Add/remove graphs via `cluster apply` and restart.** | +| CLI with config | — | two-surface config (team `cluster.yaml` dir + per-operator `~/.omnigraph/config.yaml`), scope addressing (`--store`/`--server`/`--cluster`/`--profile`/defaults, RFC-011), aliases, multi-format output (json/jsonl/csv/kv/table) | | Audit / actor tracking | — | `_as` write APIs + actor map in commit graph | -| Local RustFS bootstrap | — | `scripts/local-rustfs-bootstrap.sh` one-shot S3-backed dev environment | +| Local S3 testing | — | run RustFS/MinIO + the `AWS_*` env; see [docs/user/deployment.md](docs/user/deployment.md) → *Testing against S3 locally* | +| Agent skill | — | `skills/omnigraph` — operational playbook for driving Omnigraph; install with `npx skills add ModernRelay/omnigraph@omnigraph` | --- @@ -282,7 +290,7 @@ Rules: 7. **Re-verify before recommending.** If you cite a flag, env var, endpoint, or constant to the user or in code, grep for it in source first. Memory and docs go stale; the code is authoritative. 8. **Keep AGENTS.md short.** This file is always loaded into agent context, so every added line has a recurring context-window cost. Prefer pointers and terse invariants here; put detail in `docs/`. 9. **Keep AGENTS.md a map, not an encyclopedia.** New deep content goes into `docs/`. Add an entry to "Where to find each topic" instead of pasting prose into this file. The "Always-on rules" section is the exception — it's for invariants that should always be in scope. -10. **Re-read on schema/query/IR changes.** Edits to `schema.pest`, `query.pest`, `ir/lower.rs`, `query/typecheck.rs`, or `query/lint.rs` should trigger a re-read of [docs/user/schema-language.md](docs/user/schema-language.md), [docs/user/query-language.md](docs/user/query-language.md), and [docs/dev/execution.md](docs/dev/execution.md) to confirm they still describe reality. +10. **Re-read on schema/query/IR changes.** Edits to `schema.pest`, `query.pest`, `ir/lower.rs`, `query/typecheck.rs`, or `query/lint.rs` should trigger a re-read of [docs/user/schema/index.md](docs/user/schema/index.md), [docs/user/queries/index.md](docs/user/queries/index.md), and [docs/dev/execution.md](docs/dev/execution.md) to confirm they still describe reality. 11. **Always make smaller commits.** Each commit does one thing, compiles, and passes tests; mechanical refactors land separately from the behavior changes they enable. 12. **Test-first for bug fixes.** When fixing an identified bug, write a regression test that reproduces the failure first. Confirm it fails against the current code with the predicted symptom (not an unrelated error). Then land the fix in a separate commit and confirm the test turns green. The test commit lands just before the fix commit so the red → green pair is visible in `git log` and a reviewer can check out the test commit alone and reproduce the failure. 13. **Correct by design over symptomatic patches.** When a bug surfaces, identify the root cause and make the fix correct by construction. Don't patch the symptom. If the design admits the bug class, the fix is to close the class, not to add a guard around the latest instance. A symptomatic patch is acceptable only as a stop-gap, with an explicit note in the commit message and a follow-up issue tracking the design fix. diff --git a/Cargo.lock b/Cargo.lock index 2099055..2419e9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,9 +23,9 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -34,7 +34,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "const-random", "getrandom 0.3.4", "once_cell", @@ -83,9 +83,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -104,9 +104,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -137,6 +137,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -708,11 +717,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -980,7 +989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 1.0.4", "libc", "miniz_oxide", "object", @@ -1071,7 +1080,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1083,9 +1092,9 @@ dependencies = [ "arrayref", "arrayvec 0.7.6", "cc", - "cfg-if", + "cfg-if 1.0.4", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -1097,6 +1106,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1266,6 +1284,12 @@ dependencies = [ "smol_str", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1278,6 +1302,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1308,15 +1343,15 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] [[package]] name = "clap" -version = "4.5.58" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1324,9 +1359,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.58" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1336,9 +1371,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -1361,6 +1396,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "color-eyre" version = "0.6.5" @@ -1394,6 +1435,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1436,6 +1496,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1456,12 +1522,37 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1478,6 +1569,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "countio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" +dependencies = [ + "futures-io", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1487,6 +1587,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -1502,7 +1611,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -1574,6 +1683,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -1595,6 +1713,31 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctor" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" +dependencies = [ + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" version = "0.23.0" @@ -1635,7 +1778,7 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -1681,7 +1824,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "rand 0.9.2", "regex", @@ -1712,7 +1855,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "tokio", ] @@ -1737,7 +1880,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", ] [[package]] @@ -1756,7 +1899,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "object_store 0.13.2", + "object_store", "paste", "sqlparser", "tokio", @@ -1797,7 +1940,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", "rand 0.9.2", "tokio", "url", @@ -1823,7 +1966,7 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store 0.13.2", + "object_store", "tokio", ] @@ -1845,7 +1988,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store 0.13.2", + "object_store", "regex", "tokio", ] @@ -1868,7 +2011,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store 0.13.2", + "object_store", "serde_json", "tokio", "tokio-stream", @@ -1896,7 +2039,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "rand 0.9.2", "tempfile", @@ -1965,7 +2108,7 @@ dependencies = [ "num-traits", "rand 0.9.2", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -2284,7 +2427,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -2311,12 +2454,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "6.0.0" @@ -2358,6 +2513,21 @@ dependencies = [ "const-random", ] +[[package]] +name = "dtor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + [[package]] name = "dunce" version = "1.0.5" @@ -2403,7 +2573,7 @@ version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -2578,9 +2748,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83cf860f6a6bf0a6a60fdfe5a36c75121fad5ea4332d1d12deee3e65b6047727" +checksum = "bcd0ce0249ac12fd44fcde62d435c36d881952c2f0df4d1de24b45e1dbba5ddb" dependencies = [ "arrow-array", "rand 0.9.2", @@ -2689,6 +2859,15 @@ dependencies = [ "slab", ] +[[package]] +name = "gearhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" +dependencies = [ + "cfg-if 0.1.10", +] + [[package]] name = "generator" version = "0.8.8" @@ -2696,7 +2875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "libc", "log", "rustversion", @@ -2720,10 +2899,10 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2733,7 +2912,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "r-efi 5.3.0", @@ -2747,11 +2926,14 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -2760,6 +2942,26 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "glob" version = "0.3.3" @@ -2822,7 +3024,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crunchy", "num-traits", "zerocopy", @@ -2866,6 +3068,12 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "heapify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + [[package]] name = "heck" version = "0.5.0" @@ -2884,22 +3092,44 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-xet" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "more-asserts", + "serde", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", +] + [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] -name = "home" -version = "0.5.12" +name = "hmac" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "windows-sys 0.61.2", + "digest 0.11.3", ] [[package]] @@ -2975,6 +3205,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -3073,9 +3312,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -3271,7 +3512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "libc", ] @@ -3329,10 +3570,12 @@ checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-sys 0.61.2", ] @@ -3362,6 +3605,55 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if 1.0.4", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -3402,30 +3694,32 @@ dependencies = [ "serde_json", ] -[[package]] -name = "jsonwebtoken" -version = "9.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" -dependencies = [ - "base64", - "js-sys", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - [[package]] name = "keccak" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] +[[package]] +name = "konst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" +dependencies = [ + "const_panic", + "konst_proc_macros", + "typewit", +] + +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + [[package]] name = "lalrpop" version = "0.22.2" @@ -3460,10 +3754,11 @@ dependencies = [ [[package]] name = "lance" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34e854994e84d043897f5ec9fb609221e9e69e3fd52996cd715d979fcd349f6" +checksum = "3944aca86f4c78f4da04af1c2bf33e664a2826b7af72972ad200d6b9de59019f" dependencies = [ + "arc-swap", "arrow", "arrow-arith", "arrow-array", @@ -3478,9 +3773,11 @@ dependencies = [ "async-trait", "async_cell", "aws-credential-types", + "bitpacking", "byteorder", "bytes", "chrono", + "crossbeam-queue", "crossbeam-skiplist", "dashmap", "datafusion", @@ -3507,13 +3804,14 @@ dependencies = [ "lance-tokenizer", "log", "moka", - "object_store 0.12.5", + "object_store", "permutation", "pin-project", "prost", "prost-build", "prost-types", "rand 0.9.2", + "rayon", "roaring", "semver", "serde", @@ -3529,13 +3827,12 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7827fe404358c27d120ee8ea8ef7b9415c2911d54072bec83dd689d750ae65da" +checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc" dependencies = [ "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-ord", @@ -3552,9 +3849,9 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd0b31570d50fe13c7e4e36b03e1f1c99c3d8e5a34845b24b0665b51b40570d" +checksum = "80c4d12521b1945041dd515a56aa0854973138e7ac12111c92572e33e4ecb593" dependencies = [ "arrayref", "paste", @@ -3563,9 +3860,9 @@ dependencies = [ [[package]] name = "lance-core" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b128c213c676cb8e03c62a68670642770825171e64097cc2da97cbb19fe35d29" +checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" dependencies = [ "arrow-array", "arrow-buffer", @@ -3573,7 +3870,6 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "chrono", "datafusion-common", "datafusion-sql", "deepsize", @@ -3582,10 +3878,9 @@ dependencies = [ "lance-arrow", "libc", "log", - "mock_instant", "moka", "num_cpus", - "object_store 0.12.5", + "object_store", "pin-project", "prost", "rand 0.9.2", @@ -3602,9 +3897,9 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e03b2de71cbcd09b10bf1a17c83cacbc0176ecd97203fb72b9e59d9b8f9a3743" +checksum = "7460597a66534a75987993d4dac5bc330586d99c5b79ae73367dbcbd4e29e576" dependencies = [ "arrow", "arrow-array", @@ -3628,16 +3923,15 @@ dependencies = [ "pin-project", "prost", "prost-build", - "snafu", "tokio", "tracing", ] [[package]] name = "lance-datagen" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe7c7ea7fd397e495a1646fec360e46ee0cbd75718f1c0e887aad657c5f2944" +checksum = "046f5506ed2271cd941a050de7bf535dd3aedc291aadec836a63fa56c5926e3b" dependencies = [ "arrow", "arrow-array", @@ -3655,9 +3949,9 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3f8070835b407d8db9ea8728386bc3207ba23c66a9c22d344e231ef12b77ca" +checksum = "7af54edf43dcf9d6a56cc636eb35d457e68373c6448dca3f0891b3325b4a24e6" dependencies = [ "arrow-arith", "arrow-array", @@ -3682,9 +3976,7 @@ dependencies = [ "num-traits", "prost", "prost-build", - "prost-types", "rand 0.9.2", - "snafu", "strum", "tokio", "tracing", @@ -3694,9 +3986,9 @@ dependencies = [ [[package]] name = "lance-file" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dfcf654549330df3aef708cd7c12e170feecddd34d6c19dd005b4153213268" +checksum = "0772ae2d6207995dc1eb28aff9507f78e90b3362b58f311da001e9dc25f3d736" dependencies = [ "arrow-arith", "arrow-array", @@ -3717,21 +4009,21 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store 0.12.5", + "object_store", "prost", "prost-build", "prost-types", - "snafu", "tokio", "tracing", ] [[package]] name = "lance-index" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb8ad0bd10efa2608634a2518b7dd501231e76c56a65fbd6519e23914cc425a" +checksum = "e71fbfb51096a903cb524fe0da716f5f15fbc4a6b6f84cd6dec21abf319c5e84" dependencies = [ + "arc-swap", "arrow", "arrow-arith", "arrow-array", @@ -3750,7 +4042,6 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "datafusion-sql", "deepsize", "dirs", "fst", @@ -3772,7 +4063,7 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store 0.12.5", + "object_store", "prost", "prost-build", "prost-types", @@ -3784,7 +4075,6 @@ dependencies = [ "serde", "serde_json", "smallvec", - "snafu", "tempfile", "tokio", "tracing", @@ -3794,9 +4084,9 @@ dependencies = [ [[package]] name = "lance-io" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5314703fa8c8baed04193cc669da80ab42521c6319d3cc921a4a997690dcc0" +checksum = "bab8c98ef1b870b20541d27f3ca4efdf7c9f5c25214233be07d231ba88900219" dependencies = [ "arrow", "arrow-arith", @@ -3820,10 +4110,9 @@ dependencies = [ "lance-arrow", "lance-core", "lance-namespace", - "libc", "log", "moka", - "object_store 0.12.5", + "object_store", "object_store_opendal", "opendal", "path_abs", @@ -3831,7 +4120,6 @@ dependencies = [ "prost", "rand 0.9.2", "serde", - "snafu", "tempfile", "tokio", "tracing", @@ -3840,9 +4128,9 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51aa9b73279f505b2bec0f194c7a2390ca74ad3260131e631a7bef8d97d54b2e" +checksum = "6b4c51cad0ac780b02dc4da48528479e7693c03e8d05390510bbc69ca2a9a1f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -3858,31 +4146,29 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cd01581f55ce45c49cbe494ee86c7ba7ca4ca3654690fd820941cd9105a46e" +checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", - "serde", "snafu", ] [[package]] name = "lance-namespace-impls" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2cb89f3933060f01350ad05a5a3fbda952e8ba638799bf8ac4cd2368416ee46" +checksum = "e8d1231906a3cf92dd3dcda7d14a09c4835af6cd2bcd76dfd2481e87f20a282d" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "chrono", "futures", "lance", "lance-core", @@ -3892,10 +4178,9 @@ dependencies = [ "lance-namespace", "lance-table", "log", - "object_store 0.12.5", + "object_store", "rand 0.9.2", "serde_json", - "snafu", "tokio", "url", ] @@ -3906,7 +4191,7 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99" dependencies = [ - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_repr", @@ -3916,9 +4201,9 @@ dependencies = [ [[package]] name = "lance-table" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db70650465a1af174b7dfe6948ec91a3d466ada12e11274eb66e51132173aa0" +checksum = "b16f1355904aea4ebb04ffc70c58c97901e10bde44452b4b021de4a1f329250d" dependencies = [ "arrow", "arrow-array", @@ -3936,7 +4221,7 @@ dependencies = [ "lance-file", "lance-io", "log", - "object_store 0.12.5", + "object_store", "prost", "prost-build", "prost-types", @@ -3955,9 +4240,9 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "6.0.1" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb08ef9382c9d58036c323db2c19cc097e02d1d0d87714fc7176b5d3b36a31aa" +checksum = "b39b7f5ed9d0c0b716bf599b559d888267ed1dfe4c4e29d3648b51d2a28940cf" dependencies = [ "rust-stemmers", "serde", @@ -4140,7 +4425,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "generator", "scoped-tls", "tracing", @@ -4215,8 +4500,17 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", - "digest", + "cfg-if 1.0.4", + "digest 0.10.7", +] + +[[package]] +name = "mea" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" +dependencies = [ + "slab", ] [[package]] @@ -4231,7 +4525,7 @@ version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "miette-derive", "serde", "unicode-width 0.1.14", @@ -4281,16 +4575,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] -[[package]] -name = "mock_instant" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6" - [[package]] name = "moka" version = "0.12.15" @@ -4311,6 +4599,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "multimap" version = "0.10.1" @@ -4362,6 +4656,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4452,6 +4755,34 @@ dependencies = [ "libc", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.37.3" @@ -4463,16 +4794,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.5" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http 1.4.0", "http-body-util", "httparse", @@ -4482,11 +4815,11 @@ dependencies = [ "md-5", "parking_lot", "percent-encoding", - "quick-xml 0.38.4", - "rand 0.9.2", - "reqwest", + "quick-xml 0.39.4", + "rand 0.10.1", + "reqwest 0.12.28", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -4499,48 +4832,34 @@ dependencies = [ "web-time", ] -[[package]] -name = "object_store" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures-channel", - "futures-core", - "futures-util", - "http 1.4.0", - "humantime", - "itertools 0.14.0", - "parking_lot", - "percent-encoding", - "thiserror", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "object_store_opendal" -version = "0.55.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113ab0769e972eee585e57407b98de08bda5354fa28e8ba4d89038d6cb6a8991" +checksum = "08298874eee5935c95bcaa393148834f9c53d904461ca15584a041d8a1c907c2" dependencies = [ "async-trait", "bytes", "chrono", "futures", - "object_store 0.12.5", + "mea", + "object_store", "opendal", "pin-project", "tokio", ] +[[package]] +name = "omnigraph-api-types" +version = "0.7.0" +dependencies = [ + "omnigraph-compiler", + "omnigraph-engine", + "serde", + "serde_json", + "utoipa", +] + [[package]] name = "omnigraph-cli" version = "0.7.0" @@ -4550,13 +4869,14 @@ dependencies = [ "color-eyre", "lance", "lance-index", + "omnigraph-api-types", "omnigraph-cluster", "omnigraph-compiler", "omnigraph-engine", "omnigraph-policy", "omnigraph-server", "predicates", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_yaml", @@ -4574,7 +4894,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror", "time", @@ -4595,12 +4915,10 @@ dependencies = [ "arrow-select", "pest", "pest_derive", - "reqwest", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror", - "tokio", ] [[package]] @@ -4627,16 +4945,16 @@ dependencies = [ "lance-namespace", "lance-namespace-impls", "lance-table", - "object_store 0.12.5", + "object_store", "omnigraph-compiler", "omnigraph-policy", "proptest", "regex", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serial_test", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror", "time", @@ -4674,6 +4992,7 @@ dependencies = [ "futures", "lance", "lance-index", + "omnigraph-api-types", "omnigraph-cluster", "omnigraph-compiler", "omnigraph-engine", @@ -4683,7 +5002,7 @@ dependencies = [ "serde_json", "serde_yaml", "serial_test", - "sha2", + "sha2 0.10.9", "subtle", "tempfile", "thiserror", @@ -4708,33 +5027,226 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] -name = "opendal" -version = "0.55.0" +name = "oneshot" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "opendal" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236" +dependencies = [ + "ctor", + "opendal-core", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-azblob", + "opendal-service-azdls", + "opendal-service-gcs", + "opendal-service-hf", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849dd2687e173e776d3af5fce1ba3ae47b9dd37a09d1c4deba850ef45fe00ca" dependencies = [ "anyhow", - "backon", "base64", "bytes", - "crc32c", "futures", - "getrandom 0.2.17", "http 1.4.0", "http-body 1.0.1", "jiff", "log", "md-5", + "mea", "percent-encoding", "quick-xml 0.38.4", - "reqsign", - "reqwest", + "reqsign-core", + "reqwest 0.13.4", "serde", "serde_json", - "sha2", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048b1b29c503263bdd80a9afe46a68cd02ea9bd361185b1feab4b151078998e9" +dependencies = [ + "futures", + "http 1.4.0", + "mea", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2645adc988b12eda106e2679ae529facfbbaa868ceb706f6f8125c6af15c47b" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eac134ffa4ddda6131a640a84a5315996424b9416c85052f8c64c1a33b70ad4" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619586ab7480c2e3009f6d18eabab18957bc094778fd130bcc38924970a90f4c" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7452bf3ec61cfd81ac9ad9ada17825931e9e371d44a045c6bfab9596c0a2ac3b" +dependencies = [ + "base64", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.10.9", + "uuid", +] + +[[package]] +name = "opendal-service-azdls" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f9884c2d8cf8ba2bb077d79c877dac5863ba3bab9e2c9c1e41a2e0491404772" +dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb0e45d6c8dcf66ce2da20e241bcb80e6e540e109a4ff20f318f6c9b4c54e0c" +dependencies = [ + "http 1.4.0", + "opendal-core", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a49477a10163431896d106136117f5670717f9c9e49cf6f710528800c6633a" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-hf" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2ab7a2a8a11dfe257ef4db5c0de798acbcd0d6429c37382dad2154bc06a388" +dependencies = [ + "bytes", + "hf-xet", + "http 1.4.0", + "log", + "opendal-core", + "percent-encoding", + "reqwest 0.13.4", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-oss" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c8a917829ad06d21b639558532cb0101fe49b040d946d673a73018683fac05" +dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", +] + +[[package]] +name = "opendal-service-s3" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dadddeb9bb50b0d30927dd914c298c4ddca47e4c1cfa7674d311f0cf9b051c8" +dependencies = [ + "base64", + "bytes", + "crc32c", + "http 1.4.0", + "log", + "md-5", + "opendal-core", + "quick-xml 0.38.4", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", ] [[package]] @@ -4768,6 +5280,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] + [[package]] name = "outref" version = "0.5.2" @@ -4802,7 +5323,7 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "redox_syscall", "smallvec", @@ -4833,8 +5354,8 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", - "hmac", + "digest 0.10.7", + "hmac 0.12.1", ] [[package]] @@ -4908,7 +5429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -5020,7 +5541,7 @@ dependencies = [ "der", "pbkdf2", "scrypt", - "sha2", + "sha2 0.10.9", "spki", ] @@ -5235,9 +5756,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -5245,9 +5766,19 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", "serde", @@ -5279,6 +5810,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -5310,9 +5842,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -5356,6 +5888,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -5394,6 +5937,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -5467,6 +6016,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redb" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -5543,34 +6101,116 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] -name = "reqsign" -version = "0.16.5" +name = "reqsign-aliyun-oss" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" dependencies = [ "anyhow", - "async-trait", - "base64", - "chrono", "form_urlencoded", - "getrandom 0.2.17", - "hex", - "hmac", - "home", "http 1.4.0", - "jsonwebtoken", "log", - "once_cell", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", - "rsa", + "reqsign-core", "rust-ini", "serde", "serde_json", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +dependencies = [ + "anyhow", + "bytes", + "form_urlencoded", + "hex", + "http 1.4.0", + "log", + "percent-encoding", + "quick-xml 0.40.1", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", "sha1", - "sha2", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +dependencies = [ + "anyhow", + "base64", + "bytes", + "form_urlencoded", + "http 1.4.0", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1", +] + +[[package]] +name = "reqsign-core" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +dependencies = [ + "anyhow", + "base64", + "bytes", + "form_urlencoded", + "futures", + "hex", + "hmac 0.13.0", + "http 1.4.0", + "jiff", + "log", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +dependencies = [ + "form_urlencoded", + "http 1.4.0", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", "tokio", ] @@ -5616,11 +6256,65 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls 0.27.7", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.36", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls 0.26.4", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http 1.4.0", + "reqwest 0.13.4", + "thiserror", + "tower-service", +] + [[package]] name = "ring" version = "0.17.14" @@ -5628,7 +6322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "getrandom 0.2.17", "libc", "untrusted", @@ -5637,9 +6331,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ "bytemuck", "byteorder", @@ -5651,15 +6345,15 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", "pkcs8", "rand_core 0.6.4", - "sha2", + "sha2 0.10.9", "signature", "spki", "subtle", @@ -5672,7 +6366,7 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "ordered-multimap", ] @@ -5765,15 +6459,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -5784,6 +6469,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.9", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -5830,6 +6542,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe-transmute" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + [[package]] name = "salsa20" version = "0.10.2" @@ -5910,7 +6628,7 @@ checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -5936,7 +6654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -6114,13 +6832,13 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -6129,9 +6847,30 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "cfg-if 1.0.4", + "cpufeatures 0.2.17", + "digest 0.10.7", + "sha2-asm", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] [[package]] @@ -6140,7 +6879,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest", + "digest 0.10.7", "keccak", ] @@ -6153,6 +6892,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs", + "os_str_bytes", +] + [[package]] name = "shlex" version = "1.3.0" @@ -6175,7 +6925,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -6185,24 +6935,22 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "simple_asn1" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror", - "time", -] - [[package]] name = "siphasher" version = "1.0.2" @@ -6322,12 +7070,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "libc", "psm", "windows-sys 0.59.0", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "std_prelude" version = "0.2.12" @@ -6386,6 +7150,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -6428,6 +7198,41 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -6503,7 +7308,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6599,6 +7404,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-retry" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" +dependencies = [ + "pin-project-lite", + "rand 0.10.1", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -6708,6 +7524,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -6750,6 +7579,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.22" @@ -6760,12 +7599,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -6791,9 +7633,15 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typewit" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" [[package]] name = "ucd-trie" @@ -7014,6 +7862,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -7032,13 +7889,22 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -7051,7 +7917,7 @@ version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "futures-util", "js-sys", "once_cell", @@ -7126,6 +7992,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -7158,6 +8037,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -7167,6 +8055,35 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -7176,6 +8093,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -7189,6 +8133,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -7217,6 +8172,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -7304,6 +8280,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -7503,6 +8488,153 @@ dependencies = [ "tap", ] +[[package]] +name = "xet-client" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bytes", + "clap", + "crc32fast", + "futures", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "more-asserts", + "rand 0.10.1", + "redb", + "reqwest 0.13.4", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror", + "tokio", + "tokio-retry", + "tracing", + "tracing-subscriber", + "url", + "urlencoding", + "web-time", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-core-structures" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +dependencies = [ + "async-trait", + "base64", + "blake3", + "bytemuck", + "bytes", + "clap", + "countio", + "csv", + "futures", + "futures-util", + "getrandom 0.4.2", + "heapify", + "itertools 0.14.0", + "lazy_static", + "lz4_flex", + "more-asserts", + "rand 0.10.1", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "clap", + "gearhash", + "http 1.4.0", + "itertools 0.14.0", + "lazy_static", + "more-asserts", + "rand 0.10.1", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "url", + "uuid", + "walkdir", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored", + "const-str", + "ctor", + "dirs", + "futures", + "git-version", + "humantime", + "konst", + "lazy_static", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand 0.10.1", + "reqwest 0.13.4", + "serde", + "serde_json", + "shellexpand", + "sysinfo", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "tracing-appender", + "tracing-subscriber", + "whoami", + "winapi", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 918ac05..c442242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/omnigraph-compiler", "crates/omnigraph", "crates/omnigraph-cli", + "crates/omnigraph-api-types", "crates/omnigraph-cluster", "crates/omnigraph-policy", "crates/omnigraph-server", @@ -30,14 +31,14 @@ datafusion-common = "53" datafusion-expr = "53" datafusion-functions-aggregate = "53" -lance = { version = "6.0.1", default-features = false, features = ["aws"] } -lance-datafusion = "6.0.1" -lance-file = "6.0.1" -lance-index = "6.0.1" -lance-linalg = "6.0.1" -lance-namespace = "6.0.1" -lance-namespace-impls = "6.0.1" -lance-table = "6.0.1" +lance = { version = "7.0.0", default-features = false, features = ["aws"] } +lance-datafusion = "7.0.0" +lance-file = "7.0.0" +lance-index = "7.0.0" +lance-linalg = "7.0.0" +lance-namespace = "7.0.0" +lance-namespace-impls = "7.0.0" +lance-table = "7.0.0" ulid = "1" futures = "0.3" @@ -47,7 +48,7 @@ pest = "2" pest_derive = "2" thiserror = "2" tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net", "signal", "sync"] } -clap = { version = "4", features = ["derive"] } +clap = { version = "4.6", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" @@ -63,7 +64,7 @@ base64 = "0.22" ariadne = "0.4" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -object_store = { version = "0.12.5", default-features = false, features = ["aws", "fs"] } +object_store = { version = "0.13.2", default-features = false, features = ["aws", "fs"] } fail = "0.5" time = { version = "0.3", features = ["formatting"] } axum = { version = "0.8", features = ["json", "macros"] } diff --git a/README.md b/README.md index a75a839..9c4f8bc 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![Rust](https://img.shields.io/badge/rust-stable-orange.svg)](rust-toolchain.toml) [![Crates.io](https://img.shields.io/crates/v/omnigraph-cli.svg)](https://crates.io/crates/omnigraph-cli) -[![CI](https://github.com/ModernRelay/omnigraph/actions/workflows/ci.yml/badge.svg)](https://github.com/ModernRelay/omnigraph/actions/workflows/ci.yml) **Lakehouse native graph engine built for context assembly** -Omnigraph acts as operational state & coordination layer for agents +Omnigraph acts as operational state & coordination layer for agents. +Hundreds of agents can enrich the graph on parallel isolated branches and changes can be reviewed and merged safely. - Git-style versioning & branching - Multimodal retrieval (graph+vector/fts+filters) optimized for context assembly -- Object storage native (S3, RustFS) +- Runs on the local filesystem or any S3-compatible object store (AWS S3, R2, MinIO, RustFS) - Native blob-as-data support (docs, images, videos, etc) - VPC, On-prem, hybrid deployment - [`Lance`](https://github.com/lance-format/lance) format as open storage layer @@ -51,62 +51,138 @@ brew tap ModernRelay/tap brew install ModernRelay/tap/omnigraph ``` -For starter graphs and agent skills to bootstrap and operate Omnigraph, see [`ModernRelay/omnigraph-cookbooks`](https://github.com/ModernRelay/omnigraph-cookbooks). +## Quick start -## One-Command Local RustFS Bootstrap +The fastest path is an **embedded, local file-backed graph** — no server, no +object store, no Docker: ```bash -curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/local-rustfs-bootstrap.sh | bash +# A schema and one row of data +cat > schema.pg <<'PG' +node Person { + slug: String @key + name: String + title: String? +} +PG +echo '{"type":"Person","data":{"slug":"alice","name":"Alice","title":"Engineer"}}' > people.jsonl + +# Create → load (--mode is required) → query +omnigraph init --schema schema.pg ./graph.omni +omnigraph load --data people.jsonl --mode overwrite --store ./graph.omni +omnigraph query find_people --store ./graph.omni --params '{"t":"Engineer"}' \ + -e 'query find_people($t: String) { match { $p: Person { title: $t } } return { $p.name } }' + +# Branch, write in isolation, merge — Git-style across the whole graph +omnigraph branch create --from main review/new-hires --store ./graph.omni +omnigraph branch merge review/new-hires --into main --store ./graph.omni ``` -That bootstrap: +**Storage backends** — the same flow runs on any backend; only the graph address changes: -- starts RustFS on `127.0.0.1:9000` -- creates a bucket and S3-backed graph -- loads the checked-in context fixture -- launches `omnigraph-server` on `127.0.0.1:8080` +| Backend | Use it for | Graph address | +|---|---|---| +| **Embedded** (local filesystem) | dev, demos, single machine — the default | `./graph.omni` | +| **Object storage** (AWS S3, R2, GCS-S3) | shared, multi-host, durable | `s3://bucket/graph.omni` (+ the `AWS_*` env) | +| **RustFS / MinIO** | rehearse the S3 path locally, no cloud account | `s3://…` against a local endpoint → [deployment guide](docs/user/deployment.md#testing-against-s3-locally) | -Docker must be installed and running first. +`init` takes the address as its positional argument (`omnigraph init --schema schema.pg
`); `load`, `query`, and `branch` take it via `--store
`. -The RustFS bootstrap prefers the rolling `edge` binaries and only falls back to -source builds when release assets are unavailable. +For a **served, multi-graph deployment** (the cluster model), see [Common Commands](#common-commands) below. -If a previous run left objects under the same graph prefix but did not finish -initializing the graph, rerun with `RESET_REPO=1` or set `PREFIX` to a new -value. +## Set it up with an AI agent + +Omnigraph is built to be set up by coding agents. Paste this into Claude Code, +Cursor, or any agent that can read a URL, install a package, and run a shell +command — it installs the skill, reads the docs, and walks you through setup for +your use case: + +```text +Help me set up Omnigraph (a lakehouse-native graph engine for agents). + +1. Install the Omnigraph skill so you operate it correctly: + npx skills add ModernRelay/omnigraph@omnigraph +2. Read the docs at https://github.com/ModernRelay/omnigraph — start with + docs/user/quickstart.md, then docs/user/clusters/index.md. +3. Skim the starter graphs and seed data in the cookbooks: + https://github.com/ModernRelay/omnigraph-cookbooks +4. Ask me what I want to build (company brain, agent memory, dev graph, + research / R&D layer, …). Then install the CLI, stand up a first graph for + that use case, load a little data, and run a query so I can see it working. +``` + +Works with any agent that can browse a URL, install a package, and run a shell. + +## Agent skill & starter graphs + +This repo ships the [**`omnigraph` agent skill**](skills/omnigraph) — the +operational playbook (cluster mode, the two config surfaces, schema evolution, +query linting, data writes, branches, Cedar policy, and common gotchas) that +teaches a coding agent to drive Omnigraph correctly. Install it with: + +```bash +npx skills add ModernRelay/omnigraph@omnigraph +``` + +For ready-to-run graphs with real seed data (company brain, VC operating system, +pharma & industry intel), +[`ModernRelay/omnigraph-cookbooks`](https://github.com/ModernRelay/omnigraph-cookbooks) +is the fastest way to see Omnigraph shaped to a real domain. To rehearse the S3 +path locally, see [deployment.md → Testing against S3 locally](docs/user/deployment.md#testing-against-s3-locally). ## Common Commands -The same URI works for local paths, `s3://…`, or `http://host:port`. +A deployment is a **cluster**. A `cluster.yaml` declares its graphs, schemas, +stored queries, and policies; you converge it with `cluster apply` and serve it. +The server is cluster-first — it boots only from a cluster and serves every graph +under `/graphs/{id}/…`. Day-to-day work goes through that server: graphs are +addressed with `--server ` (+ `--graph `), and `query`/`mutate` +invoke a stored query from the catalog **by name**. ```bash -omnigraph init --schema ./schema.pg ./graph.omni -omnigraph load --data ./data.jsonl ./graph.omni -omnigraph read --query ./queries.gq --name get_person --params '{"name":"Alice"}' ./graph.omni -omnigraph change --query ./queries.gq --name insert_person --params '{"name":"Mina"}' ./graph.omni -omnigraph branch create --from main feature-x ./graph.omni -omnigraph branch merge feature-x --into main ./graph.omni +# 1. Converge the declared cluster, then serve it (--as attributes the apply) +omnigraph cluster apply --config ./company-brain --as you +omnigraph-server --cluster ./company-brain --bind 0.0.0.0:8080 +# or config-free from object storage — the bucket IS the deployment: +# omnigraph-server --cluster s3://my-bucket/company-brain --bind 0.0.0.0:8080 + +# 2. Work against the served graph — stored queries invoked by name +omnigraph query find_people --server prod --graph knowledge --params '{"q":"AI safety"}' +omnigraph mutate add_person --server prod --graph knowledge --params '{"name":"Mina"}' +omnigraph load --data ./data.jsonl --mode merge --server prod --graph knowledge + +# 3. Branch and merge, Git-style across the whole graph +omnigraph branch create --from main review/2026-06 --server prod --graph knowledge +omnigraph branch merge review/2026-06 --into main --server prod --graph knowledge ``` -See [docs/user/cli.md](docs/user/cli.md) for schema apply, snapshots, data loading, commits, and policy commands. +Set a default scope (or a `--profile`) in `~/.omnigraph/config.yaml` — operator +identity, named servers/clusters, credentials — and the `--server`/`--graph` +flags drop away (`omnigraph query find_people --params …`). + +**Local / ad-hoc.** For quick iteration on a standalone graph (no cluster, no +server), address storage directly with `--store` (or a positional `file://` / +`s3://` URI) and run ad-hoc `.gq` with `--query` (the positional then selects +which query in the file): + +```bash +omnigraph init --schema ./schema.pg ./graph.omni +omnigraph load --data ./data.jsonl --mode merge --store ./graph.omni +omnigraph query --query ./queries.gq get_person --params '{"name":"Alice"}' --store ./graph.omni +``` + +See [docs/user/cli/index.md](docs/user/cli/index.md), the +[CLI reference](docs/user/cli/reference.md), the +[cluster guide](docs/user/clusters/index.md), and the +[deployment guide](docs/user/deployment.md) for schema apply, snapshots, commits, +profiles, and policy/queries tooling. ## Clients For programmatic access to a running `omnigraph-server`: -- **TypeScript SDK** — [`@modernrelay/omnigraph`](https://www.npmjs.com/package/@modernrelay/omnigraph) ([source](https://github.com/ModernRelay/omnigraph-ts/tree/main/packages/sdk)). Instance-per-client, typed errors, camelCase types, async-iterator streaming export. - - ```bash - npm install @modernrelay/omnigraph - ``` - -- **Model Context Protocol server** — [`@modernrelay/omnigraph-mcp`](https://www.npmjs.com/package/@modernrelay/omnigraph-mcp) ([source](https://github.com/ModernRelay/omnigraph-ts/tree/main/packages/mcp)). Bridges Omnigraph to LLM hosts (Claude Desktop, Claude Code, …) over stdio. Exposes tools and resources for schema, branches, queries, mutations, ingest, and bundles curated best-practices guidance from the cookbook. - - ```bash - npm install -g @modernrelay/omnigraph-mcp - ``` - -Both packages are versioned in lockstep with `omnigraph-server` on major.minor: `@modernrelay/omnigraph@X.Y.*` targets `omnigraph-server@X.Y.*`. See [`ModernRelay/omnigraph-ts`](https://github.com/ModernRelay/omnigraph-ts) for the monorepo. +- **TypeScript SDK + MCP server** — [`@modernrelay/omnigraph`](https://www.npmjs.com/package/@modernrelay/omnigraph) and [`@modernrelay/omnigraph-mcp`](https://www.npmjs.com/package/@modernrelay/omnigraph-mcp), versioned in lockstep with `omnigraph-server`. Source, docs, and examples: [`ModernRelay/omnigraph-ts`](https://github.com/ModernRelay/omnigraph-ts). +- **Python SDK** — coming soon. ## Docs @@ -130,10 +206,13 @@ Notes: ## Workspace Crates -- `crates/omnigraph-compiler`: shared schema/query parser, typechecker, catalog, and IR lowering -- `crates/omnigraph`: storage/runtime, branching, merge, change detection, and query execution -- `crates/omnigraph-cli`: CLI for graph lifecycle (init/load), query/mutate, branch/commit/merge, schema/lint, snapshot/export, policy, and maintenance (optimize/cleanup) -- `crates/omnigraph-server`: Axum HTTP server for remote reads, changes, ingest, export, branches, and commits +- `crates/omnigraph-compiler`: shared schema/query parser, typechecker, catalog, and IR lowering (zero Lance dependency) +- `crates/omnigraph` (package `omnigraph-engine`): storage/runtime, branching, merge, change detection, query execution, and embeddings +- `crates/omnigraph-policy`: Cedar policy compilation and enforcement +- `crates/omnigraph-api-types`: shared HTTP wire DTOs used by both the server and the CLI +- `crates/omnigraph-cluster`: cluster config validation, planning, and apply (the control plane) +- `crates/omnigraph-server`: Axum HTTP server — cluster-first, serving N graphs under `/graphs/{id}/…` +- `crates/omnigraph-cli`: CLI for graph lifecycle (init/load), query/mutate, branch/commit/merge, schema/lint, snapshot/export, cluster control, policy/queries, profiles, and maintenance (optimize/repair/cleanup) ## Contributing diff --git a/crates/omnigraph-api-types/Cargo.toml b/crates/omnigraph-api-types/Cargo.toml new file mode 100644 index 0000000..d69d4fe --- /dev/null +++ b/crates/omnigraph-api-types/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "omnigraph-api-types" +version = "0.7.0" +edition = "2024" +description = "Shared HTTP wire DTOs for Omnigraph — request/response types and engine-result → DTO mappings used by both omnigraph-server and omnigraph-cli (RFC-009). Plain serde/utoipa types; no transport or server internals." +license = "MIT" +repository = "https://github.com/ModernRelay/omnigraph" +homepage = "https://github.com/ModernRelay/omnigraph" +documentation = "https://docs.rs/omnigraph-api-types" + +[dependencies] +omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.7.0" } +omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.7.0" } +serde = { workspace = true } +serde_json = { workspace = true } +utoipa = { workspace = true } diff --git a/crates/omnigraph-api-types/src/lib.rs b/crates/omnigraph-api-types/src/lib.rs new file mode 100644 index 0000000..2814602 --- /dev/null +++ b/crates/omnigraph-api-types/src/lib.rs @@ -0,0 +1,704 @@ +//! Shared HTTP wire DTOs (RFC-009 Phase 2) — moved from +//! omnigraph-server's api module so server and CLI share one definition +//! and one engine-result -> DTO mapping per verb. Plain serde/utoipa +//! types; no transport, no server internals. + +use omnigraph::db::{GraphCommit, MergeOutcome, ReadTarget, SchemaApplyResult, Snapshot}; +use omnigraph::error::{MergeConflict, MergeConflictKind}; +use omnigraph::loader::{LoadMode, LoadResult}; +use omnigraph_compiler::SchemaMigrationStep; +use omnigraph_compiler::query::ast::Param; +use omnigraph_compiler::result::QueryResult; +use omnigraph_compiler::types::{PropType, ScalarType}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use utoipa::{IntoParams, ToSchema}; + +/// Shadow enum for documenting [`LoadMode`] in the OpenAPI schema. +#[derive(ToSchema)] +#[schema(as = LoadMode)] +#[allow(dead_code)] +enum LoadModeSchema { + /// Overwrite existing data. + #[schema(rename = "overwrite")] + Overwrite, + /// Append to existing data. + #[schema(rename = "append")] + Append, + /// Merge by id key (upsert). + #[schema(rename = "merge")] + Merge, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct SnapshotTableOutput { + pub table_key: String, + pub table_path: String, + pub table_version: u64, + pub table_branch: Option, + pub row_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct SnapshotOutput { + pub branch: String, + pub manifest_version: u64, + pub tables: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchCreateRequest { + /// Parent branch to fork from. Defaults to `main`. + pub from: Option, + /// Name of the new branch. Must not already exist. + pub name: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchCreateOutput { + pub uri: String, + pub from: String, + pub name: String, + pub actor_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchListOutput { + pub branches: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchDeleteOutput { + pub uri: String, + pub name: String, + pub actor_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchMergeRequest { + /// Source branch whose commits will be merged. + pub source: String, + /// Target branch that will receive the merge. Defaults to `main`. + pub target: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum BranchMergeOutcome { + AlreadyUpToDate, + FastForward, + Merged, +} + +impl From for BranchMergeOutcome { + fn from(value: MergeOutcome) -> Self { + match value { + MergeOutcome::AlreadyUpToDate => Self::AlreadyUpToDate, + MergeOutcome::FastForward => Self::FastForward, + MergeOutcome::Merged => Self::Merged, + } + } +} + +impl BranchMergeOutcome { + pub fn as_str(self) -> &'static str { + match self { + Self::AlreadyUpToDate => "already_up_to_date", + Self::FastForward => "fast_forward", + Self::Merged => "merged", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BranchMergeOutput { + pub source: String, + pub target: String, + pub outcome: BranchMergeOutcome, + pub actor_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum MergeConflictKindOutput { + DivergentInsert, + DivergentUpdate, + DeleteVsUpdate, + OrphanEdge, + UniqueViolation, + CardinalityViolation, + ValueConstraintViolation, +} + +impl MergeConflictKindOutput { + pub fn as_str(self) -> &'static str { + match self { + Self::DivergentInsert => "divergent_insert", + Self::DivergentUpdate => "divergent_update", + Self::DeleteVsUpdate => "delete_vs_update", + Self::OrphanEdge => "orphan_edge", + Self::UniqueViolation => "unique_violation", + Self::CardinalityViolation => "cardinality_violation", + Self::ValueConstraintViolation => "value_constraint_violation", + } + } +} + +impl From for MergeConflictKindOutput { + fn from(value: MergeConflictKind) -> Self { + match value { + MergeConflictKind::DivergentInsert => Self::DivergentInsert, + MergeConflictKind::DivergentUpdate => Self::DivergentUpdate, + MergeConflictKind::DeleteVsUpdate => Self::DeleteVsUpdate, + MergeConflictKind::OrphanEdge => Self::OrphanEdge, + MergeConflictKind::UniqueViolation => Self::UniqueViolation, + MergeConflictKind::CardinalityViolation => Self::CardinalityViolation, + MergeConflictKind::ValueConstraintViolation => Self::ValueConstraintViolation, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct MergeConflictOutput { + pub table_key: String, + pub row_id: Option, + pub kind: MergeConflictKindOutput, + pub message: String, +} + +impl From<&MergeConflict> for MergeConflictOutput { + fn from(value: &MergeConflict) -> Self { + Self { + table_key: value.table_key.clone(), + row_id: value.row_id.clone(), + kind: value.kind.into(), + message: value.message.clone(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ReadTargetOutput { + pub branch: Option, + pub snapshot: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ReadOutput { + pub query_name: String, + pub target: ReadTargetOutput, + pub row_count: usize, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub columns: Vec, + pub rows: Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ChangeOutput { + pub branch: String, + pub query_name: String, + pub affected_nodes: usize, + pub affected_edges: usize, + pub actor_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct IngestTableOutput { + pub table_key: String, + pub rows_loaded: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct IngestOutput { + pub uri: String, + pub branch: String, + /// Base branch a fork was requested from (the request's `from`), echoed + /// even when the branch already existed. `null` when `from` was absent. + pub base_branch: Option, + pub branch_created: bool, + #[schema(value_type = LoadModeSchema)] + pub mode: LoadMode, + pub tables: Vec, + pub actor_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct CommitOutput { + pub graph_commit_id: String, + pub manifest_branch: Option, + pub manifest_version: u64, + pub parent_commit_id: Option, + pub merged_parent_commit_id: Option, + pub actor_id: Option, + /// Commit creation time as Unix epoch microseconds. + #[schema(example = 1714000000000000i64)] + pub created_at: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct CommitListOutput { + pub commits: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ReadRequest { + /// GQ query source. May declare one or more named queries; pick one with + /// `query_name` if there is more than one. + #[schema( + example = "query get_person($name: String) {\n match {\n $p: Person { name: $name }\n }\n return { $p.name, $p.age }\n}" + )] + pub query_source: String, + /// Name of the query to run when `query_source` declares multiple. Optional + /// when only one query is declared. + pub query_name: Option, + /// JSON object whose keys match the query's declared parameters. + pub params: Option, + /// Branch to read from. Mutually exclusive with `snapshot`. Defaults to `main`. + pub branch: Option, + /// Snapshot id to read from. Mutually exclusive with `branch`. + pub snapshot: Option, +} + +/// Inline read-query request for `POST /query`. +/// +/// Friendlier-named alternative to [`ReadRequest`] for ad-hoc reads and +/// AI-agent integration. Mutations are rejected with 400 — use `POST +/// /mutate` (or its deprecated alias `POST /change`) for write queries. +/// Field names are deliberately short (`query`, `name`) to match the GQ +/// keyword and the CLI `-e` flag. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct QueryRequest { + /// GQ read-query source. May declare one or more named queries; pick one + /// with `name` when more than one is declared. Mutations + /// (`insert`/`update`/`delete`) get 400 — use `POST /mutate` (or its + /// deprecated alias `POST /change`) instead. + #[schema(example = "query get_person($name: String) {\n match {\n $p: Person { name: $name }\n }\n return { $p.name, $p.age }\n}")] + pub query: String, + /// Name of the query to run when `query` declares multiple. Optional when + /// only one query is declared. + pub name: Option, + /// JSON object whose keys match the query's declared parameters. + pub params: Option, + /// Branch to read from. Mutually exclusive with `snapshot`. Defaults to `main`. + pub branch: Option, + /// Snapshot id to read from. Mutually exclusive with `branch`. + pub snapshot: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ChangeRequest { + /// GQ mutation source containing `insert`, `update`, or `delete` statements. + /// May declare multiple named mutations; pick one with `name`. + /// + /// Accepts the legacy field name `query_source` as a deserialization alias. + #[schema( + example = "query insert_person($name: String, $age: I32) {\n insert Person { name: $name, age: $age }\n}" + )] + #[serde(alias = "query_source")] + pub query: String, + /// Name of the mutation to run when `query` declares multiple. + /// + /// Accepts the legacy field name `query_name` as a deserialization alias. + #[serde(default, alias = "query_name")] + pub name: Option, + /// JSON object whose keys match the mutation's declared parameters. + #[serde(default)] + pub params: Option, + /// Target branch. Defaults to `main`. + #[serde(default)] + pub branch: Option, +} + +/// Body for `POST /queries/{name}` — invokes the server-side stored query +/// named in the path. The query source and name come from the registry, +/// never the body; only the runtime inputs are supplied here. +#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)] +pub struct InvokeStoredQueryRequest { + /// JSON object whose keys match the stored query's declared parameters. + #[serde(default)] + pub params: Option, + /// Branch to run against. Defaults to `main`; for a stored mutation the + /// write targets this branch. + #[serde(default)] + pub branch: Option, + /// Snapshot id to read from (read queries only — rejected for a stored + /// mutation). Mutually exclusive with `branch`. + #[serde(default)] + pub snapshot: Option, + /// The kind the caller expects (RFC-011 Decision 3): `Some(false)` for + /// `omnigraph query `, `Some(true)` for `omnigraph mutate `. + /// When set and it disagrees with the stored query's actual kind, the + /// server rejects the call (400) so the verb asserts the kind. `None` + /// (the default) skips the check — preserving older clients and aliases. + #[serde(default)] + pub expect_mutation: Option, +} + +/// Response for `POST /queries/{name}`: the read envelope for a stored +/// read, or the mutation envelope for a stored mutation. Serialized +/// **untagged**, so the wire shape is exactly [`ReadOutput`] or +/// [`ChangeOutput`] — classification follows the stored query, not a +/// wrapper field. +#[derive(Debug, Serialize, ToSchema)] +#[serde(untagged)] +pub enum InvokeStoredQueryResponse { + Read(ReadOutput), + Change(ChangeOutput), +} + +/// The kind of a stored-query parameter, decomposed so a client (e.g. an +/// MCP server) can build a typed input schema with a closed `match` and +/// never re-parse omnigraph's type spelling. `bigint`/`date`/`datetime`/ +/// `blob` are carried as JSON strings on the wire: a 64-bit integer past +/// 2^53 loses precision as a JSON number, and Date/DateTime are ISO +/// strings, Blob a blob-URI string. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum ParamKind { + String, + Bool, + Int, + #[serde(rename = "bigint")] + BigInt, + Float, + Date, + #[serde(rename = "datetime")] + DateTime, + Blob, + Vector, + List, +} + +/// One declared parameter of a stored query, projected for the catalog. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ParamDescriptor { + pub name: String, + pub kind: ParamKind, + /// Element kind when `kind == list` (always a scalar — the grammar + /// forbids lists of vectors or nested lists). + #[serde(skip_serializing_if = "Option::is_none")] + pub item_kind: Option, + /// Dimension when `kind == vector`. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_dim: Option, + /// `false` → the caller must supply it; `true` → optional. + pub nullable: bool, +} + +/// One entry in the stored-query catalog (`GET /queries`). +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct QueryCatalogEntry { + /// Registry key / invoke path segment (`POST /queries/{name}`). + pub name: String, + /// MCP tool id (the `tool_name` override, else `name`). + pub tool_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub instruction: Option, + /// `true` for a stored mutation → an MCP read-only hint of `false`. + pub mutation: bool, + pub params: Vec, +} + +/// Response for `GET /queries`: the `mcp.expose` subset of a graph's +/// stored-query registry, each with typed parameters. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct QueriesCatalogOutput { + pub queries: Vec, +} + +/// Total map from a resolved scalar to its catalog kind. Exhaustive on +/// purpose: a new `ScalarType` is a compile error here until catalogued. +fn scalar_kind(scalar: ScalarType) -> ParamKind { + match scalar { + ScalarType::String => ParamKind::String, + ScalarType::Bool => ParamKind::Bool, + ScalarType::I32 | ScalarType::U32 => ParamKind::Int, + ScalarType::I64 | ScalarType::U64 => ParamKind::BigInt, + ScalarType::F32 | ScalarType::F64 => ParamKind::Float, + ScalarType::Date => ParamKind::Date, + ScalarType::DateTime => ParamKind::DateTime, + ScalarType::Blob => ParamKind::Blob, + ScalarType::Vector(_) => ParamKind::Vector, + } +} + +pub fn param_descriptor(param: &Param) -> ParamDescriptor { + match PropType::from_param_type_name(¶m.type_name, param.nullable) { + Some(pt) if pt.list => ParamDescriptor { + name: param.name.clone(), + kind: ParamKind::List, + item_kind: Some(scalar_kind(pt.scalar)), + vector_dim: None, + nullable: param.nullable, + }, + Some(pt) => { + let (kind, vector_dim) = match pt.scalar { + ScalarType::Vector(dim) => (ParamKind::Vector, Some(dim)), + other => (scalar_kind(other), None), + }; + ParamDescriptor { + name: param.name.clone(), + kind, + item_kind: None, + vector_dim, + nullable: param.nullable, + } + } + // Unreachable for a parsed query (every declared param type is + // grammatical); fall back to an opaque string so the field is still + // usable rather than dropped. + None => ParamDescriptor { + name: param.name.clone(), + kind: ParamKind::String, + item_kind: None, + vector_dim: None, + nullable: param.nullable, + }, + } +} + + +#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)] +pub struct SchemaApplyRequest { + /// Project schema in `.pg` source form. The diff against the current + /// schema produces the migration steps that will be applied. + #[schema( + example = "node Person {\n name: String @key\n age: I32?\n}\n\nedge Knows: Person -> Person" + )] + pub schema_source: String, + /// When true, promote every `DropMode::Soft` step in the plan to + /// `DropMode::Hard`, making the prior column data unreachable + /// after the apply. Matches the CLI's `--allow-data-loss` flag. + /// Defaults to `false` (drops remain reversible via time travel). + #[serde(default)] + pub allow_data_loss: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct SchemaApplyOutput { + pub uri: String, + pub supported: bool, + pub applied: bool, + pub step_count: usize, + pub manifest_version: u64, + #[schema(value_type = Vec)] + pub steps: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct SchemaOutput { + pub schema_source: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct IngestRequest { + /// Target branch. Defaults to `main`. Without `from`, the branch must + /// already exist — a missing branch is a 404, never an implicit fork. + pub branch: Option, + /// Parent branch used to create `branch` if it does not exist. Branch + /// creation is opt-in by presence of this field; omit it to require an + /// existing branch. + pub from: Option, + /// How existing rows are handled. Defaults to `merge`. + #[schema(value_type = Option)] + pub mode: Option, + /// NDJSON payload: one record per line, each shaped + /// `{"type": "", "data": {...}}`. + #[schema( + example = "{\"type\": \"Person\", \"data\": {\"name\": \"Alice\", \"age\": 30}}\n{\"type\": \"Person\", \"data\": {\"name\": \"Bob\", \"age\": 25}}" + )] + pub data: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ExportRequest { + /// Branch to export. Defaults to `main`. + pub branch: Option, + /// Restrict the export to these node/edge type names. Empty exports all types. + #[serde(default)] + pub type_names: Vec, + /// Restrict the export to these table keys. Empty exports all tables. + #[serde(default)] + pub table_keys: Vec, +} + +#[derive(Debug, Clone, Deserialize, IntoParams)] +pub struct SnapshotQuery { + pub branch: Option, +} + +#[derive(Debug, Clone, Deserialize, IntoParams)] +pub struct CommitListQuery { + pub branch: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct HealthOutput { + pub status: String, + pub version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_version: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum ErrorCode { + Unauthorized, + Forbidden, + BadRequest, + NotFound, + /// 405 Method Not Allowed — the route exists but the active server + /// mode doesn't serve this method (e.g. `GET /graphs` in single-graph + /// mode). Distinct from 404 so clients can tell "wrong context" from + /// "no such resource." + MethodNotAllowed, + Conflict, + /// 429 Too Many Requests — per-actor admission cap exceeded. + /// Clients should respect the `Retry-After` header. + TooManyRequests, + Internal, +} + +/// Structured details for a publisher-level OCC failure. Surfaces alongside +/// HTTP 409 when a write was rejected because the caller's pre-write view of +/// one table's manifest version was stale relative to the current head. The +/// expected/actual fields tell the client which table to refresh. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ManifestConflictOutput { + pub table_key: String, + pub expected: u64, + pub actual: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ErrorOutput { + pub error: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub merge_conflicts: Vec, + /// Set when the conflict is a publisher CAS rejection + /// (`ManifestConflictDetails::ExpectedVersionMismatch`). The caller's + /// pre-write view of `table_key` was at version `expected` but the + /// manifest is now at `actual`. Refresh and retry. + #[serde(skip_serializing_if = "Option::is_none")] + pub manifest_conflict: Option, +} + +pub fn snapshot_payload(branch: &str, snapshot: &Snapshot) -> SnapshotOutput { + let mut entries: Vec<_> = snapshot.entries().cloned().collect(); + entries.sort_by(|a, b| a.table_key.cmp(&b.table_key)); + let tables = entries + .iter() + .map(|entry| SnapshotTableOutput { + table_key: entry.table_key.clone(), + table_path: entry.table_path.clone(), + table_version: entry.table_version, + table_branch: entry.table_branch.clone(), + row_count: entry.row_count, + }) + .collect::>(); + SnapshotOutput { + branch: branch.to_string(), + manifest_version: snapshot.version(), + tables, + } +} + +pub fn schema_apply_output(uri: &str, result: SchemaApplyResult) -> SchemaApplyOutput { + SchemaApplyOutput { + uri: uri.to_string(), + supported: result.supported, + applied: result.applied, + step_count: result.steps.len(), + manifest_version: result.manifest_version, + steps: result.steps, + } +} + +pub fn commit_output(commit: &GraphCommit) -> CommitOutput { + CommitOutput { + graph_commit_id: commit.graph_commit_id.clone(), + manifest_branch: commit.manifest_branch.clone(), + manifest_version: commit.manifest_version, + parent_commit_id: commit.parent_commit_id.clone(), + merged_parent_commit_id: commit.merged_parent_commit_id.clone(), + actor_id: commit.actor_id.clone(), + created_at: commit.created_at, + } +} + +pub fn read_output(query_name: String, target: &ReadTarget, result: QueryResult) -> ReadOutput { + let columns = result + .schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .collect(); + ReadOutput { + query_name, + target: read_target_output(target), + row_count: result.num_rows(), + columns, + rows: result.to_rust_json(), + } +} + +pub fn ingest_output( + uri: &str, + result: &LoadResult, + mode: LoadMode, + actor_id: Option, +) -> IngestOutput { + IngestOutput { + uri: uri.to_string(), + branch: result.branch.clone(), + base_branch: result.base_branch.clone(), + branch_created: result.branch_created, + mode, + tables: result + .to_ingest_tables() + .into_iter() + .map(|table| IngestTableOutput { + table_key: table.table_key, + rows_loaded: table.rows_loaded, + }) + .collect(), + actor_id, + } +} + +pub fn read_target_output(target: &ReadTarget) -> ReadTargetOutput { + match target { + ReadTarget::Branch(branch) => ReadTargetOutput { + branch: Some(branch.clone()), + snapshot: None, + }, + ReadTarget::Snapshot(snapshot) => ReadTargetOutput { + branch: None, + snapshot: Some(snapshot.as_str().to_string()), + }, + } +} + +// ─── MR-668 — management endpoint shapes ────────────────────────────────── + +/// One entry in the response from `GET /graphs`. Cluster operators +/// consume this list to discover which graphs the server is currently +/// serving. The shape is intentionally minimal — `graph_id` and `uri` +/// are the only fields a routing client needs. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct GraphInfo { + pub graph_id: String, + pub uri: String, +} + +/// Response from `GET /graphs`. Lists every graph registered with the +/// server in alphabetical order by `graph_id` (sorted server-side so +/// clients get deterministic output across requests). +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct GraphListResponse { + pub graphs: Vec, +} diff --git a/crates/omnigraph-cli/Cargo.toml b/crates/omnigraph-cli/Cargo.toml index 1670fb2..e21b21e 100644 --- a/crates/omnigraph-cli/Cargo.toml +++ b/crates/omnigraph-cli/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.7.0" } omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.7.0" } +omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.7.0" } omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.7.0" } omnigraph-policy = { path = "../omnigraph-policy", version = "0.7.0" } omnigraph-server = { path = "../omnigraph-server", version = "0.7.0" } diff --git a/crates/omnigraph-cli/src/cli.rs b/crates/omnigraph-cli/src/cli.rs index 7b976b4..94bec5a 100644 --- a/crates/omnigraph-cli/src/cli.rs +++ b/crates/omnigraph-cli/src/cli.rs @@ -9,90 +9,159 @@ pub(crate) const DEFAULT_BEARER_TOKEN_ENV: &str = "OMNIGRAPH_BEARER_TOKEN"; #[command(name = "omnigraph")] #[command(about = "Omnigraph graph database CLI")] #[command(version = env!("CARGO_PKG_VERSION"), disable_version_flag = true)] +// Subcommands render in declaration order (clap can't print labeled headings +// between groups), so this legend names the capability each command needs — +// the user-facing vocabulary (RFC-011). `Plane` stays the internal classifier. +#[command(after_help = "\ +COMMANDS BY CAPABILITY:\n \ +any — run against a graph, served (--server / --profile) or embedded (--store / a \ +URI): query, mutate, load, branch, snapshot, export, commit, schema show/apply.\n \ +served — require a server: graphs.\n \ +direct — direct storage access; reject --server (init, optimize, repair, cleanup, \ +schema plan, lint).\n \ +control — manage or inspect a cluster (cluster via --config; policy & queries via \ +--cluster).\n \ +local — no explicit graph scope; local config & tooling: alias, embed, login, logout, profile, version.\n\ +See the 'Command capabilities' section of the CLI reference for which flags apply where.")] pub(crate) struct Cli { - /// Actor identity for direct-engine writes (MR-722). Overrides - /// `cli.actor` from `omnigraph.yaml`. When the configured policy - /// is in effect, Cedar evaluates this actor against the requested - /// action and scope; with policy configured but neither this flag - /// nor `cli.actor` set, the engine-layer footgun guard fires and - /// the write is denied (no silent bypass). Has no effect on remote - /// HTTP writes — those resolve their actor server-side from the - /// bearer token. + /// Actor id for direct-engine writes; overrides `cli.actor`. No effect on + /// remote writes (the server resolves the actor from the bearer token). + /// With a policy configured but no actor set, the write is denied — see + /// docs/user/operations/policy.md. #[arg(long = "as", global = true, value_name = "ACTOR")] pub(crate) as_actor: Option, - /// Target an operator-defined server by name (RFC-007): resolves to - /// its `url` from `servers:` in ~/.omnigraph/config.yaml. Exclusive - /// with a positional URI or `--target`. - #[arg(long, global = true, value_name = "NAME")] + /// Address a server by name (resolves to its `url` from `servers:` in + /// ~/.omnigraph/config.yaml) or by a literal `http(s)://` URL. Exclusive + /// with a positional URI. + #[arg(long, global = true, value_name = "NAME|URL")] pub(crate) server: Option, - /// Graph id on a multi-graph `--server` (appends `/graphs/` to - /// the server url). Requires --server. - #[arg(long, global = true, value_name = "GRAPH_ID", requires = "server")] + /// Select a graph within a multi-graph scope: on a `--server` it appends + /// `/graphs/` to the server url; on a `--cluster` it picks which + /// cluster graph to maintain. Rejected on a single-graph address (a + /// positional URI / `--store`). + #[arg(long, global = true, value_name = "GRAPH_ID")] pub(crate) graph: Option, + /// Select a named scope bundle (RFC-011) from `profiles:` in + /// ~/.omnigraph/config.yaml: fills in this command's omitted addressing + /// (server/cluster/store + default graph). Falls back to + /// $OMNIGRAPH_PROFILE. Config data, not state — every command resolves + /// scope fresh. + #[arg(long, global = true, value_name = "NAME")] + pub(crate) profile: Option, + + /// Address a single graph's storage directly (RFC-011): a `file://` / + /// `s3://` store URI. Explicit, ad-hoc direct access — bypasses any + /// server. Exclusive with a positional URI / `--server`. + #[arg(long, global = true, value_name = "URI")] + pub(crate) store: Option, + + /// Address a cluster-managed graph's storage for maintenance (RFC-011): + /// a cluster directory or storage-root URI — named via `clusters:` in + /// ~/.omnigraph/config.yaml, or a literal `file://`/`s3://` root. Pair + /// with `--graph ` to select the graph. Used by optimize / repair / + /// cleanup; exclusive with a positional URI / `--store` / `--server`. + #[arg(long, global = true, value_name = "DIR|URI")] + pub(crate) cluster: Option, + + /// Skip the confirmation prompt for a destructive write (`cleanup`, + /// overwrite `load`, `branch delete`) against a non-local scope (RFC-011 + /// Decision 9). Without it, a non-local destructive write prompts on a TTY + /// and refuses (errors) when there is no TTY or `--json` is set. + #[arg(long, global = true)] + pub(crate) yes: bool, + + /// Suppress the one-line resolved-write-target diagnostic that write + /// commands echo to stderr (RFC-011 Decision 9). + #[arg(long, global = true)] + pub(crate) quiet: bool, + #[command(subcommand)] pub(crate) command: Command, } #[derive(Debug, Subcommand)] pub(crate) enum Command { - /// Print the CLI version - Version, - /// Store a bearer token for a named server in ~/.omnigraph/credentials - /// (0600). Token from --token or one line on stdin: - /// `echo $TOKEN | omnigraph login prod`. The keyed token applies to - /// requests whose URL matches the server's `url` in the operator - /// config's `servers:` map. - Login { - /// Server name (keys the credential; declare its url under - /// `servers:` in ~/.omnigraph/config.yaml) - name: String, - /// The token. Prefer piping via stdin over this flag (shell - /// history). + // ── Data plane ── run against a graph (embedded or via --server). + /// Execute a read query against a branch or snapshot. + /// + /// Canonical read endpoint. The previous name `omnigraph read` is + /// kept as a visible alias and prints a one-line deprecation warning + /// when used. Pairs with `omnigraph mutate` on the write side. + #[command(visible_alias = "read")] + Query { + /// Query name. With no `--query`/`-e`, the stored query to invoke from + /// the catalog (served — addressed via --server/--profile). With + /// `--query`/`-e`, selects which query in that ad-hoc source to run. + name: Option, + /// Ad-hoc query file (a `.gq` you're authoring / break-glass). + #[arg(long, conflicts_with = "query_string")] + query: Option, + /// Inline ad-hoc GQ source — alternative to `--query `. + #[arg(short = 'e', long = "query-string", value_name = "GQ", conflicts_with = "query")] + query_string: Option, + #[command(flatten)] + params: ParamsArgs, + #[arg(long, conflicts_with = "snapshot")] + branch: Option, + #[arg(long, conflicts_with = "branch")] + snapshot: Option, + #[arg(long, conflicts_with = "json")] + format: Option, + #[arg(long, conflicts_with = "format")] + json: bool, + }, + /// Execute a graph mutation query against a branch. + /// + /// Canonical mutation endpoint. The previous name `omnigraph change` + /// is kept as a visible alias and prints a one-line deprecation + /// warning when used. Pairs with `omnigraph query` on the read side. + #[command(visible_alias = "change")] + Mutate { + /// Query name. With no `--query`/`-e`, the stored mutation to invoke + /// from the catalog (served — addressed via --server/--profile). With + /// `--query`/`-e`, selects which query in that ad-hoc source to run. + name: Option, + /// Ad-hoc mutation file (a `.gq` you're authoring / break-glass). + #[arg(long, conflicts_with = "query_string")] + query: Option, + /// Inline ad-hoc GQ source — alternative to `--query `. + #[arg(short = 'e', long = "query-string", value_name = "GQ", conflicts_with = "query")] + query_string: Option, + #[command(flatten)] + params: ParamsArgs, #[arg(long)] - token: Option, + branch: Option, #[arg(long)] json: bool, }, - /// Legacy-config tooling (RFC-008): split omnigraph.yaml into its - /// two destinations. - Config { - #[command(subcommand)] - command: ConfigCommand, - }, - /// Remove a named server's stored credential. Idempotent. - Logout { + /// Invoke an operator alias (RFC-011 Decision 4). + /// + /// An alias is a personal binding under `aliases:` in + /// ~/.omnigraph/config.yaml — name → (server, graph, stored-query name, + /// default params). `omnigraph alias [args]` invokes the bound + /// stored query on its server. Living in its own namespace, an alias can + /// never shadow or be shadowed by a built-in verb. Replaces the removed + /// `--alias` flag on `query`/`mutate`. + Alias { + /// Alias name (a key under `aliases:` in ~/.omnigraph/config.yaml). name: String, - #[arg(long)] + /// Positional args bound to the alias's declared `args` params, in order. + args: Vec, + #[command(flatten)] + params: ParamsArgs, + #[arg(long, conflicts_with = "json")] + format: Option, + #[arg(long, conflicts_with = "format")] json: bool, }, - /// Generate, clean, or refresh explicit seed embeddings - Embed(EmbedArgs), - /// Initialize a new graph from a schema - Init { - #[arg(long)] - schema: PathBuf, - /// Graph URI (local path or s3://) - uri: String, - /// Overwrite existing schema artifacts at the URI. Without - /// this flag, init refuses to touch a URI that already holds - /// `_schema.pg`, `_schema.ir.json`, or `__schema_state.json` - /// — closes the re-init footgun (MR-668 follow-up). With the - /// flag, the operator opts in to destructive semantics. - #[arg(long)] - force: bool, - }, /// Load data into a graph (local or remote) Load { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] data: PathBuf, /// Target branch (defaults to main). Without --from it must exist. #[arg(long)] @@ -109,14 +178,11 @@ pub(crate) enum Command { json: bool, }, /// Deprecated alias of `load --from ` (defaults: --mode merge, --from main) + #[command(hide = true)] Ingest { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] data: PathBuf, #[arg(long)] branch: Option, @@ -132,11 +198,99 @@ pub(crate) enum Command { #[command(subcommand)] command: BranchCommand, }, + /// Show graph snapshot + Snapshot { + /// Graph URI + uri: Option, + #[arg(long)] + branch: Option, + #[arg(long)] + json: bool, + }, + /// Export a full graph snapshot as JSONL + Export { + /// Graph URI + uri: Option, + #[arg(long)] + branch: Option, + #[arg(long, hide = true)] + jsonl: bool, + #[arg(long = "type")] + type_names: Vec, + #[arg(long = "table")] + table_keys: Vec, + }, + /// Commit history operations + Commit { + #[command(subcommand)] + command: CommitCommand, + }, /// Schema planning operations Schema { #[command(subcommand)] command: SchemaCommand, }, + /// Manage graphs on a multi-graph server (MR-668) + Graphs { + #[command(subcommand)] + command: GraphsCommand, + }, + + // ── Storage / local graph ops ── direct storage or local files; reject --server. + /// Initialize a new graph from a schema + Init { + #[arg(long)] + schema: PathBuf, + /// Graph URI (local path or s3://) + uri: String, + /// Overwrite existing schema artifacts at the URI. Without + /// this flag, init refuses to touch a URI that already holds + /// `_schema.pg`, `_schema.ir.json`, or `__schema_state.json` + /// — closes the re-init footgun (MR-668 follow-up). With the + /// flag, the operator opts in to destructive semantics. + #[arg(long)] + force: bool, + }, + /// Compact small Lance fragments in every table of the graph + Optimize { + /// Graph URI + uri: Option, + #[arg(long)] + json: bool, + }, + /// Classify and explicitly repair manifest/head drift + Repair { + /// Graph URI + uri: Option, + /// Publish verified maintenance drift. Without this flag, repair only + /// previews what it would do. + #[arg(long)] + confirm: bool, + /// Also publish suspicious or unverifiable drift. Requires + /// `--confirm`; use only after operator review. + #[arg(long, requires = "confirm")] + force: bool, + #[arg(long)] + json: bool, + }, + /// Remove old Lance versions from every table of the graph (destructive) + Cleanup { + /// Graph URI + uri: Option, + /// Number of recent versions to keep per table. Either `--keep` or + /// `--older-than` (or both) must be set. + #[arg(long)] + keep: Option, + /// Only remove versions older than this duration. Accepts Go-style + /// durations: `7d`, `24h`, `90m`. At least one of --keep / --older-than. + #[arg(long)] + older_than: Option, + /// Required to actually run; without it, prints what would be removed + #[arg(long)] + confirm: bool, + #[arg(long)] + json: bool, + }, /// Validate queries against a schema (offline) or repo (repo-backed). /// /// Canonical name is `lint` (matches the `omnigraph_compiler::lint` @@ -152,10 +306,6 @@ pub(crate) enum Command { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] query: PathBuf, #[arg(long)] schema: Option, @@ -167,179 +317,63 @@ pub(crate) enum Command { #[command(subcommand)] command: QueriesCommand, }, - /// Show graph snapshot - Snapshot { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] - branch: Option, - #[arg(long)] - json: bool, - }, - /// Export a full graph snapshot as JSONL - Export { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] - branch: Option, - #[arg(long, hide = true)] - jsonl: bool, - #[arg(long = "type")] - type_names: Vec, - #[arg(long = "table")] - table_keys: Vec, - }, - /// Commit history operations - Commit { - #[command(subcommand)] - command: CommitCommand, - }, - /// Execute a read query against a branch or snapshot. - /// - /// Canonical read endpoint. The previous name `omnigraph read` is - /// kept as a visible alias and prints a one-line deprecation warning - /// when used. Pairs with `omnigraph mutate` on the write side. - #[command(visible_alias = "read")] - Query { - /// Graph URI - #[arg(long)] - uri: Option, - #[arg(hide = true)] - legacy_uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long, conflicts_with_all = ["query", "query_string"])] - alias: Option, - #[arg(long, conflicts_with_all = ["alias", "query_string"])] - query: Option, - /// Inline GQ source — alternative to `--query ` and `--alias `. - #[arg(short = 'e', long = "query-string", value_name = "GQ", conflicts_with_all = ["query", "alias"])] - query_string: Option, - #[arg(long)] - name: Option, - #[command(flatten)] - params: ParamsArgs, - #[arg(long, conflicts_with = "snapshot")] - branch: Option, - #[arg(long, conflicts_with = "branch")] - snapshot: Option, - #[arg(long, conflicts_with = "json")] - format: Option, - #[arg(long, conflicts_with = "format")] - json: bool, - #[arg()] - alias_args: Vec, - }, - /// Execute a graph mutation query against a branch. - /// - /// Canonical mutation endpoint. The previous name `omnigraph change` - /// is kept as a visible alias and prints a one-line deprecation - /// warning when used. Pairs with `omnigraph query` on the read side. - #[command(visible_alias = "change")] - Mutate { - /// Graph URI - #[arg(long)] - uri: Option, - #[arg(hide = true)] - legacy_uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long, conflicts_with_all = ["query", "query_string"])] - alias: Option, - #[arg(long, conflicts_with_all = ["alias", "query_string"])] - query: Option, - /// Inline GQ source — alternative to `--query ` and `--alias `. - #[arg(short = 'e', long = "query-string", value_name = "GQ", conflicts_with_all = ["query", "alias"])] - query_string: Option, - #[arg(long)] - name: Option, - #[command(flatten)] - params: ParamsArgs, - #[arg(long)] - branch: Option, - #[arg(long)] - json: bool, - #[arg()] - alias_args: Vec, - }, - /// Policy administration and diagnostics - Policy { - #[command(subcommand)] - command: PolicyCommand, - }, - /// Compact small Lance fragments in every table of the graph - Optimize { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] - json: bool, - }, - /// Classify and explicitly repair manifest/head drift - Repair { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - /// Publish verified maintenance drift. Without this flag, repair only - /// previews what it would do. - #[arg(long)] - confirm: bool, - /// Also publish suspicious or unverifiable drift. Requires - /// `--confirm`; use only after operator review. - #[arg(long, requires = "confirm")] - force: bool, - #[arg(long)] - json: bool, - }, - /// Remove old Lance versions from every table of the graph (destructive) - Cleanup { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - /// Number of recent versions to keep per table. Either `--keep` or - /// `--older-than` (or both) must be set. - #[arg(long)] - keep: Option, - /// Only remove versions older than this duration. Accepts Go-style - /// durations: `7d`, `24h`, `90m`. At least one of --keep / --older-than. - #[arg(long)] - older_than: Option, - /// Required to actually run; without it, prints what would be removed - #[arg(long)] - confirm: bool, - #[arg(long)] - json: bool, - }, + + // ── Control plane ── manage a cluster directory (--config ). /// Validate and plan read-only cluster configuration. Cluster { #[command(subcommand)] command: ClusterCommand, }, - /// Manage graphs on a multi-graph server (MR-668) - Graphs { + + /// Policy administration and diagnostics against a cluster's applied bundles + Policy { #[command(subcommand)] - command: GraphsCommand, + command: PolicyCommand, + }, + /// Generate, clean, or refresh explicit seed embeddings + Embed(EmbedArgs), + /// Store a bearer token for a named server (0600 credentials file). Token + /// via --token or piped on stdin; see the CLI reference for token resolution. + Login { + /// Server name (keys the credential; declare its url under + /// `servers:` in ~/.omnigraph/config.yaml) + name: String, + /// The token. Prefer piping via stdin over this flag (shell + /// history). + #[arg(long)] + token: Option, + #[arg(long)] + json: bool, + }, + /// Remove a named server's stored credential. Idempotent. + Logout { + name: String, + #[arg(long)] + json: bool, + }, + /// Inspect the scope profiles in ~/.omnigraph/config.yaml (read-only). + Profile { + #[command(subcommand)] + command: ProfileCommand, + }, + /// Print the CLI version + Version, +} + +#[derive(Debug, Subcommand)] +pub(crate) enum ProfileCommand { + /// List the profiles defined in ~/.omnigraph/config.yaml. + List { + #[arg(long)] + json: bool, + }, + /// Show a profile's resolved scope. With no name, shows the active + /// (`$OMNIGRAPH_PROFILE`) profile, else the flat operator defaults. + Show { + /// Profile name (optional). + name: Option, + #[arg(long)] + json: bool, }, } @@ -439,10 +473,6 @@ pub(crate) enum GraphsCommand { #[arg(long)] uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] json: bool, }, } @@ -455,10 +485,6 @@ pub(crate) enum BranchCommand { #[arg(long)] uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] from: Option, name: String, #[arg(long)] @@ -470,10 +496,6 @@ pub(crate) enum BranchCommand { #[arg(long)] uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] json: bool, }, /// Delete a branch @@ -481,10 +503,6 @@ pub(crate) enum BranchCommand { /// Graph URI #[arg(long)] uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, name: String, #[arg(long)] json: bool, @@ -494,10 +512,6 @@ pub(crate) enum BranchCommand { /// Graph URI #[arg(long)] uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, source: String, #[arg(long)] into: Option, @@ -513,10 +527,6 @@ pub(crate) enum SchemaCommand { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] schema: PathBuf, #[arg(long)] json: bool, @@ -531,10 +541,6 @@ pub(crate) enum SchemaCommand { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] schema: PathBuf, #[arg(long)] json: bool, @@ -556,10 +562,6 @@ pub(crate) enum SchemaCommand { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] json: bool, }, } @@ -572,10 +574,6 @@ pub(crate) enum CommitCommand { /// Graph URI uri: Option, #[arg(long)] - target: Option, - #[arg(long)] - config: Option, - #[arg(long)] branch: Option, #[arg(long)] json: bool, @@ -585,10 +583,6 @@ pub(crate) enum CommitCommand { /// Graph URI #[arg(long)] uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, commit_id: String, #[arg(long)] json: bool, @@ -597,20 +591,24 @@ pub(crate) enum CommitCommand { #[derive(Debug, Subcommand)] pub(crate) enum PolicyCommand { - /// Validate policy YAML and compiled Cedar policy state - Validate { - #[arg(long)] - config: Option, - }, - /// Run declarative policy tests from policy.tests.yaml + /// Compile and validate the Cedar policy bundle(s) applied in a cluster. + /// + /// Sources the bundle(s) from the cluster's applied policies + /// (`--cluster `); pass the global `--graph ` to pick one + /// graph's bundle when several apply. + Validate {}, + /// Run declarative policy tests against a cluster's applied bundle. + /// + /// The cluster model has no per-bundle tests file, so the cases are + /// supplied explicitly with `--tests ` and checked against the + /// bundle selected by `--cluster` (+ optional `--graph`). Test { + /// Path to a policy.tests.yaml file. #[arg(long)] - config: Option, + tests: PathBuf, }, - /// Explain one policy decision locally + /// Explain one policy decision against a cluster's applied bundle. Explain { - #[arg(long)] - config: Option, #[arg(long)] actor: String, #[arg(long)] @@ -624,28 +622,19 @@ pub(crate) enum PolicyCommand { #[derive(Debug, Subcommand)] pub(crate) enum QueriesCommand { - /// Type-check the stored-query registry against the live schema. + /// Type-check a cluster's stored-query registry against its schemas. /// - /// Distinct from `omnigraph lint` (which lints one `.gq` file): - /// this validates the whole `queries:` registry — opening the graph - /// to read its schema and confirming every stored query still - /// type-checks. Exits non-zero on any breakage. + /// Distinct from `omnigraph lint` (which lints one `.gq` file): this + /// validates the whole `queries:` registry of a cluster (`--cluster + /// `, optional `--graph `) by reading each graph's applied + /// schema and confirming every stored query still type-checks. Exits + /// non-zero on any breakage. Validate { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, #[arg(long)] json: bool, }, - /// List the registered stored queries (name, MCP exposure, params). + /// List a cluster's registered stored queries (name, params). List { - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, #[arg(long)] json: bool, }, @@ -676,7 +665,6 @@ impl From for LoadMode { } } } - impl CliLoadMode { pub(crate) fn as_str(self) -> &'static str { match self { @@ -686,21 +674,3 @@ impl CliLoadMode { } } } - -#[derive(Debug, Subcommand)] -pub(crate) enum ConfigCommand { - /// Propose (and with --write, apply) the RFC-008 split of a legacy - /// omnigraph.yaml: team half -> a ready-to-review cluster.yaml, - /// personal half -> ~/.omnigraph/config.yaml (key-level merge, - /// existing entries always win). Touches nothing without --write. - Migrate { - /// Path to the legacy omnigraph.yaml (default: ./omnigraph.yaml) - #[arg(long)] - config: Option, - /// Apply the split instead of only printing it - #[arg(long)] - write: bool, - #[arg(long)] - json: bool, - }, -} diff --git a/crates/omnigraph-cli/src/client.rs b/crates/omnigraph-cli/src/client.rs new file mode 100644 index 0000000..7151f5e --- /dev/null +++ b/crates/omnigraph-cli/src/client.rs @@ -0,0 +1,821 @@ +//! `GraphClient` — the one place the embedded-vs-remote split lives +//! (RFC-009 Phase 3). A CLI command body calls a verb method; the +//! enum routes to the engine (local URI) or HTTP (remote URI). The +//! 15 per-command `if graph.is_remote { … } else { … }` forks collapse +//! into two arms here. +//! +//! Phase 3a put the factory + the uniform read verbs in place. Phase 3b +//! adds the data-plane writes (`load`/`ingest`/`mutate`/`branch_*`/ +//! `apply_schema`) and `query`. The wrinkle 3a deferred: writes open the +//! local engine WITH policy (`open_local_db_with_policy`) and carry a +//! resolved actor, while reads/`query` open WITHOUT policy. So the +//! `Embedded` variant grows an optional policy context (`graph`/`actor`) +//! and a second factory (`resolve_with_policy`) fills it; `resolve()` +//! leaves it empty. The open path picks itself from whether `graph` is +//! set, preserving today's two behaviors exactly. Export + graphs-list +//! land in 3c. Behavior is unchanged per verb — the Phase-1 parity matrix +//! is the referee and stays textually unchanged. +//! +//! Enum, not a trait (RFC sketch said "trait"): only two variants ever, +//! and inherent async methods sidestep `async_trait` boxing plus the +//! `apply_schema` catalog-validator closure that is not object-safe. +//! Same one-body-two-impls collapse, less ceremony. + +use std::io::Write; + +use color_eyre::Result; +use color_eyre::eyre::bail; +use omnigraph::db::{Omnigraph, ReadTarget}; +use omnigraph_api_types::{ + BranchCreateOutput, BranchCreateRequest, BranchDeleteOutput, BranchListOutput, + BranchMergeOutput, BranchMergeRequest, ChangeOutput, CommitListOutput, CommitOutput, + ErrorOutput, ExportRequest, GraphListResponse, IngestOutput, IngestRequest, + InvokeStoredQueryRequest, ReadOutput, + ReadRequest, SchemaApplyOutput, SchemaApplyRequest, SchemaOutput, SnapshotOutput, commit_output, + ingest_output, read_output, schema_apply_output, snapshot_payload, +}; +use omnigraph_compiler::catalog::Catalog; +use reqwest::Method; +use serde_json::Value; + +use crate::cli::CliLoadMode; +use crate::helpers::{ + apply_bearer_token, apply_server_flag, build_http_client, is_remote_uri, + legacy_change_request_body, query_params_from_json, + remote_json, remote_url, resolve_cli_actor, resolve_cli_graph, resolve_remote_bearer_token, + resolve_server_flag, select_named_query, +}; +use crate::output::{LoadOutput, load_output_from_result, load_output_from_tables}; + +pub(crate) enum GraphClient { + /// Local engine at `uri`. Reads (`resolve()`) leave `actor` empty; + /// writes (`resolve_with_policy()`) attribute the resolved actor. + /// Direct-store access carries no Cedar policy (RFC-011: policy lives + /// in the cluster/server, not in per-operator addressing). + Embedded { + uri: String, + actor: Option, + }, + /// Remote HTTP server. The actor is resolved server-side from the + /// token; the client never sets identity. + Remote { + http: reqwest::Client, + base_url: String, + token: Option, + }, +} + +/// RFC-011 Decision 7: a server scope that selects no graph (no `--graph`, no +/// `default_graph`) must not silently fall through to the bare server URL when +/// the server is multi-graph. Best-effort probe `GET /graphs`: a populated list +/// forces `--graph` (listing the candidates); a single-graph/flat server (405), +/// a policy-gated `/graphs`, or an unreachable server all proceed — the bare URL +/// is then correct, or the real request surfaces the failure. Only fires on the +/// no-graph path, so a `--graph`/`default_graph` happy path does no extra I/O. +async fn require_graph_for_multi_graph_server( + scope: &crate::scope::ResolvedScope, +) -> Result<()> { + let (Some(server), None) = (scope.server.as_deref(), scope.graph.as_deref()) else { + return Ok(()); + }; + let Some(base) = resolve_server_flag(Some(server), None)? else { + return Ok(()); + }; + let token = resolve_remote_bearer_token(Some(&base))?; + let probe = GraphClient::Remote { + http: build_http_client()?, + base_url: base, + token, + }; + if let Ok(resp) = probe.list_graphs().await { + if !resp.graphs.is_empty() { + let ids: Vec<&str> = resp.graphs.iter().map(|g| g.graph_id.as_str()).collect(); + bail!( + "server scope '{server}' has {} {}: [{}]; pass --graph to select one \ + (or set `default_graph` in your operator config)", + ids.len(), + if ids.len() == 1 { "graph" } else { "graphs" }, + ids.join(", ") + ); + } + } + Ok(()) +} + +/// A remote graph must be addressed with `--server` (RFC-011): a positional or +/// `--uri` `http(s)://` URL no longer auto-dispatches to a server. A remote URL +/// produced by a server scope (`via_server`) is fine. +fn reject_positional_remote(via_server: bool, uri: &str) -> Result<()> { + if !via_server && is_remote_uri(uri) { + bail!( + "a remote graph must be addressed with `--server ` — a positional \ + (or `--uri`) http(s):// URL no longer dispatches to a server" + ); + } + Ok(()) +} + +impl GraphClient { + /// Resolve the addressing (positional URI / `--target` / `--server`) + /// and credential once, then pick the variant by URI scheme — the + /// single branch point that replaces every per-command `is_remote` + /// fork. Mirrors the read verbs' current preamble (`resolve_uri` + /// path, not the policy-bearing `resolve_cli_graph`). Used by reads + /// and `query` (which opens without policy, like the reads). + pub(crate) async fn resolve( + server: Option<&str>, + graph: Option<&str>, + uri: Option, + profile: Option<&str>, + store: Option<&str>, + ) -> Result { + // RFC-011: a scope (profile / --store / operator defaults) may stand in + // for omitted addressing. The explicit branch passes server/graph/uri + // straight through, so existing invocations are unchanged. + let scope = crate::scope::resolve_scope( + &crate::operator::load_operator_config()?, + crate::planes::Capability::Any, + crate::scope::ScopeFlags { profile, store, server, cluster: None, graph, uri }, + )?; + require_graph_for_multi_graph_server(&scope).await?; + let (server, graph, uri) = ( + scope.server.as_deref(), + scope.graph.as_deref(), + scope.uri, + ); + let via_server = server.is_some(); + let uri = apply_server_flag(server, graph, uri)?; + let token = resolve_remote_bearer_token(uri.as_deref())?; + let uri = crate::helpers::resolve_uri(uri)?; + reject_positional_remote(via_server, &uri)?; + if is_remote_uri(&uri) { + Ok(GraphClient::Remote { + http: build_http_client()?, + base_url: uri, + token, + }) + } else { + Ok(GraphClient::Embedded { uri, actor: None }) + } + } + + /// Write-path factory: the same addressing/credential resolution as + /// `resolve()`, but through the stricter `resolve_cli_graph` (which + /// carries `policy_file`/`graph_id`/`selected`), and with the actor + /// resolved up front. The embedded arm then opens WITH policy. The + /// resolution order matches the write arms exactly: server flag → + /// bearer token → graph. + pub(crate) async fn resolve_with_policy( + server: Option<&str>, + graph: Option<&str>, + uri: Option, + cli_as: Option<&str>, + profile: Option<&str>, + store: Option<&str>, + ) -> Result { + // RFC-011 scope translation (see `resolve`); explicit addressing passes + // through unchanged. + let scope = crate::scope::resolve_scope( + &crate::operator::load_operator_config()?, + crate::planes::Capability::Any, + crate::scope::ScopeFlags { profile, store, server, cluster: None, graph, uri }, + )?; + require_graph_for_multi_graph_server(&scope).await?; + let (server, graph, uri) = ( + scope.server.as_deref(), + scope.graph.as_deref(), + scope.uri, + ); + let via_server = server.is_some(); + let uri = apply_server_flag(server, graph, uri)?; + let token = resolve_remote_bearer_token(uri.as_deref())?; + let resolved = resolve_cli_graph(uri)?; + reject_positional_remote(via_server, &resolved.uri)?; + if resolved.is_remote { + // A served write resolves the actor server-side from the bearer + // token; `--as` cannot set identity here and is rejected. + if cli_as.is_some() { + bail!( + "`--as` is not allowed on a served write — the server resolves the actor \ + from the bearer token. Remove `--as`, or run the write directly against \ + storage with `--store `." + ); + } + Ok(GraphClient::Remote { + http: build_http_client()?, + base_url: resolved.uri, + token, + }) + } else { + let actor = resolve_cli_actor(cli_as)?; + Ok(GraphClient::Embedded { + uri: resolved.uri, + actor, + }) + } + } + + /// The graph URI (local path / remote base URL) this client addresses. + pub(crate) fn uri(&self) -> &str { + match self { + GraphClient::Embedded { uri, .. } => uri, + GraphClient::Remote { base_url, .. } => base_url, + } + } + + pub(crate) fn is_remote(&self) -> bool { + matches!(self, GraphClient::Remote { .. }) + } + + /// Open the local engine. Direct-store access carries no Cedar policy + /// (RFC-011), so both read and write paths open bare; the actor is still + /// attributed on the write via the `_as` engine APIs. + async fn open_embedded(uri: &str) -> Result { + Ok(Omnigraph::open(uri).await?) + } + + pub(crate) async fn branch_list(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::GET, + remote_url(base_url, &["branches"], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, .. } => { + let db = Omnigraph::open(uri).await?; + let mut branches = db.branch_list().await?; + branches.sort(); + Ok(BranchListOutput { branches }) + } + } + } + + pub(crate) async fn snapshot(&self, branch: &str) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::GET, + remote_url(base_url, &["snapshot"], &[("branch", branch)])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, .. } => { + let db = Omnigraph::open(uri).await?; + let snapshot = db.snapshot_of(ReadTarget::branch(branch)).await?; + Ok(snapshot_payload(branch, &snapshot)) + } + } + } + + pub(crate) async fn schema_source(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::GET, + remote_url(base_url, &["schema"], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, .. } => { + let db = Omnigraph::open(uri).await?; + Ok(SchemaOutput { + schema_source: db.schema_source().to_string(), + }) + } + } + } + + pub(crate) async fn list_commits(&self, branch: Option<&str>) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + let url = match branch { + Some(branch) => remote_url(base_url, &["commits"], &[("branch", branch)])?, + None => remote_url(base_url, &["commits"], &[])?, + }; + remote_json(http, Method::GET, url, None, token.as_deref()).await + } + GraphClient::Embedded { uri, .. } => { + let db = Omnigraph::open(uri).await?; + let commits = db + .list_commits(branch) + .await? + .iter() + .map(commit_output) + .collect::>(); + Ok(CommitListOutput { commits }) + } + } + } + + pub(crate) async fn get_commit(&self, commit_id: &str) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::GET, + remote_url(base_url, &["commits", commit_id], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, .. } => { + let db = Omnigraph::open(uri).await?; + Ok(commit_output(&db.get_commit(commit_id).await?)) + } + } + } + + /// `load` — bulk-load `data` (a file path) onto `branch`, forking from + /// `from` if missing. Returns the CLI `LoadOutput`; each arm keeps its + /// own mapping (remote sums the wire `IngestOutput.tables`, embedded + /// reads the richer `LoadResult` directly) — preserved exactly. + pub(crate) async fn load( + &self, + branch: &str, + from: Option<&str>, + data: &str, + mode: CliLoadMode, + ) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + let data = std::fs::read_to_string(data)?; + // RFC-009 Phase 5: the canonical `load` verb targets the + // canonical `/load` route (the deprecated `ingest` verb below + // still rides `/ingest`). + let output = remote_json::( + http, + Method::POST, + remote_url(base_url, &["load"], &[])?, + Some(serde_json::to_value(IngestRequest { + branch: Some(branch.to_string()), + from: from.map(ToOwned::to_owned), + mode: Some(mode.into()), + data, + })?), + token.as_deref(), + ) + .await?; + Ok(load_output_from_tables(base_url, branch, mode.as_str(), &output)) + } + GraphClient::Embedded { uri, actor } => { + let db = Self::open_embedded(uri).await?; + let result = db + .load_file_as(branch, from, data, mode.into(), actor.as_deref()) + .await?; + Ok(load_output_from_result(uri, branch, mode.as_str(), &result)) + } + } + } + + /// `ingest` — the deprecated alias of `load`. Same operation, but the + /// surfaced shape is the wire `IngestOutput` (printed by + /// `print_ingest_human`), so it is its own method. The embedded arm + /// echoes `actor_id: None` in the output exactly as the legacy arm did + /// (the actor is still attributed on the commit via `load_file_as`). + pub(crate) async fn ingest( + &self, + branch: &str, + from: &str, + data: &str, + mode: CliLoadMode, + ) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + let data = std::fs::read_to_string(data)?; + remote_json( + http, + Method::POST, + remote_url(base_url, &["ingest"], &[])?, + Some(serde_json::to_value(IngestRequest { + branch: Some(branch.to_string()), + from: Some(from.to_string()), + mode: Some(mode.into()), + data, + })?), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, actor } => { + let db = Self::open_embedded(uri).await?; + let result = db + .load_file_as(branch, Some(from), data, mode.into(), actor.as_deref()) + .await?; + Ok(ingest_output(uri, &result, mode.into(), None)) + } + } + } + + /// `mutate` — run a change query against `branch`. Folds + /// `execute_change` / `execute_change_remote` + the legacy request body. + pub(crate) async fn mutate( + &self, + branch: &str, + query_source: &str, + query_name: Option<&str>, + params_json: Option<&Value>, + ) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::POST, + remote_url(base_url, &["change"], &[])?, + Some(legacy_change_request_body( + query_source, + query_name, + branch, + params_json, + )), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, actor } => { + let (selected_name, query_params) = select_named_query(query_source, query_name)?; + let params = query_params_from_json(&query_params, params_json)?; + let db = Self::open_embedded(uri).await?; + let actor = actor.as_deref(); + let result = db + .mutate_as(branch, query_source, &selected_name, ¶ms, actor) + .await?; + Ok(ChangeOutput { + branch: branch.to_string(), + query_name: selected_name, + affected_nodes: result.affected_nodes, + affected_edges: result.affected_edges, + actor_id: actor.map(String::from), + }) + } + } + } + + /// `query` — run a read query against `target`. Folds `execute_read` / + /// `execute_read_remote`; the embedded arm opens WITHOUT policy (reads + /// never attach one), so this verb resolves via `resolve()`. + pub(crate) async fn query( + &self, + target: ReadTarget, + query_source: &str, + query_name: Option<&str>, + params_json: Option<&Value>, + ) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + let (branch, snapshot) = match &target { + ReadTarget::Branch(branch) => (Some(branch.clone()), None), + ReadTarget::Snapshot(snapshot) => (None, Some(snapshot.as_str().to_string())), + }; + remote_json( + http, + Method::POST, + remote_url(base_url, &["read"], &[])?, + Some(serde_json::to_value(ReadRequest { + query_source: query_source.to_string(), + query_name: query_name.map(ToOwned::to_owned), + params: params_json.cloned(), + branch, + snapshot, + })?), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, .. } => { + let (selected_name, query_params) = select_named_query(query_source, query_name)?; + let params = query_params_from_json(&query_params, params_json)?; + let db = Self::open_embedded(uri).await?; + let result = db + .query(target.clone(), query_source, &selected_name, ¶ms) + .await?; + Ok(read_output(selected_name, &target, result)) + } + } + } + + /// `invoke_named` — run a stored query **by catalog name** (RFC-011 D3). + /// Served-only: the catalog is server-owned, so a `--store` (embedded) + /// scope has nothing to resolve the name against. `expect_mutation` carries + /// the verb's asserted kind; the server rejects a mismatch (400) before + /// running, so the response is exactly the expected envelope — the caller + /// deserializes it as the concrete `T` (`ReadOutput` for `query`, + /// `ChangeOutput` for `mutate`), sidestepping the untagged wire enum. + pub(crate) async fn invoke_named( + &self, + name: &str, + expect_mutation: bool, + params_json: Option<&Value>, + branch: Option, + snapshot: Option, + ) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + let body = InvokeStoredQueryRequest { + params: params_json.cloned(), + branch, + snapshot, + expect_mutation: Some(expect_mutation), + }; + remote_json( + http, + Method::POST, + remote_url(base_url, &["queries", name], &[])?, + Some(serde_json::to_value(body)?), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { .. } => bail!( + "by-name invocation needs a server (the stored-query catalog is \ + server-owned); use -e '' or --query for an ad-hoc query \ + against --store, or address a server with --server / --profile" + ), + } + } + + pub(crate) async fn branch_create_from( + &self, + from: &str, + name: &str, + ) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::POST, + remote_url(base_url, &["branches"], &[])?, + Some(serde_json::to_value(BranchCreateRequest { + from: Some(from.to_string()), + name: name.to_string(), + })?), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, actor } => { + let db = Self::open_embedded(uri).await?; + let actor = actor.as_deref(); + db.branch_create_from_as(ReadTarget::branch(from), name, actor) + .await?; + Ok(BranchCreateOutput { + uri: uri.clone(), + from: from.to_string(), + name: name.to_string(), + actor_id: actor.map(String::from), + }) + } + } + } + + pub(crate) async fn branch_delete(&self, name: &str) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::DELETE, + remote_url(base_url, &["branches", name], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, actor } => { + let db = Self::open_embedded(uri).await?; + let actor = actor.as_deref(); + db.branch_delete_as(name, actor).await?; + Ok(BranchDeleteOutput { + uri: uri.clone(), + name: name.to_string(), + actor_id: actor.map(String::from), + }) + } + } + } + + pub(crate) async fn branch_merge(&self, source: &str, into: &str) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::POST, + remote_url(base_url, &["branches", "merge"], &[])?, + Some(serde_json::to_value(BranchMergeRequest { + source: source.to_string(), + target: Some(into.to_string()), + })?), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, actor } => { + let db = Self::open_embedded(uri).await?; + let actor = actor.as_deref(); + let outcome = db.branch_merge_as(source, into, actor).await?; + Ok(BranchMergeOutput { + source: source.to_string(), + target: into.to_string(), + outcome: outcome.into(), + actor_id: actor.map(String::from), + }) + } + } + } + + /// `apply_schema` — apply `schema_source`. The embedded arm runs the + /// caller's catalog validator (stored-query registry check) inside the + /// engine's `apply_schema_as_with_catalog_check`; the remote arm runs + /// the server's own check and IGNORES `validate`. The `impl FnOnce` + /// validator is exactly why this is an enum, not a trait (non-object- + /// safe). + pub(crate) async fn apply_schema( + &self, + schema_source: &str, + allow_data_loss: bool, + validate: F, + ) -> Result + where + F: FnOnce(&Catalog) -> omnigraph::error::Result<()>, + { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + // MR-694 PR B: SchemaApplyRequest carries allow_data_loss so + // Hard-mode drops are no longer CLI-only; the server's + // `server_schema_apply` honors it (and runs its own catalog + // check, so `validate` does not apply here). + remote_json::( + http, + Method::POST, + remote_url(base_url, &["schema", "apply"], &[])?, + Some(serde_json::to_value(SchemaApplyRequest { + schema_source: schema_source.to_string(), + allow_data_loss, + })?), + token.as_deref(), + ) + .await + } + GraphClient::Embedded { uri, actor } => { + let db = Self::open_embedded(uri).await?; + let result = db + .apply_schema_as_with_catalog_check( + schema_source, + omnigraph::db::SchemaApplyOptions { allow_data_loss }, + actor.as_deref(), + validate, + ) + .await?; + Ok(schema_apply_output(uri, result)) + } + } + } + + /// `export` — stream the branch as JSONL into `writer`. The streaming + /// shape (a `W: Write`, not a returned DTO) is why this lands in 3c + /// rather than 3b. Opens WITHOUT policy (like reads), so it is reached + /// via `resolve()`; the Embedded arm opens bare. The Remote arm streams + /// the chunked response body straight through (no buffering the whole + /// export in memory). + pub(crate) async fn export( + &self, + branch: &str, + type_names: &[String], + table_keys: &[String], + writer: &mut W, + ) -> Result<()> { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + let request = apply_bearer_token( + http.request(Method::POST, remote_url(base_url, &["export"], &[])?), + token.as_deref(), + ) + .json(&ExportRequest { + branch: Some(branch.to_string()), + type_names: type_names.to_vec(), + table_keys: table_keys.to_vec(), + }); + let mut response = request.send().await?; + let status = response.status(); + if !status.is_success() { + let text = response.text().await?; + if let Ok(error) = serde_json::from_str::(&text) { + bail!(error.error); + } + bail!("server returned {}: {}", status, text); + } + while let Some(chunk) = response.chunk().await? { + writer.write_all(&chunk)?; + } + writer.flush()?; + Ok(()) + } + GraphClient::Embedded { uri, .. } => { + let db = Omnigraph::open(uri).await?; + db.export_jsonl_to_writer(branch, type_names, table_keys, writer) + .await?; + writer.flush()?; + Ok(()) + } + } + } + + /// `graphs list` — enumerate the graphs a remote multi-graph server + /// serves (`GET /graphs`). Remote-only by design: there is no local + /// enumeration endpoint, so the Embedded arm fails loudly. Routing it + /// through the enum still buys the shared `resolve()` addressing/token + /// preamble. + pub(crate) async fn list_graphs(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::GET, + remote_url(base_url, &["graphs"], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { .. } => bail!( + "`omnigraph graphs list` requires a remote multi-graph server \ + (--server ). To enumerate the graphs in a cluster, run \ + `omnigraph cluster status --config `." + ), + } + } +} diff --git a/crates/omnigraph-cli/src/embed.rs b/crates/omnigraph-cli/src/embed.rs index 2e1c6d9..a0603b7 100644 --- a/crates/omnigraph-cli/src/embed.rs +++ b/crates/omnigraph-cli/src/embed.rs @@ -9,8 +9,6 @@ use omnigraph::embedding::EmbeddingClient; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value, json}; -const DEFAULT_EMBED_MODEL: &str = "gemini-embedding-2-preview"; - #[derive(Debug, Args, Clone)] pub(crate) struct EmbedArgs { /// Seed manifest path @@ -85,8 +83,6 @@ impl EmbedMode { #[derive(Debug, Clone, Deserialize)] struct EmbedSpec { - #[serde(default = "default_embed_model")] - model: String, dimension: usize, types: BTreeMap, } @@ -180,13 +176,6 @@ pub(crate) fn resolve_embed_job(args: &EmbedArgs) -> Result { (input, output, spec) }; - if spec.model != DEFAULT_EMBED_MODEL { - bail!( - "only {} is supported for explicit seed embeddings right now", - DEFAULT_EMBED_MODEL - ); - } - Ok(EmbedJob { input, output, @@ -305,7 +294,14 @@ pub(crate) async fn run_embed_job(job: &EmbedJob) -> Result { cleaned_rows, mode: job.mode.as_str(!job.selectors.is_empty()), dimension: job.spec.dimension, - model: job.spec.model.clone(), + // The embedding model is resolved solely from the provider config; the + // spec carries no model field, so there is no second source of truth to + // silently disagree with the API. Report what was actually used (empty + // for `--clean`, which builds no client). + model: client + .as_ref() + .map(|c| c.config().model.clone()) + .unwrap_or_default(), }) } @@ -315,10 +311,6 @@ fn temp_output_path(output: &Path) -> PathBuf { PathBuf::from(temp) } -fn default_embed_model() -> String { - DEFAULT_EMBED_MODEL.to_string() -} - fn load_embed_spec(path: &Path) -> Result { Ok(serde_json::from_str(&fs::read_to_string(path)?)?) } diff --git a/crates/omnigraph-cli/src/helpers.rs b/crates/omnigraph-cli/src/helpers.rs index 67fb6ea..971ca30 100644 --- a/crates/omnigraph-cli/src/helpers.rs +++ b/crates/omnigraph-cli/src/helpers.rs @@ -2,6 +2,8 @@ //! remote HTTP, env/token handling, scaffolding (moved verbatim from //! main.rs in the modularization). +use std::io::IsTerminal; + use super::*; use crate::operator; @@ -16,15 +18,94 @@ pub(crate) fn is_remote_uri(uri: &str) -> bool { uri.starts_with("http://") || uri.starts_with("https://") } -pub(crate) fn remote_url(base: &str, path: &str) -> String { - format!("{}{}", base.trim_end_matches('/'), path) +/// Whether a resolved write target is *local* for the purposes of the RFC-011 +/// Decision 9 destructive-confirm gate: a bare path or a `file://` URI. Anything +/// else carrying a scheme — `http(s)://` (served), `s3://` / `gs://` / … (object +/// store) — is non-local and a destructive write against it requires explicit +/// consent. Generalizes `is_remote_uri` (which only catches http(s)). +pub(crate) fn uri_is_local(uri: &str) -> bool { + !uri.contains("://") || uri.starts_with("file://") } -pub(crate) fn remote_branch_url(base: &str, branch: &str) -> Result { - let mut url = reqwest::Url::parse(&format!("{}/", base.trim_end_matches('/')))?; +/// Echo the resolved write target + access path to stderr (RFC-011 Decision 9), +/// unless `--quiet`. One line, e.g. `omnigraph load → file://g.omni (direct, +/// local)`. stderr so `--json` consumers reading stdout are unaffected; the line +/// legitimately differs embedded-vs-served (that visibility is the point). +pub(crate) fn echo_write_target(quiet: bool, label: &str, uri: &str, served: bool) { + if quiet { + return; + } + let access = if served { + "served" + } else if uri_is_local(uri) { + "direct, local" + } else { + "direct, remote" + }; + eprintln!("omnigraph {label} → {uri} ({access})"); +} + +/// Gate a destructive write (`cleanup`, overwrite `load`, `branch delete`) +/// against a non-local scope (RFC-011 Decision 9). A local target needs no +/// confirmation; otherwise `--yes` consents, an interactive TTY is prompted, and +/// a non-TTY / `--json` run refuses rather than silently proceeding. +pub(crate) fn confirm_destructive(label: &str, uri: &str, yes: bool, json: bool) -> Result<()> { + if uri_is_local(uri) || yes { + return Ok(()); + } + if json || !std::io::stdin().is_terminal() { + bail!( + "refusing destructive `{label}` against non-local target {uri} without confirmation; \ + pass --yes to confirm (an interactive TTY would be prompted instead)" + ); + } + eprint!( + "About to run a destructive `{label}` against {uri} (not local). Type 'yes' to continue: " + ); + io::stderr().flush()?; + let mut answer = String::new(); + io::stdin().read_line(&mut answer)?; + match answer.trim().to_ascii_lowercase().as_str() { + "yes" | "y" => Ok(()), + _ => bail!("aborted: destructive `{label}` not confirmed"), + } +} + +/// THE one way the CLI composes a remote request URL. Every remote call +/// routes through here so URL assembly has a single mechanism instead of +/// per-callsite string interpolation. +/// +/// - `base` is the resolved server root (single-graph) or `…/graphs/{id}` +/// (multi-graph). +/// - `segments` are appended as individual percent-encoded path segments, so +/// a dynamic component (branch name, commit id, query name) is always one +/// safe segment — e.g. a branch `etl/zendesk/run-1` becomes `%2F`-escaped. +/// - `query` pairs are percent-encoded values. +/// +/// Trailing-slash normalization happens exactly once via `pop_if_empty`: +/// `Url::parse` normalizes a path-less base (`http://host`) to a single empty +/// trailing segment, and a `…/graphs/{id}/` base keeps its own. `extend` +/// appends *after* the last segment, so without dropping a trailing empty one +/// the join emits `…/graphs/{id}//branches/{name}` — the empty `//` segment +/// misses the route and 404s. Because callers pass structured segments rather +/// than a pre-joined string, neither a stray `//` nor an un-encoded dynamic +/// component is representable here. +pub(crate) fn remote_url( + base: &str, + segments: &[&str], + query: &[(&str, &str)], +) -> Result { + let mut url = reqwest::Url::parse(base.trim_end_matches('/'))?; url.path_segments_mut() .map_err(|_| color_eyre::eyre::eyre!("invalid remote base url"))? - .extend(["branches", branch]); + .pop_if_empty() + .extend(segments); + if !query.is_empty() { + let mut pairs = url.query_pairs_mut(); + for (key, value) in query { + pairs.append_pair(key, value); + } + } Ok(url.to_string()) } @@ -38,230 +119,174 @@ pub(crate) fn bearer_token_from_env(var_name: &str) -> Option { normalize_bearer_token(std::env::var(var_name).ok()) } -pub(crate) fn parse_env_assignment(line: &str) -> Option<(String, String)> { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - return None; - } - - let line = line.strip_prefix("export ").unwrap_or(line).trim(); - let (name, value) = line.split_once('=')?; - let name = name.trim(); - if name.is_empty() { - return None; - } - - let value = value.trim(); - let value = if value.len() >= 2 - && ((value.starts_with('"') && value.ends_with('"')) - || (value.starts_with('\'') && value.ends_with('\''))) - { - &value[1..value.len() - 1] - } else { - value - }; - - Some((name.to_string(), value.to_string())) -} - -pub(crate) fn bearer_token_from_env_file(path: &Path, var_name: &str) -> Result> { - if !path.exists() { - return Ok(None); - } - - for line in fs::read_to_string(path)?.lines() { - let Some((name, value)) = parse_env_assignment(line) else { - continue; - }; - if name == var_name { - return Ok(normalize_bearer_token(Some(value))); - } - } - - Ok(None) -} - -pub(crate) fn load_env_file_into_process(path: &Path) -> Result<()> { - if !path.exists() { - return Ok(()); - } - - for line in fs::read_to_string(path)?.lines() { - let Some((name, value)) = parse_env_assignment(line) else { - continue; - }; - if std::env::var_os(&name).is_none() { - unsafe { - std::env::set_var(name, value); - } - } - } - - Ok(()) -} - -pub(crate) fn load_cli_config(config_path: Option<&PathBuf>) -> Result { - let config = load_config(config_path)?; - if let Some(path) = config.resolve_auth_env_file() { - load_env_file_into_process(&path)?; - } - Ok(config) +/// The Cedar resource id for a graph selection: the explicit graph name when one +/// is given, else the normalized URI (the anonymous fallback). Used by the +/// `policy` tooling to address a graph's bundle. +pub(crate) fn graph_resource_id_for_selection( + selected_graph: Option<&str>, + normalized_uri: &str, +) -> String { + selected_graph.unwrap_or(normalized_uri).to_string() } #[derive(Debug, Clone)] pub(crate) struct ResolvedCliGraph { pub(crate) uri: String, - pub(crate) selected: Option, - pub(crate) graph_id: String, - pub(crate) policy_file: Option, pub(crate) is_remote: bool, } -impl ResolvedCliGraph { - pub(crate) fn selected(&self) -> Option<&str> { - self.selected.as_deref() - } -} - -pub(crate) struct ResolvedPolicyContext { - pub(crate) policy_file: PathBuf, - pub(crate) graph_id: String, -} - -pub(crate) fn resolve_policy_context(config: &OmnigraphConfig) -> Result { - let selected = config.resolve_policy_tooling_graph_selection()?; - let policy_file = config.resolve_policy_file_for(selected).ok_or_else(|| { - color_eyre::eyre::eyre!( - "policy.file or graphs..policy.file must be set in omnigraph.yaml" - ) - })?; - let graph_id = match selected { - Some(name) => graph_resource_id_for_selection(Some(name), ""), - None => graph_resource_id_for_selection(None, "default"), +/// Resolve the cluster for a control-plane tooling command (`policy`, +/// `queries`) from `--cluster`. A configured name (`clusters:` in operator +/// config) is rewritten to its root; a literal dir / `s3://`/`file://` root is +/// passed through. A `--profile`/`OMNIGRAPH_PROFILE` cluster binding also +/// resolves here when `--cluster` is absent. No omnigraph.yaml. +pub(crate) fn require_cluster_scope( + cluster: Option<&str>, + profile: Option<&str>, + command: &str, +) -> Result { + let op = operator::load_operator_config()?; + let resolve_name = |name: &str| { + op.cluster_root(name) + .map(str::to_string) + .unwrap_or_else(|| name.to_string()) }; - Ok(ResolvedPolicyContext { - policy_file, - graph_id, - }) + if let Some(cluster) = cluster { + return Ok(resolve_name(cluster)); + } + // A cluster profile (flag, else OMNIGRAPH_PROFILE) binds the cluster too. + let profile_name = profile + .map(str::to_string) + .or_else(|| std::env::var(scope::PROFILE_ENV).ok().filter(|s| !s.is_empty())); + if let Some(name) = profile_name { + let profile = op.profile(&name).ok_or_else(|| { + color_eyre::eyre::eyre!("unknown profile '{name}' (not defined under `profiles:`)") + })?; + if let crate::operator::ScopeBinding::Cluster(cluster) = profile.binding(&name)? { + return Ok(resolve_name(&cluster)); + } + } + bail!( + "`{command}` needs a cluster — pass --cluster (or a name from `clusters:` \ + in ~/.omnigraph/config.yaml), or select a cluster profile" + ) } -pub(crate) fn resolve_policy_engine(context: &ResolvedPolicyContext) -> Result { - PolicyEngine::load_graph(&context.policy_file, &context.graph_id) +/// Read a cluster's serving snapshot for a control-plane tooling command, +/// flattening the readiness `Diagnostic` list into one loud error. The single +/// snapshot entry point for `policy`/`queries` so the not-servable message stays +/// identical across them. +async fn read_serving_snapshot_or_report( + cluster: &str, +) -> Result { + omnigraph_cluster::read_serving_snapshot(cluster) + .await + .map_err(|diagnostics| { + color_eyre::eyre::eyre!( + "cluster `{cluster}` is not servable:\n {}", + diagnostics + .iter() + .map(|d| d.message.clone()) + .collect::>() + .join("\n ") + ) + }) } -pub(crate) fn resolve_policy_engine_for_graph(graph: &ResolvedCliGraph) -> Result { - let policy_file = graph.policy_file.as_ref().ok_or_else(|| { - color_eyre::eyre::eyre!( - "policy.file or graphs..policy.file must be set in omnigraph.yaml" - ) - })?; - PolicyEngine::load_graph(policy_file, &graph.graph_id) +/// Resolve the Cedar policy bundle(s) for a `--cluster` policy-tooling command +/// (RFC-011). Sources the applied policies from the cluster's serving snapshot; +/// each `ServingPolicy` carries its `source` (digest-verified content) and the +/// scopes it `applies_to` (`cluster` | `graph.`). The optional `graph` +/// selects a graph's bundle when several apply. +pub(crate) async fn read_cluster_policies( + cluster: &str, +) -> Result> { + Ok(read_serving_snapshot_or_report(cluster).await?.policies) } -pub(crate) async fn open_local_db_with_policy(graph: &ResolvedCliGraph) -> Result { - let db = Omnigraph::open(&graph.uri).await?; - if graph.policy_file.is_some() { - let engine = Arc::new(resolve_policy_engine_for_graph(graph)?); - Ok(db.with_policy(engine as Arc)) - } else { - Ok(db) +/// Pick the single policy bundle that applies to the selection. With `--graph`, +/// the bundle bound to `graph.` (or the cluster-wide one); without it, the +/// sole bundle if there's exactly one. Ambiguity or absence is a loud error. +pub(crate) fn select_cluster_policy<'p>( + cluster: &str, + policies: &'p [omnigraph_cluster::ServingPolicy], + graph: Option<&str>, +) -> Result<&'p omnigraph_cluster::ServingPolicy> { + if let Some(graph_id) = graph { + let graph_ref = format!("graph.{graph_id}"); + let matching: Vec<&omnigraph_cluster::ServingPolicy> = policies + .iter() + .filter(|p| { + p.applies_to + .iter() + .any(|s| s == &graph_ref || s == "cluster") + }) + .collect(); + return match matching.as_slice() { + [only] => Ok(only), + [] => bail!( + "cluster `{cluster}` has no policy bundle bound to graph `{graph_id}` \ + (or to the cluster scope)" + ), + many => bail!( + "graph `{graph_id}` in cluster `{cluster}` matches {} policy bundles ([{}]); \ + the cluster model expects one bundle per graph scope", + many.len(), + many.iter().map(|p| p.name.as_str()).collect::>().join(", ") + ), + }; + } + match policies { + [only] => Ok(only), + [] => bail!("cluster `{cluster}` has no applied policy bundles"), + many => bail!( + "cluster `{cluster}` has {} policy bundles ([{}]); pass --graph to select one", + many.len(), + many.iter().map(|p| p.name.as_str()).collect::>().join(", ") + ), } } -/// THE actor chain (RFC-007 §D3) — every command that needs an identity +/// THE actor chain (RFC-011) — every command that needs an identity /// resolves through this one function (one path per concern): -/// `--as` > legacy `cli.actor` in omnigraph.yaml (RFC-008 window) > -/// `operator.actor` in ~/.omnigraph/config.yaml > none. -pub(crate) fn resolve_actor( - cli_as: Option<&str>, - legacy_config_actor: Option<&str>, -) -> Result> { +/// `--as` > `operator.actor` in ~/.omnigraph/config.yaml > none. +pub(crate) fn resolve_actor(cli_as: Option<&str>) -> Result> { if let Some(actor) = cli_as { return Ok(Some(actor.to_string())); } - if let Some(actor) = legacy_config_actor { - return Ok(Some(actor.to_string())); - } Ok(operator::load_operator_config()? .actor() .map(str::to_string)) } pub(crate) fn resolve_cluster_actor(cli_as: Option<&str>) -> Result> { - if let Some(actor) = cli_as { - return Ok(Some(actor.to_string())); - } - let config = load_config(None).wrap_err( - "resolving the default actor from omnigraph.yaml (pass --as to skip this lookup)", - )?; - resolve_actor(None, config.cli.actor.as_deref()) + resolve_actor(cli_as) } -pub(crate) fn resolve_cli_actor( - cli_as: Option<&str>, - config: &OmnigraphConfig, -) -> Result> { - resolve_actor(cli_as, config.cli.actor.as_deref()) +pub(crate) fn resolve_cli_actor(cli_as: Option<&str>) -> Result> { + resolve_actor(cli_as) } -pub(crate) fn resolve_policy_tests_path(context: &ResolvedPolicyContext) -> PathBuf { - context.policy_file.with_file_name("policy.tests.yaml") -} - -pub(crate) fn normalize_policy_graph_uri(uri: &str) -> Result { - if is_remote_uri(uri) { - Ok(uri.trim_end_matches('/').to_string()) - } else { - Ok(normalize_root_uri(uri)?) - } -} - -pub(crate) fn resolve_remote_bearer_token( - config: &OmnigraphConfig, - explicit_uri: Option<&str>, - explicit_target: Option<&str>, -) -> Result> { +/// The bearer token for a remote request (RFC-011): the operator keyed chain +/// for the matching server (`OMNIGRAPH_TOKEN_` env → 0600 credentials +/// file), then the default `OMNIGRAPH_BEARER_TOKEN` env. No omnigraph.yaml +/// chain. +pub(crate) fn resolve_remote_bearer_token(explicit_uri: Option<&str>) -> Result> { // The keyed hop (RFC-007 §D4, gh-host model): when the effective remote // URL belongs to an operator-defined server, that server's keyed chain // applies first — OMNIGRAPH_TOKEN_ env, then the 0600 credentials - // file. Ok(None) falls through to the legacy chain unchanged, and the - // keyed token is structurally scoped to its own server (§D5 rule 3): - // a URL matching no operator server never sees it. - if let Some(remote_url) = effective_remote_url(config, explicit_uri, explicit_target) { + // file. The keyed token is structurally scoped to its own server: a URL + // matching no operator server never sees it. + if let Some(remote_url) = explicit_uri.filter(|uri| is_remote_uri(uri)) { let operator_config = operator::load_operator_config()?; - if let Some(server) = operator_config.find_server_for_url(&remote_url) { + if let Some(server) = operator_config.find_server_for_url(remote_url) { if let Some(token) = operator::resolve_keyed_token(server)? { return Ok(Some(token)); } } } - let scoped_env = - config.graph_bearer_token_env(explicit_uri, explicit_target, config.cli_graph_name()); - let mut env_names = Vec::new(); - if let Some(name) = scoped_env { - env_names.push(name.to_string()); - } - if env_names - .iter() - .all(|name| name != DEFAULT_BEARER_TOKEN_ENV) - { - env_names.push(DEFAULT_BEARER_TOKEN_ENV.to_string()); - } - - let env_file = config.resolve_auth_env_file(); - for env_name in env_names { - if let Some(token) = bearer_token_from_env(&env_name) { - return Ok(Some(token)); - } - if let Some(path) = env_file.as_ref() { - if let Some(token) = bearer_token_from_env_file(path, &env_name)? { - return Ok(Some(token)); - } - } - } - - Ok(None) + Ok(bearer_token_from_env(DEFAULT_BEARER_TOKEN_ENV)) } /// `--server ` (RFC-007 PR 3): resolve an operator-defined server @@ -277,19 +302,26 @@ pub(crate) fn resolve_server_flag( let Some(server) = server else { return Ok(None); }; - let operator_config = operator::load_operator_config()?; - let Some(entry) = operator_config.servers.get(server) else { - let known = operator_config - .servers - .keys() - .map(String::as_str) - .collect::>() - .join(", "); - color_eyre::eyre::bail!( - "unknown server '{server}' — servers defined in the operator config: [{known}] (add it under servers: in ~/.omnigraph/config.yaml)" - ); + // RFC-011 Decision 2: a value containing `://` is a literal base URL + // (bypasses the operator-config registry); otherwise it is a config name. + let base_url = if server.contains("://") { + server.to_string() + } else { + let operator_config = operator::load_operator_config()?; + let Some(entry) = operator_config.servers.get(server) else { + let known = operator_config + .servers + .keys() + .map(String::as_str) + .collect::>() + .join(", "); + color_eyre::eyre::bail!( + "unknown server '{server}' — servers defined in the operator config: [{known}] (add it under servers: in ~/.omnigraph/config.yaml)" + ); + }; + entry.url.clone() }; - let base = entry.url.trim_end_matches('/'); + let base = base_url.trim_end_matches('/'); Ok(Some(match graph { Some(graph) => format!("{base}/graphs/{graph}"), None => base.to_string(), @@ -302,7 +334,6 @@ pub(crate) fn resolve_server_flag( /// params. The keyed token applies via the ordinary URL match. pub(crate) async fn execute_operator_alias( client: &reqwest::Client, - config: &OmnigraphConfig, alias_name: &str, alias: &crate::operator::OperatorAlias, alias_args: &[String], @@ -310,7 +341,7 @@ pub(crate) async fn execute_operator_alias( ) -> Result { let uri = resolve_server_flag(Some(&alias.server), alias.graph.as_deref())? .expect("server name is present"); - let bearer_token = resolve_remote_bearer_token(config, Some(&uri), None)?; + let bearer_token = resolve_remote_bearer_token(Some(&uri))?; let mut params = serde_json::Map::new(); for (key, value) in &alias.params { @@ -336,12 +367,16 @@ pub(crate) async fn execute_operator_alias( } } - let body = (!params.is_empty()).then(|| serde_json::json!({ "params": params })); + let mut body = serde_json::Map::new(); + body.insert("expect_mutation".to_string(), Value::Bool(false)); + if !params.is_empty() { + body.insert("params".to_string(), Value::Object(params)); + } remote_json( client, Method::POST, - remote_url(&uri, &format!("/queries/{}", alias.query)), - body, + remote_url(&uri, &["queries", &alias.query], &[])?, + Some(Value::Object(body)), bearer_token.as_deref(), ) .await @@ -353,35 +388,18 @@ pub(crate) fn apply_server_flag( server: Option<&str>, graph: Option<&str>, uri: Option, - target: Option<&str>, ) -> Result> { if server.is_none() { return Ok(uri); } - if uri.is_some() || target.is_some() { + if uri.is_some() { color_eyre::eyre::bail!( - "--server is exclusive with a positional URI and --target — pick one way to address the graph" + "--server is exclusive with a positional URI — pick one way to address the graph" ); } resolve_server_flag(server, graph) } -/// The remote base URL a token resolution is FOR — the same scoping -/// `graph_bearer_token_env` uses: an explicit http(s) `--uri` wins, else -/// the config-resolved target's uri (when remote). Local URIs → None. -fn effective_remote_url( - config: &OmnigraphConfig, - explicit_uri: Option<&str>, - explicit_target: Option<&str>, -) -> Option { - if let Some(uri) = explicit_uri { - return is_remote_uri(uri).then(|| uri.to_string()); - } - let target = config.resolve_target_name(explicit_uri, explicit_target, config.cli_graph_name())?; - let uri = &config.graphs.get(target)?.uri; - is_remote_uri(uri).then(|| uri.clone()) -} - pub(crate) fn build_http_client() -> Result { Ok(reqwest::Client::new()) } @@ -422,50 +440,38 @@ pub(crate) async fn remote_json( Ok(serde_json::from_str(&text)?) } -pub(crate) fn resolve_uri( - config: &OmnigraphConfig, - cli_uri: Option, - cli_target: Option<&str>, -) -> Result { - config.resolve_target_uri(cli_uri, cli_target, config.cli_graph_name()) +/// The graph URI a command addresses (RFC-011): the scope-resolved URI string +/// (positional URI / `--store` / `--profile` / `defaults.store`). No +/// omnigraph.yaml `cli.graph` fallback — an absent address is a loud error. +pub(crate) fn resolve_uri(cli_uri: Option) -> Result { + cli_uri.ok_or_else(|| { + color_eyre::eyre::eyre!( + "no graph addressed — pass a positional URI, --store , --server , \ + --profile , or set a default scope in ~/.omnigraph/config.yaml" + ) + }) } -pub(crate) fn resolve_cli_graph( - config: &OmnigraphConfig, - cli_uri: Option, - cli_target: Option<&str>, -) -> Result { - let selected = if cli_uri.is_some() { - None - } else { - cli_target - .map(str::to_string) - .or_else(|| config.cli_graph_name().map(str::to_string)) - }; - config.resolve_graph_selection(selected.as_deref())?; - let uri = resolve_uri(config, cli_uri, cli_target)?; - let normalized_uri = normalize_policy_graph_uri(&uri)?; - let graph_id = graph_resource_id_for_selection(selected.as_deref(), &normalized_uri); +pub(crate) fn resolve_cli_graph(cli_uri: Option) -> Result { + let uri = resolve_uri(cli_uri)?; Ok(ResolvedCliGraph { - graph_id, is_remote: is_remote_uri(&uri), - policy_file: config.resolve_policy_file_for(selected.as_deref()), - selected, uri, }) } pub(crate) fn resolve_local_graph( - config: &OmnigraphConfig, cli_uri: Option, - cli_target: Option<&str>, operation: &str, ) -> Result { - let graph = resolve_cli_graph(config, cli_uri, cli_target)?; + let graph = resolve_cli_graph(cli_uri)?; if graph.is_remote { bail!( - "{} is only supported against local graph URIs in this milestone", - operation + "`{}` is a direct (storage-native) command and needs direct storage \ + access; the resolved target is a remote server ({}). Pass the \ + graph's file:// or s3:// URI.", + operation, + graph.uri ); } Ok(graph) @@ -501,29 +507,112 @@ pub(crate) fn parse_duration_arg(s: &str) -> Result { Ok(std::time::Duration::from_secs(secs)) } -pub(crate) fn resolve_local_uri( - config: &OmnigraphConfig, +pub(crate) fn resolve_local_uri(cli_uri: Option, operation: &str) -> Result { + Ok(resolve_local_graph(cli_uri, operation)?.uri) +} + +/// Resolve a direct (storage-native) verb's address to a storage URI through the +/// one RFC-011 scope path — the maintenance verbs (optimize/repair/cleanup) plus +/// `schema plan` and `lint`'s graph-target path. Every primitive funnels here: a +/// positional URI, `--store`, `--cluster --graph `, a `--profile` +/// cluster binding, or operator defaults — all resolved at the `Direct` +/// capability (so a server scope is rejected, a cluster scope is allowed when the +/// verb opts into cluster addressing), then mapped to a storage URI by +/// `resolve_storage_uri`. +pub(crate) async fn resolve_maintenance_uri( + profile: Option<&str>, + store: Option<&str>, + cluster: Option<&str>, + graph: Option<&str>, cli_uri: Option, - cli_target: Option<&str>, operation: &str, ) -> Result { - Ok(resolve_local_graph(config, cli_uri, cli_target, operation)?.uri) + let scope = scope::resolve_scope( + &operator::load_operator_config()?, + planes::Capability::Direct, + scope::ScopeFlags { + profile, + store, + server: None, + cluster, + graph, + uri: cli_uri, + }, + )?; + resolve_storage_uri( + scope.uri, + scope.cluster.as_deref(), + scope.cluster_graph.as_deref(), + operation, + ) + .await +} + +/// Map a resolved direct address to a storage URI: a cluster scope +/// (`--cluster --graph `, or a `--profile` cluster binding) resolves +/// the graph's storage URI from the **served cluster state**; otherwise the +/// ordinary positional-URI path. When a cluster scope carries no graph +/// selection (RFC-011 D7), enumerate the catalog: a sole graph is used +/// automatically, otherwise error and list the candidates so the operator can +/// pass `--graph `. +pub(crate) async fn resolve_storage_uri( + cli_uri: Option, + cluster: Option<&str>, + cluster_graph: Option<&str>, + operation: &str, +) -> Result { + match (cluster, cluster_graph) { + (Some(cluster), Some(graph_id)) => resolve_cluster_graph_uri(cluster, graph_id).await, + (Some(cluster), None) => { + let graph_id = resolve_sole_cluster_graph(cluster).await?; + resolve_cluster_graph_uri(cluster, &graph_id).await + } + (None, None) => resolve_local_uri(cli_uri, operation), + (None, Some(_)) => { + bail!("internal error: a graph was selected without a cluster scope") + } + } +} + +/// Pick the graph for a cluster scope that has no `--graph`/`default_graph` +/// (RFC-011 D7): exactly one applied graph → use it; zero → error; more than +/// one → error and list the candidates. Never auto-picks among several. +async fn resolve_sole_cluster_graph(cluster: &str) -> Result { + let ids = omnigraph_cluster::cluster_graph_ids(cluster) + .await + .map_err(|diagnostic| color_eyre::eyre::eyre!("{}", diagnostic.message))?; + match ids.as_slice() { + [only] => Ok(only.clone()), + [] => bail!("cluster `{cluster}` has no applied graphs; run `cluster apply` first"), + many => bail!( + "cluster `{cluster}` has {} graphs: [{}]; pass --graph to select one", + many.len(), + many.join(", ") + ), + } +} + +/// Look up a graph's storage URI from a cluster's applied state ledger. Uses +/// the lightweight `resolve_graph_storage_uri` (NOT the full serving-snapshot +/// validation), so maintenance — especially `repair` — works even when an +/// unrelated catalog payload is corrupt or a recovery sweep is pending. +async fn resolve_cluster_graph_uri(cluster: &str, graph_id: &str) -> Result { + omnigraph_cluster::resolve_graph_storage_uri(cluster, graph_id) + .await + .map_err(|diagnostic| color_eyre::eyre::eyre!("{}", diagnostic.message)) } pub(crate) fn resolve_branch( - config: &OmnigraphConfig, cli_branch: Option, alias_branch: Option, default_branch: &str, ) -> String { cli_branch .or(alias_branch) - .or_else(|| config.cli.branch.clone()) .unwrap_or_else(|| default_branch.to_string()) } pub(crate) fn resolve_read_target( - config: &OmnigraphConfig, cli_branch: Option, cli_snapshot: Option, alias_branch: Option, @@ -531,19 +620,15 @@ pub(crate) fn resolve_read_target( if cli_branch.is_some() && cli_snapshot.is_some() { bail!("read target may specify branch or snapshot, not both"); } - Ok(read_target_from_cli( - cli_branch - .or(alias_branch) - .or_else(|| config.cli.branch.clone()), - cli_snapshot, - )) + Ok(read_target_from_cli(cli_branch.or(alias_branch), cli_snapshot)) } pub(crate) fn resolve_query_path( - config: &OmnigraphConfig, explicit_query: Option<&PathBuf>, alias_query: Option<&str>, ) -> Result { + // The `.gq` path is resolved plainly (cwd-relative) — no omnigraph.yaml + // `query.roots` search. explicit_query .map(PathBuf::from) .or_else(|| alias_query.map(PathBuf::from)) @@ -552,11 +637,9 @@ pub(crate) fn resolve_query_path( "exactly one of --query, --query-string, or --alias must be provided" ) }) - .and_then(|query_path| config.resolve_query_path(&query_path)) } pub(crate) fn resolve_query_source( - config: &OmnigraphConfig, explicit_query: Option<&PathBuf>, inline_query: Option<&str>, alias_query: Option<&str>, @@ -568,7 +651,6 @@ pub(crate) fn resolve_query_source( return Ok(inline.to_string()); } Ok(fs::read_to_string(resolve_query_path( - config, explicit_query, alias_query, )?)?) @@ -578,49 +660,9 @@ pub(crate) fn parse_alias_value(value: &str) -> Value { serde_json::from_str(value).unwrap_or_else(|_| Value::String(value.to_string())) } -pub(crate) fn merged_params_json( - alias_name: Option<&str>, - alias_arg_names: &[String], - alias_arg_values: &[String], - explicit: Option, -) -> Result> { - if alias_arg_values.len() > alias_arg_names.len() { - let alias = alias_name.unwrap_or(""); - bail!( - "alias '{}' expects at most {} args but got {}", - alias, - alias_arg_names.len(), - alias_arg_values.len() - ); - } - - let mut merged = serde_json::Map::new(); - for (arg_name, arg_value) in alias_arg_names.iter().zip(alias_arg_values.iter()) { - merged.insert(arg_name.clone(), parse_alias_value(arg_value)); - } - - match explicit { - Some(Value::Object(object)) => { - for (key, value) in object { - merged.insert(key, value); - } - } - Some(_) => bail!("params JSON must be an object"), - None => {} - } - - if merged.is_empty() { - Ok(None) - } else { - Ok(Some(Value::Object(merged))) - } -} - -/// The format cascade (RFC-007 §D3): `--json` > `--format` > alias format > -/// legacy `cli.output_format` (RFC-008 window) > operator `defaults.output` -/// > table. +/// The format cascade (RFC-011): `--json` > `--format` > alias format > +/// operator `defaults.output` > table. pub(crate) fn resolve_read_format( - config: &OmnigraphConfig, cli_format: Option, json: bool, alias_format: Option, @@ -630,7 +672,6 @@ pub(crate) fn resolve_read_format( } cli_format .or(alias_format) - .or(config.cli.output_format) .or_else(|| { operator::load_operator_config() .ok() @@ -639,60 +680,7 @@ pub(crate) fn resolve_read_format( .unwrap_or_default() } -pub(crate) fn resolve_alias<'a>( - config: &'a OmnigraphConfig, - alias_name: Option<&'a str>, - expected: AliasCommand, -) -> Result> { - let Some(alias_name) = alias_name else { - return Ok(None); - }; - let alias = config.alias(alias_name)?; - if alias.command != expected { - bail!( - "alias '{}' is a {:?} alias, not a {:?} alias", - alias_name, - alias.command, - expected - ); - } - Ok(Some((alias_name, alias))) -} -pub(crate) fn normalize_legacy_alias_uri( - uri: Option, - target_available: bool, - alias_name: Option<&str>, - mut alias_args: Vec, -) -> (Option, Vec) { - let Some(candidate) = uri else { - return (None, alias_args); - }; - - if alias_name.is_some() && target_available { - alias_args.insert(0, candidate); - return (None, alias_args); - } - - (Some(candidate), alias_args) -} - - -pub(crate) fn inferred_config_path(uri: &str) -> Result { - if uri.contains("://") { - return Ok(omnigraph_server::config::default_config_path()); - } - - let path = Path::new(uri); - let base = if path.is_absolute() { - path.parent() - .map(Path::to_path_buf) - .unwrap_or(std::env::current_dir()?) - } else { - std::env::current_dir()?.join(path.parent().unwrap_or_else(|| Path::new("."))) - }; - Ok(base.join(omnigraph_server::config::DEFAULT_CONFIG_FILE)) -} pub(crate) fn read_target_from_cli(branch: Option, snapshot: Option) -> ReadTarget { if let Some(snapshot) = snapshot { @@ -740,13 +728,11 @@ pub(crate) fn query_params_from_json( } pub(crate) async fn execute_query_lint( - config: &OmnigraphConfig, cli_uri: Option, - cli_target: Option<&str>, schema_path: Option<&PathBuf>, query_path: &PathBuf, ) -> Result { - let resolved_query_path = resolve_query_path(config, Some(query_path), None)?; + let resolved_query_path = resolve_query_path(Some(query_path), None)?; let query_source = fs::read_to_string(&resolved_query_path)?; let query_path = resolved_query_path.to_string_lossy().into_owned(); @@ -764,13 +750,14 @@ pub(crate) async fn execute_query_lint( )); } - let has_graph_target = - cli_uri.is_some() || cli_target.is_some() || config.cli_graph_name().is_some(); - if !has_graph_target { - bail!("query lint requires --schema or a resolvable graph target"); + if cli_uri.is_none() { + bail!( + "lint requires --schema (offline) or a graph target \ + (--store / --cluster --graph )" + ); } - let uri = resolve_local_uri(config, cli_uri, cli_target, "query lint")?; + let uri = resolve_local_uri(cli_uri, "lint")?; let db = Omnigraph::open(&uri).await?; Ok(lint_query_file( &db.catalog(), @@ -780,21 +767,24 @@ pub(crate) async fn execute_query_lint( )) } -pub(crate) fn resolve_selected_graph( - config: &OmnigraphConfig, - cli_uri: Option, - cli_target: Option<&str>, - operation: &str, -) -> Result<(String, Option)> { - let graph = resolve_local_graph(config, cli_uri, cli_target, operation)?; - Ok((graph.uri, graph.selected)) -} - -pub(crate) fn load_registry_or_report( - config: &OmnigraphConfig, - selected: Option<&str>, +/// Build a `QueryRegistry` from a cluster serving snapshot's stored queries, +/// optionally scoped to one graph. The `ServingQuery.source` is the +/// digest-verified `.gq` content, so no file I/O or omnigraph.yaml is involved. +fn registry_from_serving_queries( + queries: &[omnigraph_cluster::ServingQuery], + graph: Option<&str>, ) -> Result { - QueryRegistry::load(config, config.query_entries_for(selected)).map_err(|errors| { + let specs: Vec = queries + .iter() + .filter(|q| graph.is_none_or(|g| q.graph_id == g)) + .map(|q| omnigraph_server::queries::RegistrySpec { + name: q.name.clone(), + source: q.source.clone(), + expose: false, + tool_name: None, + }) + .collect(); + QueryRegistry::from_specs(specs).map_err(|errors| { color_eyre::eyre::eyre!( "stored-query registry failed to load:\n {}", errors @@ -806,90 +796,58 @@ pub(crate) fn load_registry_or_report( }) } -pub(crate) fn graph_query_registry_names(config: &OmnigraphConfig) -> Vec<&str> { - config - .graphs - .iter() - .filter_map(|(name, graph)| (!graph.queries.is_empty()).then_some(name.as_str())) - .collect() -} - -pub(crate) fn resolve_registry_selection_for_list( - config: &OmnigraphConfig, - target: Option<&str>, -) -> Result> { - let selected = target - .map(str::to_string) - .or_else(|| config.cli_graph_name().map(str::to_string)); - if let Some(name) = selected.as_deref() { - config.resolve_graph_selection(Some(name))?; - return Ok(selected); - } - - if !config.query_entries().is_empty() { - return Ok(None); - } - - let graph_names = graph_query_registry_names(config); - if graph_names.is_empty() { - return Ok(None); - } - - bail!( - "stored-query registries are configured for graph{} {} but no graph was selected. Pass `--target {}` or set `cli.graph`.", - if graph_names.len() == 1 { "" } else { "s" }, - graph_names.join(", "), - graph_names[0], - ) -} - -pub(crate) fn validate_registry_for_catalog( - registry: &QueryRegistry, - catalog: &omnigraph_compiler::catalog::Catalog, - label: &str, -) -> omnigraph::error::Result<()> { - let report = check(registry, catalog); - if report.has_breakages() { - return Err(omnigraph::error::OmniError::manifest( - format_check_breakages(label, &report), - )); - } - Ok(()) -} +/// `queries validate --cluster ` (RFC-011): type-check every stored query +/// in the cluster catalog against its graph's applied schema. Both the registry +/// and the schemas come from the cluster serving snapshot — no omnigraph.yaml. +/// With `--graph`, scope to a single graph. pub(crate) async fn execute_queries_validate( - uri: Option, - target: Option, - config_path: Option<&PathBuf>, + cluster: &str, + graph: Option<&str>, json: bool, ) -> Result<()> { - let config = load_cli_config(config_path)?; - // One selection drives both the schema URI and the registry, so a - // positional URI and a `--target` can't validate different graphs. - let (uri, selected) = - resolve_selected_graph(&config, uri, target.as_deref(), "queries validate")?; - let registry = load_registry_or_report(&config, selected.as_deref())?; - let db = Omnigraph::open(&uri).await?; - let report = check(®istry, &db.catalog()); + let snapshot = read_serving_snapshot_or_report(cluster).await?; - let output = QueriesValidateOutput { - ok: !report.has_breakages(), - breakages: report - .breakages - .iter() - .map(|b| QueriesIssue { + // Type-check per graph: each graph's stored queries against its own schema + // (read from the graph's applied storage root). A `--graph` filter scopes to + // exactly one graph; an unknown id is a loud error. + let mut breakages = Vec::new(); + let mut warnings = Vec::new(); + let mut total = 0usize; + let mut matched_any = false; + for serving_graph in &snapshot.graphs { + if graph.is_some_and(|g| g != serving_graph.graph_id) { + continue; + } + matched_any = true; + let registry = registry_from_serving_queries(&snapshot.queries, Some(&serving_graph.graph_id))?; + let db = Omnigraph::open(&serving_graph.root.to_string_lossy()).await?; + let report = check(®istry, &db.catalog()); + total += registry.len(); + for b in &report.breakages { + breakages.push(QueriesIssue { query: b.query.clone(), message: b.message.clone(), - }) - .collect(), - warnings: report - .warnings - .iter() - .map(|w| QueriesIssue { + }); + } + for w in &report.warnings { + warnings.push(QueriesIssue { query: w.query.clone(), message: w.message.clone(), - }) - .collect(), + }); + } + } + if let Some(graph_id) = graph { + if !matched_any { + bail!("graph `{graph_id}` is not applied in cluster `{cluster}`"); + } + } + + let has_breakages = !breakages.is_empty(); + let output = QueriesValidateOutput { + ok: !has_breakages, + breakages, + warnings, }; if json { @@ -898,8 +856,8 @@ pub(crate) async fn execute_queries_validate( if output.breakages.is_empty() { println!( "OK {} stored quer{} type-check against the schema", - registry.len(), - if registry.len() == 1 { "y" } else { "ies" } + total, + if total == 1 { "y" } else { "ies" } ); } for issue in &output.breakages { @@ -910,21 +868,22 @@ pub(crate) async fn execute_queries_validate( } } - if report.has_breakages() { + if has_breakages { io::stdout().flush()?; std::process::exit(1); } Ok(()) } -pub(crate) fn execute_queries_list( - target: Option, - config_path: Option<&PathBuf>, +/// `queries list --cluster ` (RFC-011): list the catalog's stored queries. +/// With `--graph`, scope to one graph. +pub(crate) async fn execute_queries_list( + cluster: &str, + graph: Option<&str>, json: bool, ) -> Result<()> { - let config = load_cli_config(config_path)?; - let selected = resolve_registry_selection_for_list(&config, target.as_deref())?; - let registry = load_registry_or_report(&config, selected.as_deref())?; + let snapshot = read_serving_snapshot_or_report(cluster).await?; + let registry = registry_from_serving_queries(&snapshot.queries, graph)?; let output = QueriesListOutput { queries: registry @@ -979,77 +938,6 @@ pub(crate) fn execute_queries_list( Ok(()) } -pub(crate) async fn execute_read( - uri: &str, - query_source: &str, - query_name: Option<&str>, - target: ReadTarget, - params_json: Option<&Value>, -) -> Result { - let (selected_name, query_params) = select_named_query(query_source, query_name)?; - let params = query_params_from_json(&query_params, params_json)?; - let db = Omnigraph::open(uri).await?; - let result = db - .query(target.clone(), query_source, &selected_name, ¶ms) - .await?; - Ok(read_output(selected_name, &target, result)) -} - -pub(crate) async fn execute_read_remote( - client: &reqwest::Client, - uri: &str, - query_source: &str, - query_name: Option<&str>, - target: ReadTarget, - params_json: Option<&Value>, - bearer_token: Option<&str>, -) -> Result { - let (branch, snapshot) = match &target { - ReadTarget::Branch(branch) => (Some(branch.clone()), None), - ReadTarget::Snapshot(snapshot) => (None, Some(snapshot.as_str().to_string())), - }; - remote_json( - client, - Method::POST, - remote_url(uri, "/read"), - Some(serde_json::to_value(ReadRequest { - query_source: query_source.to_string(), - query_name: query_name.map(ToOwned::to_owned), - params: params_json.cloned(), - branch, - snapshot, - })?), - bearer_token, - ) - .await -} - -pub(crate) async fn execute_change( - graph: &ResolvedCliGraph, - query_source: &str, - query_name: Option<&str>, - branch: &str, - params_json: Option<&Value>, - config: &OmnigraphConfig, - cli_as_actor: Option<&str>, -) -> Result { - let (selected_name, query_params) = select_named_query(query_source, query_name)?; - let params = query_params_from_json(&query_params, params_json)?; - let db = open_local_db_with_policy(graph).await?; - let actor = resolve_cli_actor(cli_as_actor, config)?; - let actor = actor.as_deref(); - let result = db - .mutate_as(branch, query_source, &selected_name, ¶ms, actor) - .await?; - Ok(ChangeOutput { - branch: branch.to_string(), - query_name: selected_name, - affected_nodes: result.affected_nodes, - affected_edges: result.affected_edges, - actor_id: actor.map(String::from), - }) -} - pub(crate) fn legacy_change_request_body( query_source: &str, query_name: Option<&str>, @@ -1069,79 +957,6 @@ pub(crate) fn legacy_change_request_body( body } -pub(crate) async fn execute_change_remote( - client: &reqwest::Client, - uri: &str, - query_source: &str, - query_name: Option<&str>, - branch: &str, - params_json: Option<&Value>, - bearer_token: Option<&str>, -) -> Result { - remote_json( - client, - Method::POST, - remote_url(uri, "/change"), - Some(legacy_change_request_body( - query_source, - query_name, - branch, - params_json, - )), - bearer_token, - ) - .await -} - -pub(crate) async fn execute_export_to_writer( - uri: &str, - branch: &str, - type_names: &[String], - table_keys: &[String], - writer: &mut W, -) -> Result<()> { - let db = Omnigraph::open(uri).await?; - db.export_jsonl_to_writer(branch, type_names, table_keys, writer) - .await?; - writer.flush()?; - Ok(()) -} - -pub(crate) async fn execute_export_remote_to_writer( - client: &reqwest::Client, - uri: &str, - branch: &str, - type_names: &[String], - table_keys: &[String], - bearer_token: Option<&str>, - writer: &mut W, -) -> Result<()> { - let request = apply_bearer_token( - client.request(Method::POST, remote_url(uri, "/export")), - bearer_token, - ) - .json(&ExportRequest { - branch: Some(branch.to_string()), - type_names: type_names.to_vec(), - table_keys: table_keys.to_vec(), - }); - let mut response = request.send().await?; - let status = response.status(); - if !status.is_success() { - let text = response.text().await?; - if let Ok(error) = serde_json::from_str::(&text) { - bail!(error.error); - } - bail!("server returned {}: {}", status, text); - } - - while let Some(chunk) = response.chunk().await? { - writer.write_all(&chunk)?; - } - writer.flush()?; - Ok(()) -} - pub(crate) fn rewrite_deprecated_argv(args: Vec) -> Vec { if args.len() >= 3 { let sub = args[1].to_str(); @@ -1185,3 +1000,133 @@ pub(crate) fn rewrite_deprecated_argv(args: Vec) -> Vec { } args } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn graph_resource_id_for_selection_uses_name_or_anonymous_uri() { + assert_eq!( + graph_resource_id_for_selection(Some("local"), "/tmp/graph.omni"), + "local" + ); + assert_eq!( + graph_resource_id_for_selection(None, "/tmp/graph.omni"), + "/tmp/graph.omni" + ); + } + + // RFC-011 Decision 9: locality classifier for the destructive-confirm gate. + #[test] + fn uri_is_local_truth_table() { + // Local: bare path or file://. + assert!(uri_is_local("graph.omni")); + assert!(uri_is_local("/abs/path/graph.omni")); + assert!(uri_is_local("file:///tmp/graph.omni")); + // Non-local: served or object-store schemes. + assert!(!uri_is_local("http://host/graphs/g")); + assert!(!uri_is_local("https://host/graphs/g")); + assert!(!uri_is_local("s3://bucket/graph.omni")); + assert!(!uri_is_local("gs://bucket/graph.omni")); + } + + // RFC-011 Decision 9: a non-local destructive write with `--json` (the CI + // shape — also covers the no-TTY case, since tests run without a terminal) + // refuses rather than proceeding; a local one and an explicit `--yes` pass. + #[test] + fn confirm_destructive_refuses_non_local_without_consent() { + let err = confirm_destructive("cleanup", "s3://b/g.omni", false, true) + .unwrap_err() + .to_string(); + assert!(err.contains("--yes"), "{err}"); + } + + #[test] + fn confirm_destructive_allows_local_and_explicit_yes() { + // Local needs no confirmation, even with --json. + assert!(confirm_destructive("cleanup", "file:///tmp/g.omni", false, true).is_ok()); + assert!(confirm_destructive("branch delete", "graph.omni", false, true).is_ok()); + // --yes consents to a non-local target. + assert!(confirm_destructive("cleanup", "s3://b/g.omni", true, true).is_ok()); + } + + // RFC-011 Decision 2: `--server` accepts a literal URL (value with `://`), + // bypassing the operator-config registry — so no config / OMNIGRAPH_HOME is + // read on this path (hermetic). + #[test] + fn server_flag_accepts_a_literal_url() { + assert_eq!( + resolve_server_flag(Some("https://graph.example.com"), None).unwrap(), + Some("https://graph.example.com".to_string()) + ); + // trailing slash trimmed; `--graph` appends the multi-graph path. + assert_eq!( + resolve_server_flag(Some("https://graph.example.com/"), Some("knowledge")).unwrap(), + Some("https://graph.example.com/graphs/knowledge".to_string()) + ); + } + + // `branch delete` interpolates the branch into the URL path. The composed + // path must be exactly `/branches/` with no empty `//` + // segment — an empty segment misses the + // `/graphs/{graph_id}/branches/{branch}` route and 404s. + #[test] + fn remote_url_multi_graph_base_has_no_double_slash() { + let url = remote_url("http://host/graphs/p9-os", &["branches", "tmpbranch"], &[]).unwrap(); + assert_eq!(url, "http://host/graphs/p9-os/branches/tmpbranch"); + assert!( + !url.contains("//branches"), + "double slash before branches: {url}" + ); + } + + #[test] + fn remote_url_single_graph_base_has_no_double_slash() { + let url = remote_url("http://host", &["branches", "tmpbranch"], &[]).unwrap(); + assert_eq!(url, "http://host/branches/tmpbranch"); + } + + #[test] + fn remote_url_tolerates_trailing_slash_on_base() { + let url = remote_url("http://host/graphs/p9-os/", &["branches", "tmpbranch"], &[]).unwrap(); + assert_eq!(url, "http://host/graphs/p9-os/branches/tmpbranch"); + } + + #[test] + fn remote_url_encodes_slashes_in_path_segment() { + let url = remote_url( + "http://host/graphs/p9-os", + &["branches", "etl/zendesk/run-1"], + &[], + ) + .unwrap(); + assert_eq!( + url, + "http://host/graphs/p9-os/branches/etl%2Fzendesk%2Frun-1" + ); + } + + // Sibling cases the unified builder closes by construction: a dynamic + // commit id in the path, and a branch name carried as a query value, are + // both percent-encoded instead of interpolated raw. + #[test] + fn remote_url_encodes_dynamic_path_segment_for_commits() { + let url = remote_url("http://host/graphs/p9-os", &["commits", "a/b c"], &[]).unwrap(); + assert_eq!(url, "http://host/graphs/p9-os/commits/a%2Fb%20c"); + } + + #[test] + fn remote_url_encodes_query_values() { + let url = remote_url( + "http://host/graphs/p9-os", + &["snapshot"], + &[("branch", "feature&x=1")], + ) + .unwrap(); + assert_eq!( + url, + "http://host/graphs/p9-os/snapshot?branch=feature%26x%3D1" + ); + } +} diff --git a/crates/omnigraph-cli/src/main.rs b/crates/omnigraph-cli/src/main.rs index 8178e65..bb3b062 100644 --- a/crates/omnigraph-cli/src/main.rs +++ b/crates/omnigraph-cli/src/main.rs @@ -1,15 +1,11 @@ use std::ffi::OsString; use std::fs; use std::io::{self, Write}; -use std::path::Path; use std::path::PathBuf; -use std::sync::Arc; - use clap::{Arg, ArgAction, Args, CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum}; -use color_eyre::eyre::{Result, WrapErr, bail}; +use color_eyre::eyre::{Result, bail}; use omnigraph::db::{Omnigraph, ReadTarget, SnapshotId}; use omnigraph::loader::LoadMode; -use omnigraph::storage::normalize_root_uri; use omnigraph_cluster::{ ApplyOptions, ApplyOutput, ApproveOutput, DiagnosticSeverity, ForceUnlockOutput, PlanOutput, StateSyncOutput, StatusOutput, ValidateOutput, apply_config_dir_with_options, approve_config_dir, force_unlock_config_dir, import_config_dir, plan_config_dir, @@ -22,18 +18,13 @@ use omnigraph_compiler::{ QueryLintSeverity, QueryLintStatus, SchemaMigrationPlan, SchemaMigrationStep, build_catalog, json_params_to_param_map, lint_query_file, }; -use omnigraph_server::api::{ - BranchCreateOutput, BranchCreateRequest, BranchDeleteOutput, BranchListOutput, - BranchMergeOutput, BranchMergeRequest, ChangeOutput, CommitListOutput, CommitOutput, - ErrorOutput, ExportRequest, GraphListResponse, IngestOutput, IngestRequest, ReadOutput, - ReadRequest, SchemaApplyOutput, SchemaApplyRequest, SchemaOutput, SnapshotOutput, - SnapshotTableOutput, commit_output, ingest_output, read_output, schema_apply_output, - snapshot_payload, +use omnigraph_api_types::{ + ChangeOutput, CommitOutput, ErrorOutput, IngestOutput, ReadOutput, SchemaApplyOutput, + SnapshotTableOutput, }; -use omnigraph_server::queries::{QueryRegistry, check, format_check_breakages}; +use omnigraph_server::queries::{QueryRegistry, check}; use omnigraph_server::{ - AliasCommand, OmnigraphConfig, PolicyAction, PolicyDecision, PolicyEngine, PolicyRequest, - PolicyTestConfig, ReadOutputFormat, graph_resource_id_for_selection, load_config, + PolicyAction, PolicyDecision, PolicyEngine, PolicyRequest, PolicyTestConfig, }; use reqwest::Method; use reqwest::header::AUTHORIZATION; @@ -42,16 +33,18 @@ use serde::de::DeserializeOwned; use serde_json::Value; mod embed; -mod migrate; mod operator; mod read_format; use embed::{EmbedArgs, EmbedOutput, execute_embed}; -use read_format::{ReadRenderOptions, render_read}; +use read_format::{ReadOutputFormat, ReadRenderOptions, render_read}; mod cli; +mod client; mod helpers; mod output; +mod scope; +mod planes; use cli::*; use helpers::*; use output::*; @@ -73,43 +66,11 @@ async fn main() -> Result<()> { Cli::from_arg_matches(&matches)? }; let http_client = build_http_client()?; + // RFC-010 Slice 1: reject data-plane addressing flags (--server/--graph) on + // a verb that doesn't live on the data plane, from one declared table — + // before any per-command dispatch. + planes::guard_addressing(&cli)?; match cli.command { - Command::Config { command } => match command { - ConfigCommand::Migrate { config, write, json } => { - let path = migrate::legacy_config_path(config.as_ref()); - if !path.exists() { - bail!( - "no legacy config at '{}' — nothing to migrate", - path.display() - ); - } - let legacy = load_config(Some(&path))?; - let report = migrate::build_report(&legacy, &path); - if write { - let legacy_dir = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or(std::path::Path::new(".")) - .to_path_buf(); - let written = migrate::apply_report(&report, &legacy_dir)?; - if json { - print_json(&serde_json::json!({ - "report": report, - "written": written, - }))?; - } else { - print!("{}", migrate::render_report(&report)); - for line in written { - println!("wrote: {line}"); - } - } - } else if json { - print_json(&report)?; - } else { - print!("{}", migrate::render_report(&report)); - } - } - }, Command::Login { name, token, json } => { let token = match token { Some(token) => token, @@ -133,6 +94,124 @@ async fn main() -> Result<()> { let path = crate::operator::remove_credential(&name)?; finish_logout(&name, &path, json)?; } + Command::Profile { command } => { + use crate::operator::ScopeBinding; + let op = crate::operator::load_operator_config()?; + let active = std::env::var(scope::PROFILE_ENV) + .ok() + .filter(|s| !s.is_empty()); + match command { + ProfileCommand::List { json } => { + let items: Vec = op + .profiles + .iter() + .map(|(name, profile)| { + let (binding, scope_kind, target, valid, error) = + match profile.binding(name) { + Ok(ScopeBinding::Server(s)) => ( + format!("server: {s}"), + "server".to_string(), + Some(s), + true, + None, + ), + Ok(ScopeBinding::Cluster(c)) => ( + format!("cluster: {c}"), + "cluster".to_string(), + Some(c), + true, + None, + ), + Ok(ScopeBinding::Store(u)) => ( + format!("store: {u}"), + "store".to_string(), + Some(u), + true, + None, + ), + Err(e) => ( + format!("invalid: {e}"), + "invalid".to_string(), + None, + false, + Some(e.to_string()), + ), + }; + ProfileListItem { + name: name.clone(), + binding, + scope_kind, + target, + valid, + error, + default_graph: profile.default_graph.clone(), + active: active.as_deref() == Some(name.as_str()), + } + }) + .collect(); + print_profile_list(&items, json)?; + } + ProfileCommand::Show { name, json } => { + let detail = match name.or(active) { + Some(name) => { + let profile = op.profile(&name).ok_or_else(|| { + color_eyre::eyre::eyre!( + "unknown profile '{name}' (not defined under `profiles:`)" + ) + })?; + let (kind, target, endpoint) = match profile.binding(&name)? { + ScopeBinding::Server(s) => { + let endpoint = op.servers.get(&s).map(|sv| sv.url.clone()); + ("server", Some(s), endpoint) + } + ScopeBinding::Cluster(c) => { + let endpoint = op.cluster_root(&c).map(str::to_string); + ("cluster", Some(c), endpoint) + } + ScopeBinding::Store(u) => ("store", Some(u.clone()), Some(u)), + }; + ProfileDetail { + name, + scope_kind: kind.to_string(), + target, + endpoint, + default_graph: profile + .default_graph + .clone() + .or_else(|| op.default_graph().map(str::to_string)), + output_format: op + .output() + .and_then(|f| f.to_possible_value()) + .map(|v| v.get_name().to_string()), + } + } + // No name and no active profile: the flat operator defaults. + None => { + let (kind, target, endpoint) = if let Some(s) = op.default_server() { + let endpoint = op.servers.get(s).map(|sv| sv.url.clone()); + ("server", Some(s.to_string()), endpoint) + } else if let Some(u) = op.default_store() { + ("store", Some(u.to_string()), Some(u.to_string())) + } else { + ("none", None, None) + }; + ProfileDetail { + name: "(defaults)".to_string(), + scope_kind: kind.to_string(), + target, + endpoint, + default_graph: op.default_graph().map(str::to_string), + output_format: op + .output() + .and_then(|f| f.to_possible_value()) + .map(|v| v.get_name().to_string()), + } + } + }; + print_profile_detail(&detail, json)?; + } + } + } Command::Version => { println!("omnigraph {}", env!("CARGO_PKG_VERSION")); } @@ -145,6 +224,16 @@ async fn main() -> Result<()> { } } Command::Init { schema, uri, force } => { + // RFC-010 Slice 3: graphs inside an established cluster are created + // by `cluster apply` (which records ledger/recovery/approvals), not + // by hand-running `init` into the cluster's storage layout. + if let Some(root) = omnigraph_cluster::cluster_root_for_graph_uri(&uri).await { + bail!( + "`{uri}` is inside cluster `{root}`. Graphs in a cluster are created by \ + `cluster apply` (which records ledger, recovery, and approvals), not `init`. \ + Declare the graph in cluster.yaml and run `cluster apply`." + ); + } let schema_source = fs::read_to_string(&schema)?; ensure_local_graph_parent(&uri)?; Omnigraph::init_with_options( @@ -157,63 +246,29 @@ async fn main() -> Result<()> { } Command::Load { uri, - target, - config, data, branch, from, mode, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); - let branch = resolve_branch(&config, branch, None, "main"); - let payload = if graph.is_remote { - let data = fs::read_to_string(&data)?; - let output = remote_json::( - &http_client, - Method::POST, - remote_url(&uri, "/ingest"), - Some(serde_json::to_value(IngestRequest { - branch: Some(branch.clone()), - from: from.clone(), - mode: Some(mode.into()), - data, - })?), - bearer_token.as_deref(), - ) + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let branch = resolve_branch(branch, None, "main"); + if matches!(mode, CliLoadMode::Overwrite) { + confirm_destructive("load --mode overwrite", client.uri(), cli.yes, json)?; + } + echo_write_target(cli.quiet, "load", client.uri(), client.is_remote()); + let payload = client + .load(&branch, from.as_deref(), &data.to_string_lossy(), mode) .await?; - load_output_from_tables(&uri, &branch, mode, &output) - } else { - let db = open_local_db_with_policy(&graph).await?; - let actor = resolve_cli_actor(cli.as_actor.as_deref(), &config)?; - let actor = actor.as_deref(); - let result = db - .load_file_as( - &branch, - from.as_deref(), - &data.to_string_lossy(), - mode.into(), - actor, - ) - .await?; - LoadOutput { - uri: uri.clone(), - branch: branch.clone(), - mode: mode.as_str(), - base_branch: result.base_branch.clone(), - branch_created: result.branch_created, - nodes_loaded: result.nodes_loaded.values().sum(), - edges_loaded: result.edges_loaded.values().sum(), - node_types_loaded: result.nodes_loaded.len(), - edge_types_loaded: result.edges_loaded.len(), - } - }; if json { print_json(&payload)?; } else { @@ -222,8 +277,6 @@ async fn main() -> Result<()> { } Command::Ingest { uri, - target, - config, data, branch, from, @@ -235,45 +288,21 @@ async fn main() -> Result<()> { "warning: `omnigraph ingest` is deprecated and will be removed in a future release; \ use `omnigraph load --from --mode ` (ingest defaults: --from main --mode merge)" ); - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); - let branch = resolve_branch(&config, branch, None, "main"); - let from = resolve_branch(&config, from, None, "main"); - let payload = if graph.is_remote { - let data = fs::read_to_string(&data)?; - remote_json::( - &http_client, - Method::POST, - remote_url(&uri, "/ingest"), - Some(serde_json::to_value(IngestRequest { - branch: Some(branch.clone()), - from: Some(from.clone()), - mode: Some(mode.into()), - data, - })?), - bearer_token.as_deref(), - ) - .await? - } else { - let db = open_local_db_with_policy(&graph).await?; - let actor = resolve_cli_actor(cli.as_actor.as_deref(), &config)?; - let actor = actor.as_deref(); - let result = db - .load_file_as( - &branch, - Some(&from), - &data.to_string_lossy(), - mode.into(), - actor, - ) - .await?; - ingest_output(&uri, &result, mode.into(), None) - }; + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let branch = resolve_branch(branch, None, "main"); + let from = resolve_branch(from, None, "main"); + echo_write_target(cli.quiet, "ingest", client.uri(), client.is_remote()); + let payload = client + .ingest(&branch, &from, &data.to_string_lossy(), mode) + .await?; if json { print_json(&payload)?; } else { @@ -283,45 +312,22 @@ async fn main() -> Result<()> { Command::Branch { command } => match command { BranchCommand::Create { uri, - target, - config, from, name, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); - let from = resolve_branch(&config, from, None, "main"); - let payload = if graph.is_remote { - remote_json::( - &http_client, - Method::POST, - remote_url(&uri, "/branches"), - Some(serde_json::to_value(BranchCreateRequest { - from: Some(from.clone()), - name: name.clone(), - })?), - bearer_token.as_deref(), - ) - .await? - } else { - let db = open_local_db_with_policy(&graph).await?; - let actor = resolve_cli_actor(cli.as_actor.as_deref(), &config)?; - let actor = actor.as_deref(); - db.branch_create_from_as(ReadTarget::branch(&from), &name, actor) - .await?; - BranchCreateOutput { - uri: uri.clone(), - from: from.clone(), - name: name.clone(), - actor_id: actor.map(String::from), - } - }; + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let from = resolve_branch(from, None, "main"); + echo_write_target(cli.quiet, "branch create", client.uri(), client.is_remote()); + let payload = client.branch_create_from(&from, &name).await?; if json { print_json(&payload)?; } else { @@ -330,32 +336,17 @@ async fn main() -> Result<()> { } BranchCommand::List { uri, - target, - config, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); - let payload = if graph.is_remote { - remote_json::( - &http_client, - Method::GET, - remote_url(&uri, "/branches"), - None, - bearer_token.as_deref(), - ) - .await? - } else { - let db = Omnigraph::open(&uri).await?; - let mut branches = db.branch_list().await?; - branches.sort(); - BranchListOutput { branches } - }; + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let payload = client.branch_list().await?; if json { print_json(&payload)?; } else { @@ -366,38 +357,21 @@ async fn main() -> Result<()> { } BranchCommand::Delete { uri, - target, - config, name, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); - let payload = if graph.is_remote { - remote_json::( - &http_client, - Method::DELETE, - remote_branch_url(&uri, &name)?, - None, - bearer_token.as_deref(), - ) - .await? - } else { - let db = open_local_db_with_policy(&graph).await?; - let actor = resolve_cli_actor(cli.as_actor.as_deref(), &config)?; - let actor = actor.as_deref(); - db.branch_delete_as(&name, actor).await?; - BranchDeleteOutput { - uri: uri.clone(), - name: name.clone(), - actor_id: actor.map(String::from), - } - }; + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + confirm_destructive("branch delete", client.uri(), cli.yes, json)?; + echo_write_target(cli.quiet, "branch delete", client.uri(), client.is_remote()); + let payload = client.branch_delete(&name).await?; if json { print_json(&payload)?; } else { @@ -406,44 +380,22 @@ async fn main() -> Result<()> { } BranchCommand::Merge { uri, - target, - config, source, into, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); - let into = resolve_branch(&config, into, None, "main"); - let payload = if graph.is_remote { - remote_json::( - &http_client, - Method::POST, - remote_url(&uri, "/branches/merge"), - Some(serde_json::to_value(BranchMergeRequest { - source: source.clone(), - target: Some(into.clone()), - })?), - bearer_token.as_deref(), - ) - .await? - } else { - let db = open_local_db_with_policy(&graph).await?; - let actor = resolve_cli_actor(cli.as_actor.as_deref(), &config)?; - let actor = actor.as_deref(); - let outcome = db.branch_merge_as(&source, &into, actor).await?; - BranchMergeOutput { - source: source.clone(), - target: into.clone(), - outcome: outcome.into(), - actor_id: actor.map(String::from), - } - }; + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let into = resolve_branch(into, None, "main"); + echo_write_target(cli.quiet, "branch merge", client.uri(), client.is_remote()); + let payload = client.branch_merge(&source, &into).await?; if json { print_json(&payload)?; } else { @@ -459,71 +411,38 @@ async fn main() -> Result<()> { Command::Commit { command } => match command { CommitCommand::List { uri, - target, - config, branch, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; - let commits = if is_remote_uri(&uri) { - remote_json::( - &http_client, - Method::GET, - if let Some(branch) = branch.as_deref() { - format!("{}?branch={}", remote_url(&uri, "/commits"), branch) - } else { - remote_url(&uri, "/commits") - }, - None, - bearer_token.as_deref(), - ) - .await? - .commits - } else { - let db = Omnigraph::open(&uri).await?; - db.list_commits(branch.as_deref()) - .await? - .iter() - .map(commit_output) - .collect::>() - }; + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let payload = client.list_commits(branch.as_deref()).await?; if json { - print_json(&CommitListOutput { commits })?; + print_json(&payload)?; } else { - print_commit_list_human(&commits); + print_commit_list_human(&payload.commits); } } CommitCommand::Show { uri, - target, - config, commit_id, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; - let commit = if is_remote_uri(&uri) { - remote_json::( - &http_client, - Method::GET, - remote_url(&uri, &format!("/commits/{}", commit_id)), - None, - bearer_token.as_deref(), - ) - .await? - } else { - let db = Omnigraph::open(&uri).await?; - commit_output(&db.get_commit(&commit_id).await?) - }; + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let commit = client.get_commit(&commit_id).await?; if json { print_json(&commit)?; } else { @@ -534,14 +453,19 @@ async fn main() -> Result<()> { Command::Schema { command } => match command { SchemaCommand::Plan { uri, - target, - config, schema, json, allow_data_loss, } => { - let config = load_cli_config(config.as_ref())?; - let uri = resolve_local_uri(&config, uri, target.as_deref(), "schema plan")?; + let uri = resolve_maintenance_uri( + cli.profile.as_deref(), + cli.store.as_deref(), + cli.cluster.as_deref(), + cli.graph.as_deref(), + uri, + "schema plan", + ) + .await?; let schema_source = fs::read_to_string(&schema)?; let db = Omnigraph::open(&uri).await?; let plan = db @@ -564,59 +488,47 @@ async fn main() -> Result<()> { } SchemaCommand::Apply { uri, - target, - config, schema, json, allow_data_loss, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let graph = resolve_cli_graph(&config, uri, target.as_deref())?; - let uri = graph.uri.clone(); + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + // RFC-011 Decision 10: a graph managed by a cluster evolves via + // `cluster apply` (ledger/recovery/approvals), not a direct + // `schema apply` against its storage root — that would bypass the + // ledger. Mirrors `init`'s refusal. Only the embedded path can + // address a storage root; a served apply (`--server`) is the + // server's concern. + if !client.is_remote() { + if let Some(root) = + omnigraph_cluster::cluster_root_for_graph_uri(client.uri()).await + { + bail!( + "`{}` is inside cluster `{root}`. A graph in a cluster evolves via \ + `cluster apply` (which records ledger, recovery, and approvals), not \ + `schema apply`. Update the schema in cluster.yaml and run `cluster apply`.", + client.uri() + ); + } + } let schema_source = fs::read_to_string(&schema)?; - let output = if graph.is_remote { - // MR-694 PR B: SchemaApplyRequest gained an - // allow_data_loss field so Hard-mode drops are no - // longer CLI-only. The previous bail is gone; the - // field is forwarded into the JSON payload, and - // the server's `server_schema_apply` honors it. - remote_json::( - &http_client, - Method::POST, - remote_url(&uri, "/schema/apply"), - Some(serde_json::to_value(SchemaApplyRequest { - schema_source: schema_source.clone(), - allow_data_loss, - })?), - bearer_token.as_deref(), - ) - .await? - } else { - let db = open_local_db_with_policy(&graph).await?; - let actor = resolve_cli_actor(cli.as_actor.as_deref(), &config)?; - let actor = actor.as_deref(); - let registry = load_registry_or_report(&config, graph.selected())?; - let registry = (!registry.is_empty()).then_some(registry); - let label = graph.selected().unwrap_or(&uri).to_string(); - let result = db - .apply_schema_as_with_catalog_check( - &schema_source, - omnigraph::db::SchemaApplyOptions { allow_data_loss }, - actor, - |catalog| { - if let Some(registry) = registry.as_ref() { - validate_registry_for_catalog(registry, catalog, &label)?; - } - Ok(()) - }, - ) - .await?; - schema_apply_output(&uri, result) - }; + // The embedded (direct-store) arm carries no stored-query + // registry — the registry is cluster-owned (RFC-011), so a + // direct apply has nothing to validate against. The served arm + // runs the server's own catalog check. So the validator is a + // no-op here on both arms. + echo_write_target(cli.quiet, "schema apply", client.uri(), client.is_remote()); + let output = client + .apply_schema(&schema_source, allow_data_loss, |_catalog| Ok(())) + .await?; if json { print_json(&output)?; } else { @@ -625,31 +537,17 @@ async fn main() -> Result<()> { } SchemaCommand::Show { uri, - target, - config, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; - let output = if is_remote_uri(&uri) { - remote_json::( - &http_client, - Method::GET, - remote_url(&uri, "/schema"), - None, - bearer_token.as_deref(), - ) - .await? - } else { - let db = Omnigraph::open(&uri).await?; - SchemaOutput { - schema_source: db.schema_source().to_string(), - } - }; + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let output = client.schema_source().await?; if json { print_json(&output)?; } else { @@ -659,64 +557,59 @@ async fn main() -> Result<()> { }, Command::Lint { uri, - target, - config, query, schema, json, } => { - let config = load_cli_config(config.as_ref())?; - let output = - execute_query_lint(&config, uri, target.as_deref(), schema.as_ref(), &query) - .await?; + // A graph target (when `--schema` is absent) resolves through the + // direct scope path (positional URI / --store / --profile / + // defaults.store). Offline (`--schema`) needs no graph, so leave + // the uri unresolved in that case. + let graph_uri = if schema.is_some() { + uri + } else { + Some( + resolve_maintenance_uri( + cli.profile.as_deref(), + cli.store.as_deref(), + cli.cluster.as_deref(), + cli.graph.as_deref(), + uri, + "lint", + ) + .await?, + ) + }; + let output = execute_query_lint(graph_uri, schema.as_ref(), &query).await?; finish_query_lint(&output, json)?; } - Command::Queries { command } => match command { - QueriesCommand::Validate { - uri, - target, - config, - json, - } => { - execute_queries_validate(uri, target, config.as_ref(), json).await?; + Command::Queries { command } => { + let cluster = + require_cluster_scope(cli.cluster.as_deref(), cli.profile.as_deref(), "queries")?; + match command { + QueriesCommand::Validate { json } => { + execute_queries_validate(&cluster, cli.graph.as_deref(), json).await?; + } + QueriesCommand::List { json } => { + execute_queries_list(&cluster, cli.graph.as_deref(), json).await?; + } } - QueriesCommand::List { - target, - config, - json, - } => { - execute_queries_list(target, config.as_ref(), json)?; - } - }, + } Command::Snapshot { uri, - target, - config, branch, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; - let branch = resolve_branch(&config, branch, None, "main"); - let payload = if is_remote_uri(&uri) { - remote_json::( - &http_client, - Method::GET, - format!("{}?branch={}", remote_url(&uri, "/snapshot"), branch), - None, - bearer_token.as_deref(), - ) - .await? - } else { - let db = Omnigraph::open(&uri).await?; - let snapshot = db.snapshot_of(ReadTarget::branch(branch.as_str())).await?; - snapshot_payload(&branch, &snapshot) - }; - + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let branch = resolve_branch(branch, None, "main"); + let payload = client.snapshot(&branch).await?; if json { print_json(&payload)?; } else { @@ -725,248 +618,114 @@ async fn main() -> Result<()> { } Command::Export { uri, - target, - config, branch, jsonl, type_names, table_keys, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; - let branch = resolve_branch(&config, branch, None, "main"); + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let branch = resolve_branch(branch, None, "main"); if jsonl { eprintln!("warning: --jsonl is deprecated; `omnigraph export` always emits JSONL"); } let stdout = io::stdout(); let mut stdout = stdout.lock(); - if is_remote_uri(&uri) { - execute_export_remote_to_writer( - &http_client, - &uri, - &branch, - &type_names, - &table_keys, - bearer_token.as_deref(), - &mut stdout, - ) + client + .export(&branch, &type_names, &table_keys, &mut stdout) .await?; - } else { - execute_export_to_writer(&uri, &branch, &type_names, &table_keys, &mut stdout) - .await?; - } } Command::Query { - uri, - legacy_uri, - target, - config, - alias, + name, query, query_string, - name, params, branch, snapshot, format, json, - alias_args, } => { - if alias.is_none() && query.is_none() && query_string.is_none() { - bail!("exactly one of --query, --query-string, or --alias must be provided"); - } - - let config = load_cli_config(config.as_ref())?; - // Operator aliases (RFC-007 PR 3): pure bindings to stored - // queries. A legacy file-alias with the same name wins during - // the RFC-008 window (with a warning); an alias name found - // only in the operator layer takes the invoke path here. - if let Some(alias_name) = alias.as_deref() { - let operator_config = crate::operator::load_operator_config()?; - if let Some(operator_alias) = operator_config.aliases.get(alias_name) { - if config.alias(alias_name).is_ok() { - eprintln!( - "warning: alias '{alias_name}' is defined in both omnigraph.yaml (legacy, wins during the deprecation window) and the operator config; the legacy definition applies" - ); - } else { - // The hidden legacy-uri positional swallows the first - // bare arg; an operator alias always knows its target, - // so reclaim it as the first positional param. - let (_, alias_args) = normalize_legacy_alias_uri( - legacy_uri.clone(), - true, - Some(alias_name), - alias_args.clone(), - ); - let output = execute_operator_alias( - &http_client, - &config, - alias_name, - operator_alias, - &alias_args, - load_params_json(¶ms)?, - ) - .await?; - let format = - resolve_read_format(&config, format, json, operator_alias.format); - print_read_output(&output, format, &config)?; - return Ok(()); - } - } - } - let alias = resolve_alias(&config, alias.as_deref(), AliasCommand::Read)?; - let alias_name = alias.as_ref().map(|(name, _)| *name); - let alias_config = alias.as_ref().map(|(_, alias)| *alias); - let target_available = target.is_some() - || alias_config - .and_then(|alias| alias.graph.as_deref()) - .is_some() - || config.cli_graph_name().is_some(); - let (legacy_uri, alias_args) = - normalize_legacy_alias_uri(legacy_uri, target_available, alias_name, alias_args); - let uri = uri.or(legacy_uri); - let target_name = target - .as_deref() - .or_else(|| alias_config.and_then(|alias| alias.graph.as_deref())); - let uri = apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target_name)?; - let bearer_token = resolve_remote_bearer_token(&config, uri.as_deref(), target_name)?; - let graph = resolve_cli_graph(&config, uri, target_name)?; - let uri = graph.uri.clone(); - let query_source = resolve_query_source( - &config, - query.as_ref(), - query_string.as_deref(), - alias_config.map(|a| a.query.as_str()), - )?; - let params_json = merged_params_json( - alias_name, - alias_config - .map(|alias| alias.args.as_slice()) - .unwrap_or(&[]), - &alias_args, - load_params_json(¶ms)?, - )?; - let target = resolve_read_target( - &config, - branch, - snapshot, - alias_config.and_then(|alias| alias.branch.clone()), - )?; - let query_name = name.or_else(|| alias_config.and_then(|alias| alias.name.clone())); - let output = if graph.is_remote { - execute_read_remote( - &http_client, - &uri, - &query_source, - query_name.as_deref(), - target, - params_json.as_ref(), - bearer_token.as_deref(), - ) - .await? + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + None, + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let params_json = load_params_json(¶ms)?; + let target = resolve_read_target(branch, snapshot, None)?; + let output: ReadOutput = if query.is_some() || query_string.is_some() { + // Ad-hoc lane: run the source; the positional `name` selects + // within it when it holds more than one query. + let query_source = + resolve_query_source(query.as_ref(), query_string.as_deref(), None)?; + client + .query(target, &query_source, name.as_deref(), params_json.as_ref()) + .await? } else { - execute_read( - &uri, - &query_source, - query_name.as_deref(), - target, - params_json.as_ref(), - ) - .await? + // Catalog lane (served-only): invoke the stored query by name. + let Some(name) = name else { + bail!( + "provide a query name to invoke from the catalog, or -e '' / \ + --query for an ad-hoc query" + ); + }; + let (branch, snapshot) = match &target { + ReadTarget::Branch(b) => (Some(b.clone()), None), + ReadTarget::Snapshot(s) => (None, Some(s.as_str().to_string())), + }; + client + .invoke_named(&name, false, params_json.as_ref(), branch, snapshot) + .await? }; - let format = resolve_read_format( - &config, - format, - json, - alias_config.and_then(|alias| alias.format), - ); - print_read_output(&output, format, &config)?; + let format = resolve_read_format(format, json, None); + print_read_output(&output, format)?; } Command::Mutate { - uri, - legacy_uri, - target, - config, - alias, + name, query, query_string, - name, params, branch, json, - alias_args, } => { - if alias.is_none() && query.is_none() && query_string.is_none() { - bail!("exactly one of --query, --query-string, or --alias must be provided"); - } - - let config = load_cli_config(config.as_ref())?; - let alias = resolve_alias(&config, alias.as_deref(), AliasCommand::Change)?; - let alias_name = alias.as_ref().map(|(name, _)| *name); - let alias_config = alias.as_ref().map(|(_, alias)| *alias); - let target_available = target.is_some() - || alias_config - .and_then(|alias| alias.graph.as_deref()) - .is_some() - || config.cli_graph_name().is_some(); - let (legacy_uri, alias_args) = - normalize_legacy_alias_uri(legacy_uri, target_available, alias_name, alias_args); - let uri = uri.or(legacy_uri); - let target_name = target - .as_deref() - .or_else(|| alias_config.and_then(|alias| alias.graph.as_deref())); - let uri = apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target_name)?; - let bearer_token = resolve_remote_bearer_token(&config, uri.as_deref(), target_name)?; - let graph = resolve_cli_graph(&config, uri, target_name)?; - let uri = graph.uri.clone(); - let query_source = resolve_query_source( - &config, - query.as_ref(), - query_string.as_deref(), - alias_config.map(|a| a.query.as_str()), - )?; - let params_json = merged_params_json( - alias_name, - alias_config - .map(|alias| alias.args.as_slice()) - .unwrap_or(&[]), - &alias_args, - load_params_json(¶ms)?, - )?; - let branch = resolve_branch( - &config, - branch, - alias_config.and_then(|alias| alias.branch.clone()), - "main", - ); - let query_name = name.or_else(|| alias_config.and_then(|alias| alias.name.clone())); - let output = if graph.is_remote { - execute_change_remote( - &http_client, - &uri, - &query_source, - query_name.as_deref(), - &branch, - params_json.as_ref(), - bearer_token.as_deref(), - ) - .await? + let client = client::GraphClient::resolve_with_policy( + cli.server.as_deref(), + cli.graph.as_deref(), + None, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let params_json = load_params_json(¶ms)?; + let branch = resolve_branch(branch, None, "main"); + let output: ChangeOutput = if query.is_some() || query_string.is_some() { + // Ad-hoc lane: run the source; positional `name` selects within it. + let query_source = + resolve_query_source(query.as_ref(), query_string.as_deref(), None)?; + client + .mutate(&branch, &query_source, name.as_deref(), params_json.as_ref()) + .await? } else { - execute_change( - &graph, - &query_source, - query_name.as_deref(), - &branch, - params_json.as_ref(), - &config, - cli.as_actor.as_deref(), - ) - .await? + // Catalog lane (served-only): invoke the stored mutation by name. + let Some(name) = name else { + bail!( + "provide a mutation name to invoke from the catalog, or -e '' / \ + --query for an ad-hoc mutation" + ); + }; + client + .invoke_named(&name, true, params_json.as_ref(), Some(branch), None) + .await? }; if json { print_json(&output)?; @@ -974,53 +733,92 @@ async fn main() -> Result<()> { print_change_human(&output); } } - Command::Policy { command } => match command { - PolicyCommand::Validate { config } => { - let config = load_cli_config(config.as_ref())?; - let context = resolve_policy_context(&config)?; - let engine = resolve_policy_engine(&context)?; - println!( - "policy valid: {} [{} actors]", - context.policy_file.display(), - engine.known_actor_count() + Command::Alias { + name, + args, + params, + format, + json, + } => { + let operator_config = crate::operator::load_operator_config()?; + let Some(operator_alias) = operator_config.aliases.get(&name) else { + let defined: Vec<&str> = + operator_config.aliases.keys().map(String::as_str).collect(); + bail!( + "unknown alias '{name}'; defined aliases: [{}] \ + (add it under `aliases:` in ~/.omnigraph/config.yaml)", + defined.join(", ") ); - } - PolicyCommand::Test { config } => { - let config = load_cli_config(config.as_ref())?; - let context = resolve_policy_context(&config)?; - let engine = resolve_policy_engine(&context)?; - let tests_path = resolve_policy_tests_path(&context); - let tests = PolicyTestConfig::load(&tests_path)?; - engine.run_tests(&tests)?; - println!("policy tests passed: {} cases", tests.cases.len()); - } - PolicyCommand::Explain { - config, - actor, - action, - branch, - target_branch, - } => { - let config = load_cli_config(config.as_ref())?; - let context = resolve_policy_context(&config)?; - let engine = resolve_policy_engine(&context)?; - let request = PolicyRequest { + }; + let output = execute_operator_alias( + &http_client, + &name, + operator_alias, + &args, + load_params_json(¶ms)?, + ) + .await?; + let format = resolve_read_format(format, json, operator_alias.format); + print_read_output(&output, format)?; + } + Command::Policy { command } => { + // Policy tooling sources the Cedar bundle(s) from the cluster's + // applied policies (RFC-011): --cluster , + the global --graph + // to pick a graph's bundle when several apply. + let cluster = + require_cluster_scope(cli.cluster.as_deref(), cli.profile.as_deref(), "policy")?; + let graph = cli.graph.as_deref(); + let graph_id = match graph { + Some(id) => graph_resource_id_for_selection(Some(id), ""), + None => graph_resource_id_for_selection(None, "default"), + }; + let policies = read_cluster_policies(&cluster).await?; + match command { + PolicyCommand::Validate {} => { + let bundle = select_cluster_policy(&cluster, &policies, graph)?; + let engine = PolicyEngine::load_graph_from_source(&bundle.source, &graph_id)?; + println!( + "policy valid: bundle '{}' [{} actors]", + bundle.name, + engine.known_actor_count() + ); + } + PolicyCommand::Test { tests } => { + let bundle = select_cluster_policy(&cluster, &policies, graph)?; + let engine = PolicyEngine::load_graph_from_source(&bundle.source, &graph_id)?; + let tests = PolicyTestConfig::load(&tests)?; + engine.run_tests(&tests)?; + println!("policy tests passed: {} cases", tests.cases.len()); + } + PolicyCommand::Explain { + actor, action, branch, target_branch, - }; - let decision = engine.authorize(&actor, &request)?; - print_policy_explain(&decision, &actor, &request); + } => { + let bundle = select_cluster_policy(&cluster, &policies, graph)?; + let engine = PolicyEngine::load_graph_from_source(&bundle.source, &graph_id)?; + let request = PolicyRequest { + action, + branch, + target_branch, + }; + let decision = engine.authorize(&actor, &request)?; + print_policy_explain(&decision, &actor, &request); + } } - }, - Command::Optimize { - uri, - target, - config, - json, - } => { - let config = load_cli_config(config.as_ref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; + } + Command::Optimize { uri, json } => { + let uri = resolve_maintenance_uri( + cli.profile.as_deref(), + cli.store.as_deref(), + cli.cluster.as_deref(), + cli.graph.as_deref(), + uri, + "optimize", + ) + .await?; + echo_write_target(cli.quiet, "optimize", &uri, false); let db = Omnigraph::open(&uri).await?; let stats = db.optimize().await?; if json { @@ -1034,6 +832,10 @@ async fn main() -> Result<()> { "skipped": s.skipped.map(|r| r.as_str()), "manifest_version": s.manifest_version, "lance_head_version": s.lance_head_version, + "pending_indexes": s.pending_indexes.iter().map(|p| serde_json::json!({ + "column": p.column, + "reason": p.reason, + })).collect::>(), })).collect::>(), }); print_json(&value)?; @@ -1050,19 +852,28 @@ async fn main() -> Result<()> { } else { println!(" {:<40} no-op", s.table_key); } + for p in &s.pending_indexes { + println!(" ↳ index pending on '{}': {}", p.column, p.reason); + } } } } Command::Repair { uri, - target, - config, confirm, force, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; + let uri = resolve_maintenance_uri( + cli.profile.as_deref(), + cli.store.as_deref(), + cli.cluster.as_deref(), + cli.graph.as_deref(), + uri, + "repair", + ) + .await?; + echo_write_target(cli.quiet, "repair", &uri, false); let db = Omnigraph::open(&uri).await?; let stats = db .repair(omnigraph::db::RepairOptions { confirm, force }) @@ -1138,15 +949,20 @@ async fn main() -> Result<()> { } Command::Cleanup { uri, - target, - config, keep, older_than, confirm, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; + let uri = resolve_maintenance_uri( + cli.profile.as_deref(), + cli.store.as_deref(), + cli.cluster.as_deref(), + cli.graph.as_deref(), + uri, + "cleanup", + ) + .await?; let older_than_dur = older_than.as_deref().map(parse_duration_arg).transpose()?; @@ -1170,6 +986,11 @@ async fn main() -> Result<()> { ); return Ok(()); } + // Past the preview gate: a real destructive run. Against a non-local + // scope this additionally requires --yes (or an interactive yes), so + // `cleanup --confirm s3://…` in CI refuses rather than destroying. + confirm_destructive("cleanup", &uri, cli.yes, json)?; + echo_write_target(cli.quiet, "cleanup", &uri, false); let options = omnigraph::db::CleanupPolicyOptions { keep_versions: keep, @@ -1271,31 +1092,17 @@ async fn main() -> Result<()> { Command::Graphs { command } => match command { GraphsCommand::List { uri, - target, - config, json, } => { - let config = load_cli_config(config.as_ref())?; - let uri = - apply_server_flag(cli.server.as_deref(), cli.graph.as_deref(), uri, target.as_deref())?; - let bearer_token = - resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?; - let uri = resolve_uri(&config, uri, target.as_deref())?; - if !is_remote_uri(&uri) { - bail!( - "`omnigraph graphs list` requires a remote multi-graph server URL \ - (http:// or https://). To enumerate local graphs, read `omnigraph.yaml` \ - directly." - ); - } - let payload = remote_json::( - &http_client, - Method::GET, - remote_url(&uri, "/graphs"), - None, - bearer_token.as_deref(), + let client = client::GraphClient::resolve( + cli.server.as_deref(), + cli.graph.as_deref(), + uri, + cli.profile.as_deref(), + cli.store.as_deref(), ) .await?; + let payload = client.list_graphs().await?; if json { print_json(&payload)?; } else { diff --git a/crates/omnigraph-cli/src/main_tests.rs b/crates/omnigraph-cli/src/main_tests.rs index 8380c36..4f93277 100644 --- a/crates/omnigraph-cli/src/main_tests.rs +++ b/crates/omnigraph-cli/src/main_tests.rs @@ -1,22 +1,16 @@ //! In-source test suite for the CLI binary (moved verbatim from //! main.rs; `use super::*` resolves through the #[path] declaration). - use std::fs; - use super::{ - DEFAULT_BEARER_TOKEN_ENV, apply_bearer_token, bearer_token_from_env_file, - legacy_change_request_body, load_cli_config, load_env_file_into_process, - normalize_bearer_token, parse_env_assignment, resolve_cli_graph, resolve_policy_context, - resolve_remote_bearer_token, + DEFAULT_BEARER_TOKEN_ENV, apply_bearer_token, legacy_change_request_body, + normalize_bearer_token, resolve_remote_bearer_token, }; - use omnigraph_server::load_config; use reqwest::header::AUTHORIZATION; use serde_json::json; - use tempfile::tempdir; #[test] fn legacy_change_request_body_uses_legacy_field_names() { - // `execute_change_remote` hits `POST /change`, which old + // `mutate`'s remote arm hits `POST /change`, which old // `omnigraph-server` builds deserialize as `ChangeRequest` with // **required** `query_source` and optional `query_name` keys. // Newer servers accept both spellings via serde alias, but a @@ -96,126 +90,20 @@ } #[test] - fn parse_env_assignment_supports_plain_and_exported_values() { - assert_eq!( - parse_env_assignment("DEMO_TOKEN=demo-token"), - Some(("DEMO_TOKEN".to_string(), "demo-token".to_string())) - ); - assert_eq!( - parse_env_assignment("export DEMO_TOKEN=\"quoted-token\""), - Some(("DEMO_TOKEN".to_string(), "quoted-token".to_string())) - ); - assert_eq!(parse_env_assignment("# comment"), None); - assert_eq!(parse_env_assignment(" "), None); - } - - #[test] - fn bearer_token_from_env_file_reads_named_value() { - let temp = tempdir().unwrap(); - let env_file = temp.path().join(".env.omni"); - fs::write( - &env_file, - "FIRST=ignore\nexport DEMO_TOKEN=\" demo-token \"\n", - ) - .unwrap(); - - assert_eq!( - bearer_token_from_env_file(&env_file, "DEMO_TOKEN") - .unwrap() - .as_deref(), - Some("demo-token") - ); - assert_eq!( - bearer_token_from_env_file(&env_file, "MISSING").unwrap(), - None - ); - } - - #[test] - fn load_env_file_into_process_sets_missing_values_without_overriding_existing_ones() { - let temp = tempdir().unwrap(); - let env_file = temp.path().join(".env.omni"); - fs::write( - &env_file, - "AUTOLOAD_ONLY=from-file\nAUTOLOAD_PRESET=from-file\n", - ) - .unwrap(); - - let missing_key = "AUTOLOAD_ONLY"; - let preset_key = "AUTOLOAD_PRESET"; - let previous_missing = std::env::var_os(missing_key); - let previous_preset = std::env::var_os(preset_key); - - unsafe { - std::env::remove_var(missing_key); - std::env::set_var(preset_key, "from-env"); - } - - load_env_file_into_process(&env_file).unwrap(); - - assert_eq!(std::env::var(missing_key).unwrap(), "from-file"); - assert_eq!(std::env::var(preset_key).unwrap(), "from-env"); - - unsafe { - if let Some(value) = previous_missing { - std::env::set_var(missing_key, value); - } else { - std::env::remove_var(missing_key); - } - - if let Some(value) = previous_preset { - std::env::set_var(preset_key, value); - } else { - std::env::remove_var(preset_key); - } - } - } - - #[test] - fn resolve_remote_bearer_token_uses_scoped_env_file_with_global_fallback() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - r#" -graphs: - demo: - uri: https://example.com - bearer_token_env: DEMO_TOKEN -auth: - env_file: .env.omni -cli: - graph: demo -"#, - ) - .unwrap(); - fs::write( - temp.path().join(".env.omni"), - "DEMO_TOKEN=scoped-token\nOMNIGRAPH_BEARER_TOKEN=global-token\n", - ) - .unwrap(); - + fn resolve_remote_bearer_token_falls_back_to_default_env() { + // RFC-011: with no operator server matching the URL, the only chain + // left is the default `OMNIGRAPH_BEARER_TOKEN` env (no omnigraph.yaml + // scoped chain). Hermetic: no operator config is read for a literal URL + // that matches no `servers:` entry. let previous = std::env::var_os(DEFAULT_BEARER_TOKEN_ENV); let previous_home = std::env::var_os("OMNIGRAPH_HOME"); unsafe { - std::env::remove_var(DEFAULT_BEARER_TOKEN_ENV); - // Hermetic: the keyed hop (RFC-007 PR 2) must not pick up a real - // ~/.omnigraph on the developer's machine — and with no operator - // servers defined, the legacy chain below must behave - // byte-identically to pre-PR-2 (tested-as-untouched). - std::env::set_var("OMNIGRAPH_HOME", temp.path().join("no-operator-config")); + std::env::set_var(DEFAULT_BEARER_TOKEN_ENV, "global-token"); + std::env::set_var("OMNIGRAPH_HOME", "/nonexistent/omnigraph-test-home"); } - let config_path = temp.path().join("omnigraph.yaml"); - let config = load_config(Some(&config_path)).unwrap(); - assert_eq!( - resolve_remote_bearer_token(&config, None, Some("demo")) - .unwrap() - .as_deref(), - Some("scoped-token") - ); - assert_eq!( - resolve_remote_bearer_token(&config, Some("https://override.example.com"), None) + resolve_remote_bearer_token(Some("https://override.example.com")) .unwrap() .as_deref(), Some("global-token") @@ -234,194 +122,3 @@ cli: } } } - - #[test] - fn load_cli_config_autoloads_env_file_into_process() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - r#" -auth: - env_file: .env.omni -graphs: - demo: - uri: s3://bucket/prefix -"#, - ) - .unwrap(); - fs::write( - temp.path().join(".env.omni"), - "AUTOLOAD_FROM_CONFIG=loaded\n", - ) - .unwrap(); - - let key = "AUTOLOAD_FROM_CONFIG"; - let previous = std::env::var_os(key); - unsafe { - std::env::remove_var(key); - } - - let config_path = temp.path().join("omnigraph.yaml"); - let config = load_cli_config(Some(&config_path)).unwrap(); - - assert_eq!( - config.resolve_target_uri(None, Some("demo"), None).unwrap(), - "s3://bucket/prefix" - ); - assert_eq!(std::env::var(key).unwrap(), "loaded"); - - unsafe { - if let Some(value) = previous { - std::env::set_var(key, value); - } else { - std::env::remove_var(key); - } - } - } - - #[test] - fn graph_identity_resolve_policy_context_named_cli_graph_uses_graph_key_not_project_name_or_uri() - { - let temp = tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -project: - name: misleading-project -graphs: - local: - uri: /tmp/local-policy-graph.omni - policy: - file: ./policy.yaml -cli: - graph: local -"#, - ) - .unwrap(); - - let config = load_config(Some(&config_path)).unwrap(); - let context = resolve_policy_context(&config).unwrap(); - assert_eq!(context.graph_id, "local"); - } - - #[test] - fn graph_identity_resolve_policy_context_server_graph_uses_graph_key_when_cli_graph_absent() { - let temp = tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -project: - name: misleading-project -graphs: - local: - uri: /tmp/local-policy-graph.omni - policy: - file: ./server-policy.yaml -server: - graph: local -"#, - ) - .unwrap(); - - let config = load_config(Some(&config_path)).unwrap(); - let context = resolve_policy_context(&config).unwrap(); - assert_eq!(context.graph_id, "local"); - assert!(context.policy_file.ends_with("server-policy.yaml")); - } - - #[test] - fn graph_identity_resolve_policy_context_anonymous_uses_top_level_default_identity() { - let temp = tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -project: - name: misleading-project -graphs: - local: - uri: /tmp/local-policy-graph.omni -policy: - file: ./top-policy.yaml -"#, - ) - .unwrap(); - - let config = load_config(Some(&config_path)).unwrap(); - let context = resolve_policy_context(&config).unwrap(); - assert_eq!(context.graph_id, "default"); - assert!(context.policy_file.ends_with("top-policy.yaml")); - } - - #[test] - fn graph_identity_resolve_cli_graph_named_target_uses_graph_key_not_project_name_or_uri() { - let temp = tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -project: - name: misleading-project -graphs: - prod: - uri: s3://bucket/prod-graph/ - policy: - file: ./prod-policy.yaml -"#, - ) - .unwrap(); - - let config = load_config(Some(&config_path)).unwrap(); - let graph = resolve_cli_graph(&config, None, Some("prod")).unwrap(); - assert_eq!(graph.selected(), Some("prod")); - assert_eq!(graph.graph_id, "prod"); - assert_eq!(graph.uri, "s3://bucket/prod-graph/"); - } - - #[test] - fn graph_identity_resolve_cli_graph_positional_uri_uses_anonymous_normalized_uri() { - let temp = tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -project: - name: misleading-project -graphs: - local: - uri: /tmp/configured-graph.omni - policy: - file: ./policy.yaml -cli: - graph: local -"#, - ) - .unwrap(); - - let config = load_config(Some(&config_path)).unwrap(); - let local_graph_path = temp.path().join("explicit-graph.omni"); - let local_graph = resolve_cli_graph( - &config, - Some(format!("file://{}", local_graph_path.display())), - None, - ) - .unwrap(); - assert_eq!(local_graph.selected(), None); - assert_eq!( - local_graph.graph_id, - local_graph_path.to_string_lossy().as_ref() - ); - assert_eq!(local_graph.policy_file, None); - - let s3_graph = resolve_cli_graph( - &config, - Some("s3://bucket/anonymous-graph/".to_string()), - None, - ) - .unwrap(); - assert_eq!(s3_graph.selected(), None); - assert_eq!(s3_graph.graph_id, "s3://bucket/anonymous-graph"); - assert_eq!(s3_graph.policy_file, None); - } diff --git a/crates/omnigraph-cli/src/migrate.rs b/crates/omnigraph-cli/src/migrate.rs deleted file mode 100644 index 3891061..0000000 --- a/crates/omnigraph-cli/src/migrate.rs +++ /dev/null @@ -1,408 +0,0 @@ -//! `omnigraph config migrate` (RFC-008 stage 2): split a legacy -//! `omnigraph.yaml` into its two destinations — the team half as a -//! ready-to-review `cluster.yaml` proposal, the personal half merged into -//! `~/.omnigraph/config.yaml` — and name what's obsolete. The command is -//! the completeness test of RFC-008's migration map: any key it cannot -//! place is a bug in the RFC. -//! -//! Touches nothing without `--write`. Referenced `.gq`/policy files are -//! never moved; manual steps are printed instead. - -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; - -use color_eyre::Result; -use color_eyre::eyre::eyre; -use omnigraph_server::OmnigraphConfig; -use serde::Serialize; - -use crate::operator; - -#[derive(Debug, Serialize)] -pub(crate) struct MigrateReport { - pub(crate) source: String, - /// The ready-to-review cluster.yaml text (None when the legacy file - /// declares nothing team-shaped). - pub(crate) cluster_yaml: Option, - /// Operator keys to merge: dotted key -> YAML value text. - pub(crate) operator_merge: BTreeMap, - /// Keys with no destination, and why. - pub(crate) dropped: Vec, - /// Steps the command will not do for you. - pub(crate) manual_steps: Vec, -} - -#[derive(Debug, Serialize)] -pub(crate) struct DroppedKey { - pub(crate) key: String, - pub(crate) reason: String, -} - -/// Classify a parsed legacy config into the report. Pure — no I/O. -pub(crate) fn build_report(config: &OmnigraphConfig, source: &Path) -> MigrateReport { - let mut dropped = Vec::new(); - let mut manual_steps = Vec::new(); - let mut operator_merge: BTreeMap = BTreeMap::new(); - - // ---- personal half ---- - if let Some(actor) = &config.cli.actor { - operator_merge.insert("operator.actor".into(), actor.clone()); - } - if let Some(format) = config.cli.output_format { - operator_merge.insert( - "defaults.output".into(), - serde_yaml::to_string(&format).unwrap_or_default().trim().to_string(), - ); - } - if let Some(width) = config.cli.table_max_column_width { - operator_merge.insert("defaults.table_max_column_width".into(), width.to_string()); - } - if let Some(layout) = config.cli.table_cell_layout { - operator_merge.insert( - "defaults.table_cell_layout".into(), - serde_yaml::to_string(&layout).unwrap_or_default().trim().to_string(), - ); - } - if config.cli.graph.is_some() { - dropped.push(DroppedKey { - key: "cli.graph".into(), - reason: "no operator default-target yet — address graphs explicitly via --target/--server (RFC-002 locator territory)".into(), - }); - } - if config.cli.branch.is_some() { - dropped.push(DroppedKey { - key: "cli.branch".into(), - reason: "pass --branch explicitly".into(), - }); - } - - // Remote graphs with a token env become operator servers (the keyed - // chain replaces invented env-var names). - for (name, target) in &config.graphs { - if target.uri.starts_with("http://") || target.uri.starts_with("https://") { - operator_merge.insert(format!("servers.{name}.url"), target.uri.clone()); - if target.bearer_token_env.is_some() { - manual_steps.push(format!( - "store the '{name}' token in the keyed chain: echo $TOKEN | omnigraph login {name} (replaces bearer_token_env)" - )); - } - } - } - if config.auth.env_file.is_some() { - manual_steps.push( - "auth.env_file keeps working during the window; prefer `omnigraph login ` per server going forward".into(), - ); - } - - // Legacy aliases split: content -> catalog stored query, binding -> - // operator alias referencing the name. - for (name, alias) in &config.aliases { - let query_name = alias.name.clone().unwrap_or_else(|| name.clone()); - operator_merge.insert( - format!("aliases.{name}"), - format!( - "{{ server: TODO-server-name, graph: {}, query: {query_name}, args: [{}] }}", - alias.graph.as_deref().unwrap_or("TODO-graph-id"), - alias.args.join(", ") - ), - ); - manual_steps.push(format!( - "alias '{name}': move its query content ('{}') into the cluster checkout's queries/ so '{query_name}' becomes a catalog stored query", - alias.query - )); - } - - // ---- team half ---- - let has_team_content = !config.graphs.is_empty() - || !config.queries.is_empty() - || config.policy.file.is_some() - || config.server.policy.file.is_some(); - let cluster_yaml = has_team_content.then(|| { - let mut out = String::from("version: 1\n"); - if let Some(name) = &config.project.name { - out.push_str(&format!("metadata:\n name: {name}\n")); - } - out.push_str("# storage: s3://bucket/prefix # or omit: this folder is the root\n"); - if !config.graphs.is_empty() || !config.queries.is_empty() { - out.push_str("graphs:\n"); - } - // Single-graph top-level queries belong to a graph the legacy file - // never named; propose one. - if !config.queries.is_empty() && config.graphs.is_empty() { - out.push_str(" default: # TODO: pick the graph id\n schema: # TODO: path to this graph's .pg schema\n queries: queries/\n"); - } - for (name, target) in &config.graphs { - out.push_str(&format!(" {name}:\n")); - out.push_str(" schema: # TODO: path to this graph's .pg schema\n"); - if !target.queries.is_empty() { - out.push_str(" queries: queries/ # move the .gq files here\n"); - } - out.push_str(&format!( - " # legacy root: {} — the cluster manages graph roots under its storage; run `omnigraph cluster import` after reviewing\n", - target.uri - )); - } - let mut policies: Vec<(String, String, String)> = Vec::new(); - if let Some(file) = &config.policy.file { - policies.push(("default".into(), file.clone(), "graph. # TODO: bind".into())); - } - if let Some(file) = &config.server.policy.file { - policies.push(("server".into(), file.clone(), "cluster".into())); - } - for (name, target) in &config.graphs { - if let Some(file) = &target.policy.file { - policies.push((name.clone(), file.clone(), format!("graph.{name}"))); - } - } - if !policies.is_empty() { - out.push_str("policies:\n"); - for (name, file, binding) in policies { - out.push_str(&format!( - " {name}:\n file: {file}\n applies_to: [{binding}]\n" - )); - } - } - out - }); - - if !config.query.roots.is_empty() { - dropped.push(DroppedKey { - key: "query.roots".into(), - reason: "obsolete — cluster query discovery (queries: ) replaced it".into(), - }); - } - if config.server.bind.is_some() || config.server.graph.is_some() { - dropped.push(DroppedKey { - key: "server.bind / server.graph".into(), - reason: "deployment runtime — pass --bind / target flags or env".into(), - }); - } - if config.project.name.is_some() && cluster_yaml.is_none() { - dropped.push(DroppedKey { - key: "project.name".into(), - reason: "the cluster's metadata.name is the deployment label".into(), - }); - } - - MigrateReport { - source: source.display().to_string(), - cluster_yaml, - operator_merge, - dropped, - manual_steps, - } -} - -pub(crate) fn render_report(report: &MigrateReport) -> String { - let mut out = format!("migration plan for {}\n", report.source); - if let Some(cluster) = &report.cluster_yaml { - out.push_str("\n== team half -> cluster.yaml (ready to review) ==\n"); - out.push_str(cluster); - } - if !report.operator_merge.is_empty() { - out.push_str("\n== personal half -> ~/.omnigraph/config.yaml ==\n"); - for (key, value) in &report.operator_merge { - out.push_str(&format!(" {key}: {value}\n")); - } - } - if !report.dropped.is_empty() { - out.push_str("\n== no destination ==\n"); - for dropped in &report.dropped { - out.push_str(&format!(" {} — {}\n", dropped.key, dropped.reason)); - } - } - if !report.manual_steps.is_empty() { - out.push_str("\n== manual steps ==\n"); - for step in &report.manual_steps { - out.push_str(&format!(" - {step}\n")); - } - } - out.push_str("\n(nothing written; pass --write to apply the operator merge and emit cluster.yaml)\n"); - out -} - -/// `--write`: merge the personal half into the operator config (key-level, -/// existing entries always win; the prior file is backed up) and write the -/// team half to cluster.yaml in the legacy config's directory (or -/// cluster.yaml.proposed when one already exists). -pub(crate) fn apply_report(report: &MigrateReport, legacy_dir: &Path) -> Result> { - let mut written = Vec::new(); - - if !report.operator_merge.is_empty() { - let dir = operator::operator_dir() - .ok_or_else(|| eyre!("no home directory resolvable for the operator config"))?; - std::fs::create_dir_all(&dir)?; - let path = dir.join(operator::OPERATOR_CONFIG_FILE); - let existing_text = std::fs::read_to_string(&path).unwrap_or_default(); - let mut mapping: serde_yaml::Mapping = if existing_text.trim().is_empty() { - serde_yaml::Mapping::new() - } else { - serde_yaml::from_str(&existing_text) - .map_err(|err| eyre!("operator config '{}' does not parse: {err}", path.display()))? - }; - let mut merged_any = false; - for (dotted, value_text) in &report.operator_merge { - if merge_dotted_if_absent(&mut mapping, dotted, value_text)? { - merged_any = true; - } - } - if merged_any { - if !existing_text.is_empty() { - let backup = path.with_extension("yaml.bak"); - std::fs::write(&backup, &existing_text)?; - written.push(format!("backed up prior operator config to {}", backup.display())); - } - let rendered = serde_yaml::to_string(&mapping)?; - let tmp = path.with_extension(format!("yaml.tmp.{}", std::process::id())); - std::fs::write(&tmp, &rendered)?; - std::fs::rename(&tmp, &path)?; - written.push(format!("merged personal keys into {}", path.display())); - } else { - written.push("operator config already carries every personal key (nothing merged)".into()); - } - } - - if let Some(cluster) = &report.cluster_yaml { - let target = legacy_dir.join("cluster.yaml"); - let target = if target.exists() { - legacy_dir.join("cluster.yaml.proposed") - } else { - target - }; - std::fs::write(&target, cluster)?; - written.push(format!("wrote team-half proposal to {}", target.display())); - } - - Ok(written) -} - -/// Set `a.b.c` in the mapping only when absent; returns whether it wrote. -fn merge_dotted_if_absent( - mapping: &mut serde_yaml::Mapping, - dotted: &str, - value_text: &str, -) -> Result { - let value: serde_yaml::Value = - serde_yaml::from_str(value_text).unwrap_or(serde_yaml::Value::String(value_text.into())); - let parts: Vec<&str> = dotted.split('.').collect(); - let mut current = mapping; - for part in &parts[..parts.len() - 1] { - let key = serde_yaml::Value::String((*part).into()); - let entry = current - .entry(key) - .or_insert_with(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); - current = entry - .as_mapping_mut() - .ok_or_else(|| eyre!("operator config key '{dotted}' collides with a non-mapping"))?; - } - let leaf = serde_yaml::Value::String(parts[parts.len() - 1].into()); - if current.contains_key(&leaf) { - return Ok(false); - } - current.insert(leaf, value); - Ok(true) -} - -pub(crate) fn legacy_config_path(explicit: Option<&PathBuf>) -> PathBuf { - explicit.cloned().unwrap_or_else(|| PathBuf::from("omnigraph.yaml")) -} - -#[cfg(test)] -mod tests { - use super::*; - use omnigraph_server::config::load_config; - - fn full_legacy_fixture(dir: &Path) -> PathBuf { - let path = dir.join("omnigraph.yaml"); - std::fs::write( - &path, - r#" -project: { name: brain } -graphs: - prod: - uri: https://graph.example.com - bearer_token_env: PROD_TOKEN - policy: { file: ./prod.policy.yaml } - queries: - find: { file: ./find.gq } - local: - uri: /tmp/local.omni -server: { bind: "0.0.0.0:9999", policy: { file: ./server.policy.yaml } } -auth: { env_file: .env.omni } -cli: - graph: prod - branch: main - actor: act-me - output_format: json - table_max_column_width: 40 -query: { roots: ["."] } -aliases: - triage: { command: query, query: ./triage.gq, name: weekly_triage, args: [since], graph: prod } -policy: { file: ./top.policy.yaml } -queries: - top_q: { file: ./top.gq } -"#, - ) - .unwrap(); - path - } - - /// The RFC-008 completeness contract: every top-level key of the - /// legacy schema must appear in the report somewhere (team half, - /// operator merge, dropped, or manual steps). - #[test] - fn every_legacy_key_is_classified() { - let dir = tempfile::tempdir().unwrap(); - let path = full_legacy_fixture(dir.path()); - let config = load_config(Some(&path)).unwrap(); - let report = build_report(&config, &path); - let rendered = render_report(&report); - - let serialized = - serde_yaml::to_value(OmnigraphConfig::default()).expect("default serializes"); - for key in serialized.as_mapping().unwrap().keys() { - let key = key.as_str().unwrap(); - assert!( - rendered.contains(key) - || report.operator_merge.keys().any(|k| k.contains(key)) - || matches!(key, "graphs" | "queries" | "policy" | "project") - && report.cluster_yaml.is_some(), - "legacy key '{key}' is unclassified — fix the RFC-008 map: {rendered}" - ); - } - - // spot checks on each section - assert_eq!(report.operator_merge["operator.actor"], "act-me"); - assert_eq!(report.operator_merge["defaults.output"], "json"); - assert_eq!( - report.operator_merge["servers.prod.url"], - "https://graph.example.com" - ); - assert!(report.operator_merge["aliases.triage"].contains("query: weekly_triage")); - let cluster = report.cluster_yaml.as_deref().unwrap(); - assert!(cluster.contains("version: 1")); - assert!(cluster.contains("name: brain")); - assert!(cluster.contains(" prod:")); - assert!(cluster.contains("applies_to: [cluster]")); - assert!(cluster.contains("applies_to: [graph.prod]")); - assert!(report.dropped.iter().any(|d| d.key == "query.roots")); - assert!(report.dropped.iter().any(|d| d.key.contains("server.bind"))); - assert!( - report - .manual_steps - .iter() - .any(|s| s.contains("omnigraph login prod")) - ); - } - - #[test] - fn merge_dotted_never_clobbers_existing() { - let mut mapping: serde_yaml::Mapping = - serde_yaml::from_str("operator:\n actor: keep-me\n").unwrap(); - assert!(!merge_dotted_if_absent(&mut mapping, "operator.actor", "new").unwrap()); - assert!(merge_dotted_if_absent(&mut mapping, "defaults.output", "json").unwrap()); - let text = serde_yaml::to_string(&mapping).unwrap(); - assert!(text.contains("keep-me") && !text.contains("new")); - assert!(text.contains("output: json")); - } -} diff --git a/crates/omnigraph-cli/src/operator.rs b/crates/omnigraph-cli/src/operator.rs index fb8658d..dbfe781 100644 --- a/crates/omnigraph-cli/src/operator.rs +++ b/crates/omnigraph-cli/src/operator.rs @@ -18,10 +18,10 @@ use std::env; use std::path::{Path, PathBuf}; use color_eyre::Result; -use color_eyre::eyre::eyre; +use color_eyre::eyre::{bail, eyre}; use serde::Deserialize; -use omnigraph_server::config::ReadOutputFormat; +use crate::read_format::{ReadOutputFormat, TableCellLayout}; pub(crate) const OPERATOR_HOME_ENV: &str = "OMNIGRAPH_HOME"; pub(crate) const OPERATOR_DIR: &str = ".omnigraph"; @@ -41,6 +41,17 @@ pub(crate) struct OperatorConfig { /// Personal alias bindings (RFC-007 PR 3); see OperatorAlias. #[serde(default)] pub(crate) aliases: BTreeMap, + /// Named scope bundles (RFC-011): each binds exactly one of + /// {server, cluster, store} plus an optional default graph. Config data, + /// not state — selecting one (`--profile`/`OMNIGRAPH_PROFILE`) fills in a + /// command's omitted addressing; it never puts you "in" a mode. + #[serde(default)] + pub(crate) profiles: BTreeMap, + /// Managed-cluster storage roots (RFC-011): name → root URI. The ONLY + /// place a storage root appears in operator config — admin-only and + /// opt-in; a normal operator's file has none. + #[serde(default)] + pub(crate) clusters: BTreeMap, /// Everything this CLI version doesn't know. Warned once at load, /// otherwise ignored (forward compatibility within the operator layer). #[serde(flatten)] @@ -80,8 +91,7 @@ pub(crate) struct OperatorServer { #[derive(Debug, Default, Deserialize)] pub(crate) struct OperatorIdentity { /// Default actor for every `--as` cascade (CLI direct-engine writes and - /// cluster commands alike): `--as` > legacy config actor (RFC-008 - /// window) > this > none. + /// cluster commands alike): `--as` > this > none. pub(crate) actor: Option, #[serde(flatten)] unknown: serde_yaml::Mapping, @@ -91,14 +101,67 @@ pub(crate) struct OperatorIdentity { pub(crate) struct OperatorDefaults { /// Default read output format, below every more-specific source. pub(crate) output: Option, - /// Table rendering preferences (below the legacy cli.table_* keys - /// during the RFC-008 window). + /// Table rendering preferences for `--format table`. pub(crate) table_max_column_width: Option, - pub(crate) table_cell_layout: Option, + pub(crate) table_cell_layout: Option, + /// Default server scope (RFC-011): the everyday addressing when no + /// `--profile` / primitive / legacy address is given. Names an entry + /// under `servers:`. Mutually exclusive with `store` — a scope binds one + /// entity. + pub(crate) server: Option, + /// Default **store** scope (RFC-011): a `file://` / `s3://` graph storage + /// URI used as the zero-flag local default for graph commands when no + /// `--profile` / primitive address is given. The local-dev counterpart of + /// `server`; mutually exclusive with it. + pub(crate) store: Option, + /// Default graph selected within a server/cluster scope when no + /// `--graph` is passed (RFC-011). + pub(crate) default_graph: Option, #[serde(flatten)] unknown: serde_yaml::Mapping, } +/// A named scope bundle (RFC-011): exactly one of {server, cluster, store} +/// plus an optional default graph. Validated on use (`binding()`), not at +/// parse time, so an unknown CLI's profile still loads. +#[derive(Debug, Default, Deserialize)] +pub(crate) struct OperatorProfile { + /// Names an entry under `servers:` — a served scope. + pub(crate) server: Option, + /// Names an entry under `clusters:` — a privileged direct cluster scope. + pub(crate) cluster: Option, + /// A single graph's storage URI — a direct store scope. + pub(crate) store: Option, + /// Default graph within a server/cluster scope (ignored for a store, + /// which is already one graph). + pub(crate) default_graph: Option, + #[serde(flatten)] + unknown: serde_yaml::Mapping, +} + +/// A managed-cluster storage root (RFC-011). +#[derive(Debug, Default, Deserialize)] +pub(crate) struct OperatorCluster { + /// The cluster's storage-root URI (`file://` / `s3://`). + pub(crate) root: String, + #[serde(flatten)] + unknown: serde_yaml::Mapping, +} + +/// The one entity a profile (or flat default) binds. Exactly one variant — +/// the scope resolver consumes this; "exactly one of server/cluster/store" +/// is enforced when producing it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ScopeBinding { + /// Served scope: a server name (resolved against `servers:`) or a literal URL. + Server(String), + /// Direct cluster scope: a cluster name (resolved against `clusters:`) or a + /// literal root URI. + Cluster(String), + /// Direct store scope: a single graph's storage URI. + Store(String), +} + impl OperatorConfig { pub(crate) fn actor(&self) -> Option<&str> { self.operator.actor.as_deref() @@ -127,6 +190,83 @@ impl OperatorConfig { } best.map(|(name, _)| name) } + + /// A named profile, if defined (RFC-011). + pub(crate) fn profile(&self, name: &str) -> Option<&OperatorProfile> { + self.profiles.get(name) + } + + /// The storage root of a named cluster, if defined (RFC-011). + pub(crate) fn cluster_root(&self, name: &str) -> Option<&str> { + self.clusters.get(name).map(|c| c.root.as_str()) + } + + /// The flat-default server scope name, if set (RFC-011). + pub(crate) fn default_server(&self) -> Option<&str> { + self.defaults.server.as_deref() + } + + /// The flat-default store scope URI, if set (RFC-011) — the zero-flag + /// local-dev default. + pub(crate) fn default_store(&self) -> Option<&str> { + self.defaults.store.as_deref() + } + + /// The flat-default graph within a server/cluster scope, if set (RFC-011). + pub(crate) fn default_graph(&self) -> Option<&str> { + self.defaults.default_graph.as_deref() + } + + /// A scope binds one entity (Decision 6): `defaults.server` and + /// `defaults.store` are mutually exclusive, and a `store` (already a single + /// graph) cannot carry a `default_graph`. Both are refused loudly rather + /// than silently dropped. + fn validate_defaults(&self) -> Result<()> { + if self.defaults.server.is_some() && self.defaults.store.is_some() { + bail!( + "operator config `defaults` sets both `server` and `store` — a default scope \ + binds one entity; keep one (use a `profile` if you need both)" + ); + } + if self.defaults.store.is_some() && self.defaults.default_graph.is_some() { + bail!( + "operator config `defaults` sets both `store` and `default_graph` — a store is \ + already a single graph; drop `default_graph` (it applies only to a server/cluster scope)" + ); + } + Ok(()) + } +} + +impl OperatorProfile { + /// The single entity this profile binds, or a loud error if it binds zero + /// or more than one of {server, cluster, store} (Decision 6: a scope binds + /// exactly one entity). Validated here, on use, rather than at parse time. + pub(crate) fn binding(&self, profile_name: &str) -> Result { + let set: Vec<&str> = [ + self.server.as_ref().map(|_| "server"), + self.cluster.as_ref().map(|_| "cluster"), + self.store.as_ref().map(|_| "store"), + ] + .into_iter() + .flatten() + .collect(); + match set.as_slice() { + ["server"] => Ok(ScopeBinding::Server(self.server.clone().unwrap())), + ["cluster"] => Ok(ScopeBinding::Cluster(self.cluster.clone().unwrap())), + ["store"] => Ok(ScopeBinding::Store(self.store.clone().unwrap())), + [] => Err(eyre!( + "profile '{profile_name}' binds no scope; set exactly one of \ + `server`, `cluster`, or `store`" + )), + many => Err(eyre!( + "profile '{profile_name}' binds {} scopes ({}); a profile must \ + bind exactly one of `server`, `cluster`, or `store`", + many.len(), + many.join(", ") + )), + } + } } /// The operator dir: `$OMNIGRAPH_HOME` if set (tilde-expanded), else @@ -172,6 +312,7 @@ pub(crate) fn load_operator_config_at(path: &Path) -> Result { for warning in config.unknown_key_warnings() { eprintln!("warning: {warning} in operator config '{}'", path.display()); } + config.validate_defaults()?; Ok(config) } @@ -196,6 +337,12 @@ impl OperatorConfig { for (name, alias) in &self.aliases { collect(&alias.unknown, &format!("aliases.{name}.")); } + for (name, profile) in &self.profiles { + collect(&profile.unknown, &format!("profiles.{name}.")); + } + for (name, cluster) in &self.clusters { + collect(&cluster.unknown, &format!("clusters.{name}.")); + } warnings } } @@ -444,6 +591,42 @@ mod tests { assert_eq!(config.output(), Some(ReadOutputFormat::Json)); } + #[test] + fn defaults_store_parses_and_is_accessible() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yaml"); + fs::write(&path, "defaults:\n store: file:///tmp/dev.omni\n").unwrap(); + let config = load_operator_config_at(&path).unwrap(); + assert_eq!(config.default_store(), Some("file:///tmp/dev.omni")); + assert_eq!(config.default_server(), None); + } + + #[test] + fn defaults_server_and_store_together_is_a_loud_error() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yaml"); + fs::write( + &path, + "defaults:\n server: prod\n store: file:///tmp/dev.omni\n", + ) + .unwrap(); + let err = load_operator_config_at(&path).unwrap_err().to_string(); + assert!(err.contains("binds one entity"), "{err}"); + } + + #[test] + fn defaults_store_with_default_graph_is_a_loud_error() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yaml"); + fs::write( + &path, + "defaults:\n store: file:///tmp/dev.omni\n default_graph: knowledge\n", + ) + .unwrap(); + let err = load_operator_config_at(&path).unwrap_err().to_string(); + assert!(err.contains("already a single graph"), "{err}"); + } + #[test] fn unknown_keys_warn_but_load() { // A file written for a later slice (servers/aliases) must load @@ -464,6 +647,82 @@ mod tests { assert_eq!(config.servers["prod"].url, "https://example.com"); } + #[test] + fn parses_profiles_clusters_and_scope_defaults() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yaml"); + let yaml = "\ +defaults: + server: prod + default_graph: knowledge +servers: + prod: + url: https://example.com +clusters: + brain: + root: s3://acme/clusters/brain +profiles: + staging: + server: staging + default_graph: knowledge + brain-admin: + cluster: brain + default_graph: knowledge +"; + fs::write(&path, yaml).unwrap(); + let config = load_operator_config_at(&path).unwrap(); + assert_eq!(config.default_server(), Some("prod")); + assert_eq!(config.default_graph(), Some("knowledge")); + assert_eq!(config.cluster_root("brain"), Some("s3://acme/clusters/brain")); + assert_eq!( + config.profile("staging").unwrap().binding("staging").unwrap(), + ScopeBinding::Server("staging".into()) + ); + assert_eq!( + config + .profile("brain-admin") + .unwrap() + .binding("brain-admin") + .unwrap(), + ScopeBinding::Cluster("brain".into()) + ); + // No unknown-key warnings for the new blocks. + assert!(config.unknown_key_warnings().is_empty(), "{:?}", config.unknown_key_warnings()); + } + + #[test] + fn profile_binding_rejects_zero_or_multiple_entities() { + let none = OperatorProfile::default(); + let err = none.binding("p").unwrap_err().to_string(); + assert!(err.contains("binds no scope"), "{err}"); + + let two = OperatorProfile { + server: Some("prod".into()), + store: Some("graph.omni".into()), + ..Default::default() + }; + let err = two.binding("p").unwrap_err().to_string(); + assert!(err.contains("binds 2 scopes"), "{err}"); + assert!(err.contains("server") && err.contains("store"), "{err}"); + } + + #[test] + fn unknown_keys_in_a_profile_warn() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yaml"); + fs::write( + &path, + "profiles:\n p:\n server: prod\n flavour: spicy\n", + ) + .unwrap(); + let config = load_operator_config_at(&path).unwrap(); + let warnings = config.unknown_key_warnings(); + assert!( + warnings.iter().any(|w| w.contains("`profiles.p.flavour`")), + "{warnings:?}" + ); + } + #[test] fn malformed_yaml_is_a_loud_error() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph-cli/src/output.rs b/crates/omnigraph-cli/src/output.rs index 964307b..d6903f4 100644 --- a/crates/omnigraph-cli/src/output.rs +++ b/crates/omnigraph-cli/src/output.rs @@ -21,7 +21,7 @@ pub(crate) struct LoadOutput { pub(crate) fn load_output_from_tables( uri: &str, branch: &str, - mode: CliLoadMode, + mode: &'static str, output: &IngestOutput, ) -> LoadOutput { let mut nodes_loaded = 0; @@ -40,7 +40,7 @@ pub(crate) fn load_output_from_tables( LoadOutput { uri: uri.to_string(), branch: branch.to_string(), - mode: mode.as_str(), + mode, base_branch: output.base_branch.clone(), branch_created: output.branch_created, nodes_loaded, @@ -50,6 +50,31 @@ pub(crate) fn load_output_from_tables( } } +/// The local arm's twin of `load_output_from_tables`: build the same +/// `LoadOutput` from the engine `LoadResult` directly (the remote arm only +/// has the wire `IngestOutput`'s table list; the local arm has the full +/// result). Both load mappings live here, next to the struct — RFC-009 +/// Phase 2's "one place" for the `-> LoadOutput` mapping that used to fork +/// between this file and main.rs's inline construction. +pub(crate) fn load_output_from_result( + uri: &str, + branch: &str, + mode: &'static str, + result: &omnigraph::loader::LoadResult, +) -> LoadOutput { + LoadOutput { + uri: uri.to_string(), + branch: branch.to_string(), + mode, + base_branch: result.base_branch.clone(), + branch_created: result.branch_created, + nodes_loaded: result.nodes_loaded.values().sum(), + edges_loaded: result.edges_loaded.values().sum(), + node_types_loaded: result.nodes_loaded.len(), + edge_types_loaded: result.edges_loaded.len(), + } +} + #[derive(Debug, Serialize)] pub(crate) struct SchemaPlanOutput<'a> { pub(crate) uri: &'a str, @@ -671,9 +696,24 @@ pub(crate) fn render_constraint(constraint: &omnigraph_compiler::schema::ast::Co pub(crate) fn render_annotations(annotations: &[omnigraph_compiler::schema::ast::Annotation]) -> String { annotations .iter() - .map(|annotation| match &annotation.value { - Some(value) => format!("@{}({})", annotation.name, value), - None => format!("@{}", annotation.name), + .map(|annotation| { + let mut args: Vec = Vec::new(); + // Values are parsed via `decode_string_literal` (quotes stripped), so + // re-quote them as string literals on render — otherwise a value with + // non-ident chars (e.g. `model=openai/text-embedding-3-large`) fails to + // round-trip back through the schema parser (`annotation_kwarg` wants a + // quoted `literal`, not a bare token). + if let Some(value) = &annotation.value { + args.push(format!("\"{}\"", value)); + } + for (key, val) in &annotation.kwargs { + args.push(format!("{}=\"{}\"", key, val)); + } + if args.is_empty() { + format!("@{}", annotation.name) + } else { + format!("@{}({})", annotation.name, args.join(", ")) + } }) .collect::>() .join(", ") @@ -709,15 +749,10 @@ pub(crate) fn print_snapshot_human(branch: &str, manifest_version: u64, entries: pub(crate) fn print_read_output( output: &ReadOutput, format: ReadOutputFormat, - config: &OmnigraphConfig, ) -> Result<()> { println!( "{}", - render_read( - output, - format, - &resolve_table_render_options(config), - )? + render_read(output, format, &resolve_table_render_options())? ); Ok(()) } @@ -787,10 +822,6 @@ pub(crate) fn print_policy_explain(decision: &PolicyDecision, actor_id: &str, re println!("message: {}", decision.message); } -pub(crate) fn yaml_string(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - #[derive(serde::Serialize)] pub(crate) struct QueriesIssue { pub(crate) query: String, @@ -871,20 +902,126 @@ pub(crate) fn finish_logout( Ok(()) } -/// Table prefs cascade (RFC-007/008): legacy cli.table_* (window) > -/// operator defaults.table_* > built-in. -pub(crate) fn resolve_table_render_options(config: &OmnigraphConfig) -> ReadRenderOptions { +#[derive(Debug, Serialize)] +pub(crate) struct ProfileListItem { + pub(crate) name: String, + /// `server: ` / `cluster: ` / `store: ` / `invalid: `. + pub(crate) binding: String, + /// `server` | `cluster` | `store` | `invalid`. + pub(crate) scope_kind: String, + /// The bound server/cluster name, or the store URI. `None` when invalid. + pub(crate) target: Option, + pub(crate) valid: bool, + pub(crate) error: Option, + pub(crate) default_graph: Option, + pub(crate) active: bool, +} + +#[derive(Debug, Serialize)] +pub(crate) struct ProfileDetail { + /// Profile name, or `(defaults)` for the no-name flat-defaults view. + pub(crate) name: String, + /// `server` | `cluster` | `store` | `none`. + pub(crate) scope_kind: String, + /// The bound server/cluster name, or the store URI. + pub(crate) target: Option, + /// Resolved endpoint: a server's URL / a cluster's root / the store URI; + /// `None` if a named server/cluster isn't defined in this config. + pub(crate) endpoint: Option, + pub(crate) default_graph: Option, + pub(crate) output_format: Option, +} + +pub(crate) fn print_profile_list(items: &[ProfileListItem], json: bool) -> Result<()> { + if json { + return print_json(&items); + } + if items.is_empty() { + println!("no profiles defined in the operator config"); + return Ok(()); + } + for item in items { + let active = if item.active { " (active)" } else { "" }; + let graph = item + .default_graph + .as_deref() + .map(|g| format!(" · graph: {g}")) + .unwrap_or_default(); + println!("{}{active} {}{graph}", item.name, item.binding); + } + Ok(()) +} + +pub(crate) fn print_profile_detail(detail: &ProfileDetail, json: bool) -> Result<()> { + if json { + return print_json(detail); + } + println!("profile: {}", detail.name); + let target = detail + .target + .as_deref() + .map(|t| format!(" {t}")) + .unwrap_or_default(); + println!(" scope: {}{target}", detail.scope_kind); + if let Some(endpoint) = &detail.endpoint { + println!(" endpoint: {endpoint}"); + } else if matches!(detail.scope_kind.as_str(), "server" | "cluster") { + println!(" endpoint: (undefined — name not in this config)"); + } + if let Some(graph) = &detail.default_graph { + println!(" default graph: {graph}"); + } + if let Some(format) = &detail.output_format { + println!(" output: {format}"); + } + Ok(()) +} + +/// Table prefs cascade (RFC-011): operator defaults.table_* > built-in. +pub(crate) fn resolve_table_render_options() -> ReadRenderOptions { let operator = crate::operator::load_operator_config().unwrap_or_default(); ReadRenderOptions { - max_column_width: config - .cli - .table_max_column_width - .or(operator.defaults.table_max_column_width) - .unwrap_or(80), - cell_layout: config - .cli - .table_cell_layout - .or(operator.defaults.table_cell_layout) - .unwrap_or_default(), + max_column_width: operator.defaults.table_max_column_width.unwrap_or(80), + cell_layout: operator.defaults.table_cell_layout.unwrap_or_default(), + } +} + +#[cfg(test)] +mod tests { + use omnigraph_compiler::schema::ast::Annotation; + use omnigraph_compiler::schema::parser::parse_schema; + use std::collections::BTreeMap; + + use super::render_annotations; + + #[test] + fn render_annotations_quotes_values_so_embed_round_trips() { + let mut kwargs = BTreeMap::new(); + kwargs.insert( + "model".to_string(), + "openai/text-embedding-3-large".to_string(), + ); + let embed = Annotation { + name: "embed".to_string(), + value: Some("title".to_string()), + kwargs, + }; + + let rendered = render_annotations(std::slice::from_ref(&embed)); + assert_eq!( + rendered, + r#"@embed("title", model="openai/text-embedding-3-large")"# + ); + + // The bug: an unquoted `model=openai/text-embedding-3-large` is not a + // valid `annotation_kwarg` literal, so `schema show` output did not + // re-parse. The rendered form must round-trip through the grammar. + let schema = format!("node Doc {{\ntitle: String\nembedding: Vector(3) {rendered}\n}}\n"); + let parsed = parse_schema(&schema); + assert!( + parsed.is_ok(), + "rendered @embed must re-parse: {:?}", + parsed.err() + ); } } diff --git a/crates/omnigraph-cli/src/planes.rs b/crates/omnigraph-cli/src/planes.rs new file mode 100644 index 0000000..b599076 --- /dev/null +++ b/crates/omnigraph-cli/src/planes.rs @@ -0,0 +1,357 @@ +//! Declared CLI "planes" (RFC-010 Slice 1). +//! +//! Every subcommand belongs to exactly one plane. This classification is the +//! single source of truth the wrong-plane guard consumes — and that later +//! RFC-010 slices (the capability surface, plane-grouped help) will consume +//! too. The `command_plane` match is **exhaustive on purpose**: adding a +//! `Command` variant is a compile error until its plane is declared, so the +//! surface cannot silently drift from the command set. +//! +//! See [docs/dev/rfc-010-cli-planes-restructure.md]. + +use color_eyre::Result; +use color_eyre::eyre::bail; + +use crate::cli::{Cli, Command, QueriesCommand, SchemaCommand}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Plane { + /// Runs against a graph, embedded **or** via `--server` (the `GraphClient` + /// axis). The only plane on which the data-plane addressing flags + /// (`--server`/`--graph`) apply. + Data, + /// Direct storage access; no server. Maintenance + local-only inspection + /// that must work with the server down. + Storage, + /// Operates on a cluster directory, not a graph URI. + Control, + /// Touches no graph at all — session / config / local tooling. + Session, +} + +impl std::fmt::Display for Plane { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Plane::Data => "data", + Plane::Storage => "storage", + Plane::Control => "control", + Plane::Session => "session", + }) + } +} + +/// What a command *needs*, in the user-facing vocabulary (RFC-011). This is the +/// language CLI errors and `--help` speak; `Plane` stays the internal classifier +/// (`Capability` is derived from it, so the two cannot drift). +/// +/// - `any` — graph-scoped data; served via a server scope, or direct against a +/// store scope. Accepts `--server`/`--graph`. +/// - `served` — requires a server. Accepts `--server`/`--graph`. +/// - `direct` — storage-native; opens storage directly, never through a server. +/// - `control` — operates on a cluster (control plane). +/// - `local` — addresses no graph at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Capability { + Any, + Served, + Direct, + Control, + Local, +} + +impl Capability { + /// A human phrase for error messages (`` `optimize` is a {…} command ``). + pub(crate) fn describe(self) -> &'static str { + match self { + Capability::Any => "data", + Capability::Served => "served", + Capability::Direct => "direct (storage-native)", + Capability::Control => "cluster control", + Capability::Local => "local", + } + } + + /// `--server`/`--graph` are served-graph addressing: they apply only to the + /// capabilities that reach a graph through a server. + fn accepts_server_addressing(self) -> bool { + matches!(self, Capability::Any | Capability::Served) + } +} + +/// The capability a subcommand needs, derived from its `Plane` (the exhaustive +/// classifier) plus the one Data→Served refinement: `graphs` is remote-only. +/// +/// This reflects *current enforced behavior*, so messages stay truthful: +/// `queries`/`policy` read a cluster's applied state (`Control`). +pub(crate) fn command_capability(cmd: &Command) -> Capability { + if let Command::Graphs { .. } = cmd { + return Capability::Served; + } + match command_plane(cmd) { + Plane::Data => Capability::Any, + Plane::Storage => Capability::Direct, + Plane::Control => Capability::Control, + Plane::Session => Capability::Local, + } +} + +/// The plane a subcommand belongs to. Exhaustive — a new `Command` variant +/// will not compile until classified. Descends into the nested enums where +/// the plane differs per subcommand (`schema plan` is storage while `schema +/// show`/`apply` are data; `queries`/`policy` read cluster applied state). +pub(crate) fn command_plane(cmd: &Command) -> Plane { + match cmd { + Command::Query { .. } + | Command::Mutate { .. } + | Command::Load { .. } + | Command::Ingest { .. } + | Command::Branch { .. } + | Command::Snapshot { .. } + | Command::Export { .. } + | Command::Commit { .. } + | Command::Graphs { .. } => Plane::Data, + Command::Schema { + command: SchemaCommand::Show { .. } | SchemaCommand::Apply { .. }, + } => Plane::Data, + Command::Schema { + command: SchemaCommand::Plan { .. }, + } => Plane::Storage, + // `queries` and `policy` tooling now source their inputs from a + // cluster's applied state (`--cluster`), so they live on the control + // plane (RFC-011 — omnigraph.yaml excised from the CLI). + Command::Queries { .. } => Plane::Control, + Command::Policy { .. } => Plane::Control, + Command::Init { .. } + | Command::Optimize { .. } + | Command::Repair { .. } + | Command::Cleanup { .. } + | Command::Lint { .. } => Plane::Storage, + Command::Cluster { .. } => Plane::Control, + Command::Alias { .. } + | Command::Embed(_) + | Command::Login { .. } + | Command::Logout { .. } + | Command::Profile { .. } + | Command::Version => Plane::Session, + } +} + +/// User-facing label for a subcommand (descends one level for the nested +/// families so messages read `schema plan`, `queries validate`, etc.). +pub(crate) fn command_label(cmd: &Command) -> &'static str { + match cmd { + Command::Version => "version", + Command::Login { .. } => "login", + Command::Logout { .. } => "logout", + Command::Profile { .. } => "profile", + Command::Embed(_) => "embed", + Command::Init { .. } => "init", + Command::Load { .. } => "load", + Command::Ingest { .. } => "ingest", + Command::Branch { .. } => "branch", + Command::Schema { command } => match command { + SchemaCommand::Plan { .. } => "schema plan", + SchemaCommand::Apply { .. } => "schema apply", + SchemaCommand::Show { .. } => "schema show", + }, + Command::Lint { .. } => "lint", + Command::Queries { command } => match command { + QueriesCommand::Validate { .. } => "queries validate", + QueriesCommand::List { .. } => "queries list", + }, + Command::Snapshot { .. } => "snapshot", + Command::Export { .. } => "export", + Command::Commit { .. } => "commit", + Command::Query { .. } => "query", + Command::Mutate { .. } => "mutate", + Command::Alias { .. } => "alias", + Command::Policy { .. } => "policy", + Command::Optimize { .. } => "optimize", + Command::Repair { .. } => "repair", + Command::Cleanup { .. } => "cleanup", + Command::Cluster { .. } => "cluster", + Command::Graphs { .. } => "graphs", + } +} + +/// The verbs that consume a cluster scope. Maintenance/lint select a graph with +/// `--cluster --graph `; policy/queries inspect the cluster's +/// applied control-plane state and may optionally use `--graph` to select one +/// bundle/registry. `init` is storage-plane too but *creates* a graph (cluster +/// graphs are born from `cluster apply`, not `init`), and `schema plan` takes a +/// positional URI, so the guard rejects `--cluster`/`--graph` there rather than +/// silently dropping the flag. +pub(crate) fn accepts_cluster_addressing(cmd: &Command) -> bool { + matches!( + cmd, + Command::Optimize { .. } + | Command::Repair { .. } + | Command::Cleanup { .. } + // `lint` can type-check a `.gq` against a cluster graph's schema + // (RFC-011): `--cluster --graph `. + | Command::Lint { .. } + // The policy/queries tooling addresses a cluster's applied state + // (RFC-011): `--cluster ` selects the cluster, `--graph ` + // picks a graph's bundle/registry within it. + | Command::Policy { .. } + | Command::Queries { .. } + ) +} + +/// Reject a scope-addressing flag (`--server`/`--cluster`/`--graph`) on a verb +/// that cannot consume it, rather than silently dropping it (the old behavior: +/// e.g. `optimize --server prod` dropped `--server` and failed later with an +/// unrelated message). `alias` gets an extra guard because its binding owns all +/// addressing and several ignored globals sit outside this three-flag guard. +/// Each flag has a distinct valid surface: +/// - `--server` → served-graph scopes (`any`/`served`); +/// - `--cluster` → cluster-scoped direct/control verbs; +/// - `--graph` → any multi-graph scope: a served scope *or* a cluster one. +/// RFC-010 Slice 1, generalized for RFC-011 cluster addressing. +pub(crate) fn guard_addressing(cli: &Cli) -> Result<()> { + if let Command::Alias { .. } = &cli.command { + let mut flags = Vec::new(); + if cli.server.is_some() { + flags.push("--server"); + } + if cli.graph.is_some() { + flags.push("--graph"); + } + if cli.store.is_some() { + flags.push("--store"); + } + if cli.cluster.is_some() { + flags.push("--cluster"); + } + if cli.profile.is_some() { + flags.push("--profile"); + } + if cli.as_actor.is_some() { + flags.push("--as"); + } + if !flags.is_empty() { + bail!( + "`alias` uses the server, graph, and stored query declared in \ + `aliases.` in ~/.omnigraph/config.yaml; remove global scope \ + flag(s): {}", + flags.join(", ") + ); + } + } + if cli.server.is_none() && cli.cluster.is_none() && cli.graph.is_none() { + return Ok(()); + } + let capability = command_capability(&cli.command); + let label = command_label(&cli.command); + let cluster_ok = accepts_cluster_addressing(&cli.command); + + if cli.server.is_some() && !capability.accepts_server_addressing() { + bail!( + "`{label}` is a {} command; --server addresses a served graph and does not apply.{}", + capability.describe(), + remediation(capability, &cli.command), + ); + } + if cli.cluster.is_some() && !cluster_ok { + bail!( + "`{label}` is a {} command; --cluster addresses a cluster-scoped command \ + and does not apply.{}", + capability.describe(), + remediation(capability, &cli.command), + ); + } + if cli.graph.is_some() && !(capability.accepts_server_addressing() || cluster_ok) { + bail!( + "`{label}` is a {} command; --graph selects a graph within a server or cluster \ + scope and does not apply.{}", + capability.describe(), + remediation(capability, &cli.command), + ); + } + Ok(()) +} + +/// The "what to do instead" tail for a wrong-address error, by capability. +/// Includes its own leading space when non-empty so the caller appends it +/// directly — an empty tail (the served-addressing capabilities, which only +/// reach this fn for a misplaced `--cluster`/`--graph`) leaves no trailing space. +fn remediation(capability: Capability, cmd: &Command) -> &'static str { + match capability { + Capability::Direct => match cmd { + Command::Init { .. } => " Pass a storage URI.", + Command::Optimize { .. } | Command::Repair { .. } | Command::Cleanup { .. } => { + " Pass a storage URI, or --cluster --graph ." + } + _ => " Pass a storage URI.", + }, + Capability::Control => match cmd { + Command::Cluster { .. } => { + " It operates on a cluster config directory (pass --config )." + } + Command::Policy { .. } | Command::Queries { .. } => { + " It operates on a cluster (pass --cluster , or select a cluster profile)." + } + _ => " It operates on a cluster.", + }, + Capability::Local => " It does not address a graph.", + Capability::Any | Capability::Served => "", + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + + #[test] + fn server_addressing_allowed_exactly_on_any_and_served() { + // The behavior-preservation contract: `--server`/`--graph` apply to the + // served-graph capabilities (`any`, `served`) and nothing else. This is + // the old "Data plane only" allow set, re-expressed — graphs (the one + // Data→Served verb) was already allowed. + assert!(Capability::Any.accepts_server_addressing()); + assert!(Capability::Served.accepts_server_addressing()); + assert!(!Capability::Direct.accepts_server_addressing()); + assert!(!Capability::Control.accepts_server_addressing()); + assert!(!Capability::Local.accepts_server_addressing()); + } + + #[test] + fn command_capability_classifies_representative_verbs() { + let cap = |args: &[&str]| { + command_capability(&Cli::try_parse_from(args).unwrap().command) + }; + // The one Data→Served refinement — if the `graphs` guard were deleted, + // every other assertion here would still pass. + assert_eq!(cap(&["omnigraph", "graphs", "list"]), Capability::Served); + assert_eq!(cap(&["omnigraph", "alias", "who"]), Capability::Local); + assert_eq!(cap(&["omnigraph", "optimize", "graph.omni"]), Capability::Direct); + assert_eq!(cap(&["omnigraph", "schema", "plan", "--schema", "s.pg", "graph.omni"]), Capability::Direct); + assert_eq!(cap(&["omnigraph", "cluster", "status", "--config", "."]), Capability::Control); + assert_eq!(cap(&["omnigraph", "version"]), Capability::Local); + // `queries`/`policy` tooling reads cluster state now (control plane). + assert_eq!(cap(&["omnigraph", "queries", "list"]), Capability::Control); + assert_eq!( + cap(&["omnigraph", "policy", "validate"]), + Capability::Control + ); + } + + #[test] + fn every_capability_describes_distinctly() { + let phrases = [ + Capability::Any.describe(), + Capability::Served.describe(), + Capability::Direct.describe(), + Capability::Control.describe(), + Capability::Local.describe(), + ]; + for (i, a) in phrases.iter().enumerate() { + assert!(!a.is_empty()); + for b in &phrases[i + 1..] { + assert_ne!(a, b); + } + } + } +} diff --git a/crates/omnigraph-cli/src/read_format.rs b/crates/omnigraph-cli/src/read_format.rs index b205b19..3ffa9e6 100644 --- a/crates/omnigraph-cli/src/read_format.rs +++ b/crates/omnigraph-cli/src/read_format.rs @@ -1,9 +1,31 @@ +use clap::ValueEnum; use color_eyre::eyre::Result; -use omnigraph_server::ReadOutputFormat; use omnigraph_server::api::ReadOutput; -use omnigraph_server::config::TableCellLayout; +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +/// Output rendering format for read-shaped commands (`read`/`query`/`alias`). +/// A CLI presentation concern — lives here, not in the server. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "snake_case")] +pub enum ReadOutputFormat { + #[default] + Table, + Kv, + Csv, + Jsonl, + Json, +} + +/// How an over-wide table cell is laid out when rendering `--format table`. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, ValueEnum)] +#[serde(rename_all = "snake_case")] +pub enum TableCellLayout { + #[default] + Truncate, + Wrap, +} + pub struct ReadRenderOptions { pub max_column_width: usize, pub cell_layout: TableCellLayout, diff --git a/crates/omnigraph-cli/src/scope.rs b/crates/omnigraph-cli/src/scope.rs new file mode 100644 index 0000000..257907d --- /dev/null +++ b/crates/omnigraph-cli/src/scope.rs @@ -0,0 +1,529 @@ +//! RFC-011 Slice A scope resolution. +//! +//! Translates the new scope inputs (`--profile` / `--store` / operator-config +//! `profiles`/`clusters`/`defaults`) into the SAME effective addressing tuple +//! the existing `GraphClient` factories (`client.rs`) and the maintenance +//! resolver (`helpers::resolve_storage_uri`) already consume. This is a +//! translation layer that sits *in front* of those resolvers — it is purely +//! additive: an explicit legacy address (`--uri`/`--target`/`--server`/ +//! `--store`) wins and reproduces today's behavior exactly, so existing +//! invocations are unaffected. +//! +//! The access path (served vs direct) is never chosen here; it falls out of the +//! scope's binding × the verb's capability. The capability→scope check rejects +//! mismatches (e.g. a server scope on a maintenance verb) only on the *new* +//! resolution paths. + +use std::env; + +use color_eyre::Result; +use color_eyre::eyre::{bail, eyre}; + +use crate::operator::{OperatorConfig, ScopeBinding}; +use crate::planes::Capability; + +pub(crate) const PROFILE_ENV: &str = "OMNIGRAPH_PROFILE"; + +/// The effective addressing a command should use, in the terms the existing +/// resolvers consume. Data/served verbs read `server`/`graph`/`uri`/`target`; +/// maintenance verbs read `cluster`/`cluster_graph`. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct ResolvedScope { + pub(crate) server: Option, + pub(crate) graph: Option, + pub(crate) uri: Option, + pub(crate) cluster: Option, + pub(crate) cluster_graph: Option, +} + +/// The raw addressing inputs for one command: the global scope flags plus the +/// command's own positional URI. +pub(crate) struct ScopeFlags<'a> { + pub(crate) profile: Option<&'a str>, + pub(crate) store: Option<&'a str>, + pub(crate) server: Option<&'a str>, + pub(crate) cluster: Option<&'a str>, + pub(crate) graph: Option<&'a str>, + pub(crate) uri: Option, +} + +/// Resolve the scope for a command with `capability`. Precedence (RFC-011): +/// 1. explicit primitive address (`uri`/`--server`/`--store`) → passthrough; +/// 2. `--profile` / `OMNIGRAPH_PROFILE`; +/// 3. flat `defaults.server` + `defaults.default_graph`; +/// 4. nothing — downstream behaves as today. +pub(crate) fn resolve_scope( + op: &OperatorConfig, + capability: Capability, + flags: ScopeFlags<'_>, +) -> Result { + // At most one explicit scope primitive may address a command — a positional + // URI, `--store`, `--server`, or `--cluster` are mutually exclusive ways to + // name the graph. Combining them is a contradiction, not a silent precedence. + let primitives: Vec<&str> = [ + flags.uri.as_deref().map(|_| "a positional URI"), + flags.store.map(|_| "--store"), + flags.server.map(|_| "--server"), + flags.cluster.map(|_| "--cluster"), + ] + .into_iter() + .flatten() + .collect(); + if primitives.len() > 1 { + bail!( + "{} are mutually exclusive — pick one way to address the graph", + primitives.join(" and ") + ); + } + + // 1a. `--cluster` is the cluster scope primitive (maintenance): resolve its + // root + select the graph with `--graph`. + if let Some(cluster) = flags.cluster { + return scope_from_binding( + op, + capability, + ScopeBinding::Cluster(cluster.to_string()), + flags.graph.map(str::to_string), + "--cluster", + ); + } + + // 1b. Any other explicit address wins; reproduce today's behavior untouched. + // `--store` is an explicit store URI — fold it into `uri`. + if flags.uri.is_some() || flags.server.is_some() || flags.store.is_some() { + // `--graph` selects within a multi-graph scope; a bare positional URI / + // `--store` is already a single graph, so a stray `--graph` is an error + // rather than a silently-dropped flag. + if flags.graph.is_some() && flags.server.is_none() { + bail!( + "--graph selects a graph within a server or cluster scope; a positional \ + URI / --store is already a single graph" + ); + } + return Ok(ResolvedScope { + server: flags.server.map(str::to_string), + graph: flags.graph.map(str::to_string), + uri: flags.store.map(str::to_string).or(flags.uri), + ..Default::default() + }); + } + + // 2. A named profile (flag, else env). + let profile_name = flags + .profile + .map(str::to_string) + .or_else(|| env::var(PROFILE_ENV).ok().filter(|s| !s.is_empty())); + if let Some(name) = profile_name { + let profile = op.profile(&name).ok_or_else(|| { + eyre!("unknown profile '{name}' (not defined under `profiles:` in operator config)") + })?; + let binding = profile.binding(&name)?; + let graph = flags + .graph + .map(str::to_string) + .or_else(|| profile.default_graph.clone()); + return scope_from_binding(op, capability, binding, graph, &format!("profile '{name}'")); + } + + // 3. Flat default server scope. + if let Some(server) = op.default_server() { + let graph = flags + .graph + .map(str::to_string) + .or_else(|| op.default_graph().map(str::to_string)); + return scope_from_binding( + op, + capability, + ScopeBinding::Server(server.to_string()), + graph, + "operator defaults", + ); + } + + // 3b. Flat default store scope — the zero-flag local-dev default (RFC-011). + // Mutually exclusive with `defaults.server` (enforced at config load). + if let Some(store) = op.default_store() { + return scope_from_binding( + op, + capability, + ScopeBinding::Store(store.to_string()), + flags.graph.map(str::to_string), + "operator defaults", + ); + } + + // 4. Nothing resolved — leave the tuple empty; downstream falls through to + // today's behavior (legacy `cli.graph` default or a no-address error). + Ok(ResolvedScope::default()) +} + +/// Map a resolved binding to the effective tuple, enforcing scope × capability +/// capability (RFC-011): a server scope is served (data only); a cluster scope +/// is privileged direct (maintenance/control only); a store scope is direct +/// (either). +fn scope_from_binding( + op: &OperatorConfig, + capability: Capability, + binding: ScopeBinding, + graph: Option, + source: &str, +) -> Result { + match binding { + ScopeBinding::Server(server) => { + if capability == Capability::Direct { + bail!( + "this command needs direct storage access, but {source} resolves a \ + server scope; name storage explicitly with --store (or \ + --cluster --graph for a managed graph)" + ); + } + Ok(ResolvedScope { + server: Some(server), + graph, + ..Default::default() + }) + } + ScopeBinding::Cluster(cluster) => { + if capability == Capability::Any { + bail!( + "{source} resolves a cluster scope, which is not valid for graph data \ + commands; run data commands through a server, or use --store \ + for ad-hoc direct access" + ); + } + // A cluster value is a config name (resolved against `clusters:`) + // or a literal root: an `s3://`/`file://` URI or a local cluster + // directory. Only a configured name is rewritten; anything else is + // passed through to the cluster-state resolver verbatim, so a bare + // directory path keeps working as it did for per-command `--cluster`. + let root = op + .cluster_root(&cluster) + .map(str::to_string) + .unwrap_or(cluster); + // A cluster holds many graphs; maintenance addresses one at a time. + // When no `--graph`/`default_graph` is given, leave `cluster_graph` + // empty and defer to the async storage-URI resolver (RFC-011 D7), + // which enumerates the catalog: auto-use a sole graph, else error + // and list the candidates. + Ok(ResolvedScope { + cluster: Some(root), + cluster_graph: graph, + ..Default::default() + }) + } + ScopeBinding::Store(uri) => { + if graph.is_some() { + bail!( + "--graph does not apply to a store scope ({source}): a store is already \ + a single graph" + ); + } + Ok(ResolvedScope { + uri: Some(uri), + ..Default::default() + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg(yaml: &str) -> OperatorConfig { + serde_yaml::from_str(yaml).unwrap() + } + + fn flags<'a>() -> ScopeFlags<'a> { + ScopeFlags { + profile: None, + store: None, + server: None, + cluster: None, + graph: None, + uri: None, + } + } + + #[test] + fn explicit_legacy_address_wins_unchanged() { + let op = cfg("defaults:\n server: prod\nservers:\n prod:\n url: https://x\n"); + // A positional URI given → profile/defaults are ignored entirely. + let scope = resolve_scope( + &op, + Capability::Any, + ScopeFlags { + uri: Some("graph.omni".into()), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.uri.as_deref(), Some("graph.omni")); + assert_eq!(scope.server, None); + } + + #[test] + fn store_flag_folds_into_uri_and_rejects_graph() { + let op = OperatorConfig::default(); + let scope = resolve_scope( + &op, + Capability::Any, + ScopeFlags { + store: Some("s3://b/g.omni"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.uri.as_deref(), Some("s3://b/g.omni")); + } + + #[test] + fn scope_primitives_are_mutually_exclusive() { + let op = OperatorConfig::default(); + for flags in [ + ScopeFlags { + store: Some("s3://b/g.omni"), + uri: Some("file://other.omni".into()), + ..flags() + }, + ScopeFlags { + store: Some("s3://b/g.omni"), + server: Some("prod"), + ..flags() + }, + ScopeFlags { + cluster: Some("./brain"), + uri: Some("file://other.omni".into()), + ..flags() + }, + ScopeFlags { + cluster: Some("./brain"), + server: Some("prod"), + ..flags() + }, + ] { + let err = resolve_scope(&op, Capability::Direct, flags) + .unwrap_err() + .to_string(); + assert!(err.contains("mutually exclusive"), "{err}"); + } + } + + #[test] + fn cluster_flag_resolves_root_and_graph_for_maintenance() { + let op = cfg("clusters:\n brain:\n root: s3://acme/brain\n"); + let scope = resolve_scope( + &op, + Capability::Direct, + ScopeFlags { + cluster: Some("brain"), + graph: Some("knowledge"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.cluster.as_deref(), Some("s3://acme/brain")); + assert_eq!(scope.cluster_graph.as_deref(), Some("knowledge")); + } + + #[test] + fn cluster_flag_accepts_a_literal_root_uri() { + let op = OperatorConfig::default(); + let scope = resolve_scope( + &op, + Capability::Direct, + ScopeFlags { + cluster: Some("s3://bucket/clusters/brain"), + graph: Some("knowledge"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.cluster.as_deref(), Some("s3://bucket/clusters/brain")); + assert_eq!(scope.cluster_graph.as_deref(), Some("knowledge")); + } + + #[test] + fn cluster_scope_without_a_graph_defers_to_catalog_enumeration() { + // RFC-011 D7: with no `--graph`/`default_graph`, resolution no longer + // bails here — it resolves the cluster root and leaves `cluster_graph` + // empty, deferring to the async storage-URI resolver (which enumerates + // the catalog: auto-use a sole graph, else error listing candidates). + let op = cfg("clusters:\n brain:\n root: s3://acme/brain\n"); + let scope = resolve_scope( + &op, + Capability::Direct, + ScopeFlags { + cluster: Some("brain"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.cluster.as_deref(), Some("s3://acme/brain")); + assert_eq!(scope.cluster_graph, None); + } + + #[test] + fn graph_on_a_bare_store_or_uri_is_rejected() { + let op = OperatorConfig::default(); + for flags in [ + ScopeFlags { + uri: Some("graph.omni".into()), + graph: Some("knowledge"), + ..flags() + }, + ScopeFlags { + store: Some("s3://b/g.omni"), + graph: Some("knowledge"), + ..flags() + }, + ] { + let err = resolve_scope(&op, Capability::Any, flags) + .unwrap_err() + .to_string(); + assert!(err.contains("already a single graph"), "{err}"); + } + } + + #[test] + fn flat_default_store_drives_local_verbs() { + // RFC-011: `defaults.store` is the zero-flag local default — no flags, + // no profile → the store URI resolves as the (single-graph) store scope. + let op = cfg("defaults:\n store: file:///tmp/dev.omni\n"); + let scope = resolve_scope(&op, Capability::Any, flags()).unwrap(); + assert_eq!(scope.uri.as_deref(), Some("file:///tmp/dev.omni")); + assert_eq!(scope.server, None); + } + + #[test] + fn flat_default_store_rejects_graph() { + // A store is already a single graph, so `--graph` against a default + // store is a loud error. + let op = cfg("defaults:\n store: file:///tmp/dev.omni\n"); + let err = resolve_scope( + &op, + Capability::Any, + ScopeFlags { + graph: Some("knowledge"), + ..flags() + }, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("does not apply to a store scope"), "{err}"); + } + + #[test] + fn flat_default_server_drives_data_verbs() { + let op = cfg("defaults:\n server: prod\n default_graph: knowledge\nservers:\n prod:\n url: https://x\n"); + let scope = resolve_scope(&op, Capability::Any, flags()).unwrap(); + assert_eq!(scope.server.as_deref(), Some("prod")); + assert_eq!(scope.graph.as_deref(), Some("knowledge")); + } + + #[test] + fn profile_server_scope_with_graph_override() { + let op = cfg( + "servers:\n staging:\n url: https://s\nprofiles:\n staging:\n server: staging\n default_graph: knowledge\n", + ); + let scope = resolve_scope( + &op, + Capability::Any, + ScopeFlags { + profile: Some("staging"), + graph: Some("archive"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.server.as_deref(), Some("staging")); + assert_eq!(scope.graph.as_deref(), Some("archive")); // flag beats profile default + } + + #[test] + fn profile_cluster_scope_resolves_root_for_maintenance() { + let op = cfg( + "clusters:\n brain:\n root: s3://acme/brain\nprofiles:\n admin:\n cluster: brain\n default_graph: knowledge\n", + ); + let scope = resolve_scope( + &op, + Capability::Direct, + ScopeFlags { + profile: Some("admin"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.cluster.as_deref(), Some("s3://acme/brain")); + assert_eq!(scope.cluster_graph.as_deref(), Some("knowledge")); + } + + #[test] + fn profile_cluster_scope_with_graph_override() { + // The deferral closed by this slice: a `--graph` flag overrides a + // profile cluster's default_graph, exactly as it does for a server scope. + let op = cfg( + "clusters:\n brain:\n root: s3://acme/brain\nprofiles:\n admin:\n cluster: brain\n default_graph: knowledge\n", + ); + let scope = resolve_scope( + &op, + Capability::Direct, + ScopeFlags { + profile: Some("admin"), + graph: Some("archive"), + ..flags() + }, + ) + .unwrap(); + assert_eq!(scope.cluster.as_deref(), Some("s3://acme/brain")); + assert_eq!(scope.cluster_graph.as_deref(), Some("archive")); // flag beats profile default + } + + #[test] + fn server_scope_on_maintenance_verb_errors() { + let op = cfg("defaults:\n server: prod\nservers:\n prod:\n url: https://x\n"); + let err = resolve_scope(&op, Capability::Direct, flags()).unwrap_err().to_string(); + assert!(err.contains("direct storage access"), "{err}"); + } + + #[test] + fn cluster_scope_on_data_verb_errors() { + let op = cfg( + "clusters:\n brain:\n root: s3://acme/brain\nprofiles:\n admin:\n cluster: brain\n", + ); + let err = resolve_scope( + &op, + Capability::Any, + ScopeFlags { + profile: Some("admin"), + ..flags() + }, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("not valid for graph data commands"), "{err}"); + } + + #[test] + fn unknown_profile_is_a_loud_error() { + let op = OperatorConfig::default(); + let err = resolve_scope( + &op, + Capability::Any, + ScopeFlags { + profile: Some("nope"), + ..flags() + }, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("unknown profile 'nope'"), "{err}"); + } + + #[test] + fn no_address_resolves_empty_for_legacy_fallthrough() { + let op = OperatorConfig::default(); + let scope = resolve_scope(&op, Capability::Any, flags()).unwrap(); + assert_eq!(scope, ResolvedScope::default()); + } +} diff --git a/crates/omnigraph-cli/tests/cli_cluster.rs b/crates/omnigraph-cli/tests/cli_cluster.rs index 3b2eed3..e35a54d 100644 --- a/crates/omnigraph-cli/tests/cli_cluster.rs +++ b/crates/omnigraph-cli/tests/cli_cluster.rs @@ -683,51 +683,8 @@ fn cluster_apply_locked_exits_nonzero() { assert!(!temp.path().join("__cluster/resources").exists()); } -#[test] -fn cluster_apply_uses_cli_actor_from_local_config() { - let temp = tempdir().unwrap(); - write_cluster_config_fixture(temp.path()); - fs::write( - temp.path().join("omnigraph.yaml"), - "cli:\n actor: act-local\n", - ) - .unwrap(); - // Phase 1: import once (setup, not under test). - let output = cli() - .current_dir(temp.path()) - .arg("cluster") - .arg("import") - .arg("--config") - .arg(temp.path()) - .output() - .unwrap(); - assert!(output.status.success(), "{output:?}"); - - // Phase 2: apply alone, capturing the echoed actor (idempotent re-runs). - let apply = |extra: &[&str]| { - let mut command = cli(); - command.current_dir(temp.path()); - for arg in extra { - command.arg(arg); - } - let output = command - .arg("cluster") - .arg("apply") - .arg("--config") - .arg(temp.path()) - .arg("--json") - .output() - .unwrap(); - let json: serde_json::Value = - serde_json::from_str(String::from_utf8_lossy(&output.stdout).trim()).unwrap(); - json["actor"].clone() - }; - assert_eq!(apply(&[]), "act-local", "cli.actor is the no-flag default"); - assert_eq!(apply(&["--as", "andrew"]), "andrew", "--as overrides cli.actor"); -} - -/// RFC-007 PR 1: the operator layer joins the actor chain — -/// `--as` > legacy `cli.actor` (RFC-008 window) > `operator.actor` > none. +/// RFC-011: the actor chain is `--as` > `operator.actor` > none. The CLI no +/// longer reads omnigraph.yaml `cli.actor`. #[test] fn cluster_apply_uses_operator_actor_from_omnigraph_home() { let temp = tempdir().unwrap(); @@ -771,41 +728,31 @@ fn cluster_apply_uses_operator_actor_from_omnigraph_home() { json["actor"].clone() }; - // No --as, no omnigraph.yaml: the operator identity applies. + // No --as: the operator identity applies. assert_eq!( apply(&[]), "act-operator", - "operator.actor is the no-flag, no-legacy-config default" + "operator.actor is the no-flag default" ); - // --as still wins over everything. + // --as still wins over the operator layer. assert_eq!(apply(&["--as", "andrew"]), "andrew"); - - // A legacy cli.actor (RFC-008 window) outranks the operator layer. - fs::write( - temp.path().join("omnigraph.yaml"), - "cli:\n actor: act-legacy\n", - ) - .unwrap(); - assert_eq!( - apply(&[]), - "act-legacy", - "legacy cli.actor wins over operator.actor during the deprecation window" - ); } #[test] -fn cluster_approve_uses_cli_actor_fallback() { +fn cluster_approve_uses_operator_actor_fallback() { let temp = tempdir().unwrap(); write_cluster_config_fixture(temp.path()); + let operator_home = tempdir().unwrap(); fs::write( - temp.path().join("omnigraph.yaml"), - "cli:\n actor: act-local\n", + operator_home.path().join("config.yaml"), + "operator:\n actor: act-operator\n", ) .unwrap(); // Converge, then remove the graph so a gated delete is pending. for command in ["import", "apply"] { let output = cli() .current_dir(temp.path()) + .env("OMNIGRAPH_HOME", operator_home.path()) .arg("cluster") .arg(command) .arg("--config") @@ -818,6 +765,7 @@ fn cluster_approve_uses_cli_actor_fallback() { let output = cli() .current_dir(temp.path()) + .env("OMNIGRAPH_HOME", operator_home.path()) .arg("cluster") .arg("approve") .arg("graph.knowledge") @@ -829,14 +777,17 @@ fn cluster_approve_uses_cli_actor_fallback() { assert!(output.status.success(), "{output:?}"); let json: serde_json::Value = serde_json::from_str(String::from_utf8_lossy(&output.stdout).trim()).unwrap(); - assert_eq!(json["approved_by"], "act-local"); + assert_eq!(json["approved_by"], "act-operator"); - // With neither flag nor config: refused with the actionable message. + // With neither flag nor operator config: refused with the actionable + // message (an approval without an approver is meaningless). let bare = tempdir().unwrap(); write_cluster_config_fixture(bare.path()); + let bare_home = tempdir().unwrap(); let output = output_failure( cli() .current_dir(bare.path()) + .env("OMNIGRAPH_HOME", bare_home.path()) .arg("cluster") .arg("approve") .arg("graph.knowledge") @@ -845,11 +796,13 @@ fn cluster_approve_uses_cli_actor_fallback() { ); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("--as"), "{stderr}"); - assert!(stderr.contains("cli.actor"), "{stderr}"); } #[test] -fn cluster_commands_ignore_malformed_local_config() { +fn cluster_commands_ignore_legacy_omnigraph_yaml() { + // RFC-011: the CLI never reads omnigraph.yaml for cluster commands — a + // present (even malformed) legacy file is inert. The actor falls back to + // `operator.actor`, then to none (no loud failure on absence). let temp = tempdir().unwrap(); write_cluster_config_fixture(temp.path()); fs::write(temp.path().join("omnigraph.yaml"), "{{{{ not yaml").unwrap(); @@ -873,14 +826,11 @@ fn cluster_commands_ignore_malformed_local_config() { "cluster {command} touched omnigraph.yaml" ); } - // import + apply with an explicit --as: the config is never loaded. - for (command, args) in [("import", vec![]), ("apply", vec!["--as", "andrew"])] { - let mut invocation = cli(); - invocation.current_dir(temp.path()); - for arg in &args { - invocation.arg(arg); - } - let output = invocation + // import + apply (no --as, no operator config): the legacy file is never + // loaded and the no-actor apply succeeds (actor defaults to none). + for command in ["import", "apply"] { + let output = cli() + .current_dir(temp.path()) .arg("cluster") .arg(command) .arg("--config") @@ -893,20 +843,6 @@ fn cluster_commands_ignore_malformed_local_config() { String::from_utf8_lossy(&output.stderr) ); } - // Only the no-flag actor lookup is allowed to fail, and loudly. - let output = output_failure( - cli() - .current_dir(temp.path()) - .arg("cluster") - .arg("apply") - .arg("--config") - .arg(temp.path()), - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("omnigraph.yaml") && stderr.contains("--as"), - "the actor-default config read must fail loudly and actionably: {stderr}" - ); } #[test] @@ -950,3 +886,240 @@ graphs: assert!(!leaked.contains("phantom") && !leaked.contains("9999"), "{leaked}"); } + +// ── RFC-010 Slice 3: cluster-managed maintenance addressing + init signpost ── + +/// Stand up an applied, served cluster with the `knowledge` graph and return +/// its directory guard. Mirrors the e2e setup (fixture → init → import → apply). +fn applied_knowledge_cluster() -> tempfile::TempDir { + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + init_cluster_derived_graph(temp.path()); + let import = cluster_json(temp.path(), "import"); + assert_eq!(import["ok"], true, "{import}"); + let apply = cluster_json(temp.path(), "apply"); + assert_eq!(apply["converged"], true, "{apply}"); + temp +} + +#[test] +fn optimize_resolves_a_cluster_graph_by_id() { + let temp = applied_knowledge_cluster(); + // No hand-typed storage path: address the graph by cluster dir + id. + let out = output_success( + cli() + .arg("optimize") + .arg("--cluster") + .arg(temp.path()) + .arg("--graph") + .arg("knowledge") + .arg("--json"), + ); + let payload = parse_stdout_json(&out); + assert!( + payload["tables"].as_array().is_some(), + "optimize did not run against the resolved cluster graph: {payload}" + ); +} + +#[test] +fn optimize_unknown_cluster_graph_id_errors() { + let temp = applied_knowledge_cluster(); + let out = output_failure( + cli() + .arg("optimize") + .arg("--cluster") + .arg(temp.path()) + .arg("--graph") + .arg("does-not-exist") + .arg("--json"), + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("is not applied in cluster") && stderr.contains("cluster apply"), + "expected an unapplied-graph error pointing at cluster apply; got: {stderr}" + ); +} + +#[test] +fn optimize_auto_uses_the_sole_cluster_graph() { + // RFC-011 D7: a cluster with exactly one applied graph needs no --graph — + // the resolver enumerates the catalog and uses the only candidate. + let temp = applied_knowledge_cluster(); + let out = output_success( + cli() + .arg("optimize") + .arg("--cluster") + .arg(temp.path()) + .arg("--json"), + ); + assert!( + parse_stdout_json(&out)["tables"].as_array().is_some(), + "optimize should auto-resolve the sole cluster graph" + ); +} + +/// Stand up an applied cluster with two graphs (`knowledge`, `archive`). +fn applied_two_graph_cluster() -> tempfile::TempDir { + let temp = tempdir().unwrap(); + let root = temp.path(); + fs::write( + root.join("people.pg"), + "node Person {\n name: String @key\n age: I32?\n}\n", + ) + .unwrap(); + fs::write(root.join("base.policy.yaml"), "rules: []\n").unwrap(); + fs::write( + root.join("cluster.yaml"), + r#" +version: 1 +metadata: + name: two-graph +state: + backend: cluster + lock: true +graphs: + knowledge: + schema: ./people.pg + archive: + schema: ./people.pg +policies: + base: + file: ./base.policy.yaml + applies_to: [knowledge, archive] +"#, + ) + .unwrap(); + init_named_cluster_graph(root, "knowledge", "people.pg"); + init_named_cluster_graph(root, "archive", "people.pg"); + assert_eq!(cluster_json(root, "import")["ok"], true); + assert_eq!(cluster_json(root, "apply")["converged"], true); + temp +} + +#[test] +fn optimize_on_multi_graph_cluster_without_graph_lists_candidates() { + // RFC-011 D7: >1 graph and no --graph → error naming every candidate, + // never an auto-pick. + let temp = applied_two_graph_cluster(); + let out = output_failure( + cli() + .arg("optimize") + .arg("--cluster") + .arg(temp.path()) + .arg("--json"), + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("2 graphs") + && stderr.contains("archive") + && stderr.contains("knowledge") + && stderr.contains("--graph "), + "expected a candidate-listing error; got: {stderr}" + ); +} + +#[test] +fn init_refuses_a_cluster_managed_path_and_signposts_cluster_apply() { + let temp = applied_knowledge_cluster(); + // Hand-init a NEW graph into the established cluster's storage layout. + let out = output_failure( + cli() + .arg("init") + .arg("--schema") + .arg(temp.path().join("people.pg")) + .arg(temp.path().join("graphs").join("sneaky.omni")), + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("cluster apply"), + "init into a cluster-managed path should signpost `cluster apply`; got: {stderr}" + ); + // And it did not create the graph. + assert!(!temp.path().join("graphs").join("sneaky.omni").exists()); +} + +#[test] +fn schema_apply_refuses_a_cluster_managed_graph_and_signposts_cluster_apply() { + // RFC-011 Decision 10: a direct `schema apply` against a cluster-managed + // graph's storage root would bypass the ledger/recovery/approvals, so it is + // refused and points at `cluster apply` (mirrors `init`'s refusal). + let temp = applied_knowledge_cluster(); + // A schema that WOULD change the graph (adds `bio`) — so the no-mutation + // assertion below is meaningful, not a no-op re-apply. + fs::write( + temp.path().join("people_v2.pg"), + "node Person {\n name: String @key\n age: I32?\n bio: String?\n}\n", + ) + .unwrap(); + let out = output_failure( + cli() + .arg("schema") + .arg("apply") + .arg("--schema") + .arg(temp.path().join("people_v2.pg")) + .arg("--store") + .arg(temp.path().join("graphs").join("knowledge.omni")), + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("cluster apply"), + "schema apply against a cluster-managed graph should signpost `cluster apply`; got: {stderr}" + ); + // And it bailed BEFORE mutating: the live schema still lacks `bio`. + let show = output_success( + cli() + .arg("schema") + .arg("show") + .arg(temp.path().join("graphs").join("knowledge.omni")), + ); + assert!( + !stdout_string(&show).contains("bio"), + "the refused apply must not have changed the live schema; got: {}", + stdout_string(&show) + ); +} + +#[test] +fn init_outside_a_cluster_still_works() { + // Regression guard: ordinary init (no cluster layout) is unaffected. + let temp = tempdir().unwrap(); + let schema = fixture("test.pg"); + let out = output_success( + cli() + .arg("init") + .arg("--schema") + .arg(&schema) + .arg(temp.path().join("plain.omni")), + ); + assert!(stdout_string(&out).contains("initialized")); +} + +#[test] +fn optimize_by_cluster_works_when_catalog_payloads_are_degraded() { + // Robustness (Greptile, #221): maintenance resolves the graph URI from the + // state ledger alone, so an unrelated corrupt/missing catalog payload (or a + // pending recovery sweep) does NOT block it — unlike the full serving-snapshot + // read. This is what keeps `repair --cluster` usable on a degraded cluster. + let temp = applied_knowledge_cluster(); + // Remove the verified catalog payloads (queries/policies) — a serving read + // would refuse with a catalog-payload diagnostic; the ledger-only resolve + // must not care. + let resources = temp.path().join("__cluster").join("resources"); + if resources.exists() { + fs::remove_dir_all(&resources).unwrap(); + } + let out = output_success( + cli() + .arg("optimize") + .arg("--cluster") + .arg(temp.path()) + .arg("--graph") + .arg("knowledge") + .arg("--json"), + ); + assert!( + parse_stdout_json(&out)["tables"].as_array().is_some(), + "optimize should resolve via the ledger despite degraded catalog payloads" + ); +} diff --git a/crates/omnigraph-cli/tests/cli_cluster_e2e.rs b/crates/omnigraph-cli/tests/cli_cluster_e2e.rs index 36b476a..35ded58 100644 --- a/crates/omnigraph-cli/tests/cli_cluster_e2e.rs +++ b/crates/omnigraph-cli/tests/cli_cluster_e2e.rs @@ -3,6 +3,7 @@ use std::fs; +use omnigraph::db::Omnigraph; use tempfile::tempdir; mod support; @@ -236,27 +237,28 @@ fn cluster_e2e_out_of_band_schema_drift_then_apply_converges_it() { let apply = cluster_json(temp.path(), "apply"); assert_eq!(apply["converged"], true, "{apply}"); - // Out-of-band: the live graph evolves, cluster.yaml stays put. - fs::write( - temp.path().join("people_v2.pg"), - r#" + // Out-of-band: the live graph evolves while cluster.yaml stays put. RFC-011 + // D10 makes the CLI `schema apply` refuse a cluster-managed graph, so this + // simulates a true bypass — a direct engine apply against the storage root, + // exactly the drift the control plane must still detect and converge. + let people_v2 = r#" node Person { name: String @key age: I32? bio: String? } -"#, - ) - .unwrap(); - output_success( - cli() - .arg("schema") - .arg("apply") - .arg(temp.path().join("graphs/knowledge.omni")) - .arg("--schema") - .arg(temp.path().join("people_v2.pg")) - .arg("--json"), - ); +"#; + tokio::runtime::Runtime::new().unwrap().block_on(async { + let db = Omnigraph::open( + temp.path() + .join("graphs/knowledge.omni") + .to_string_lossy() + .as_ref(), + ) + .await + .unwrap(); + db.apply_schema(people_v2).await.unwrap(); + }); // Drift is visible... let refresh = cluster_json(temp.path(), "refresh"); diff --git a/crates/omnigraph-cli/tests/cli_data.rs b/crates/omnigraph-cli/tests/cli_data.rs index 203a7c2..81e1aab 100644 --- a/crates/omnigraph-cli/tests/cli_data.rs +++ b/crates/omnigraph-cli/tests/cli_data.rs @@ -3,6 +3,7 @@ use std::fs; +use assert_cmd::Command; use serde_json::Value; use tempfile::tempdir; @@ -142,6 +143,122 @@ fn embed_seed_preserves_non_entity_rows() { assert_eq!(embedded[2]["to"], "dec-alpha"); } +#[test] +fn optimize_json_succeeds_on_local_graph() { + // Happy path for the resolve_local_uri swap (RFC-010 Slice 1): a positional + // local path still resolves and runs embedded. + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + load_fixture(&graph); + + let output = output_success(cli().arg("optimize").arg("--json").arg(&graph)); + let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!(payload["tables"].as_array().is_some()); +} + +#[test] +fn optimize_with_server_flag_errors_wrong_plane() { + // RFC-010 Slice 1: --server is a data-plane addressing flag; on a + // storage-plane verb the guard rejects it loudly (was: silently ignored). + let output = output_failure(cli().arg("optimize").arg("--server").arg("prod")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("`optimize` is a direct (storage-native) command") + && stderr.contains("--server addresses a served graph and does not apply") + && stderr.contains("Pass a storage URI, or --cluster --graph ."), + "wrong-capability guard message not found; got: {stderr}" + ); +} + +#[test] +fn wrong_address_guard_message_has_no_trailing_space() { + // The remediation tail is empty for served-addressing capabilities, so a + // misplaced --cluster on a data verb must not leave "… does not apply. " + // with a dangling space (error text is observable contract). NO_COLOR keeps + // the assertion off ANSI styling. + let output = output_failure( + cli() + .env("NO_COLOR", "1") + .arg("query") + .arg("--cluster") + .arg("./brain") + .arg("-e") + .arg("query q { Person { id } }"), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("and does not apply."), + "expected the wrong-address message; got: {stderr}" + ); + assert!( + !stderr.contains("and does not apply. "), + "trailing space after the message; got: {stderr}" + ); +} + +#[test] +fn graph_flag_on_a_positional_uri_errors() { + // RFC-011: `--graph` selects within a multi-graph scope (a server or + // cluster). An explicit `--store ` is already a single graph, so + // pairing it with `--graph` is a loud error, not a silently-dropped flag. + // (The guard lets `--graph` reach a data verb; the scope resolver rejects + // it.) + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + let output = output_failure( + cli() + .arg("query") + .arg("--store") + .arg(&graph) + .arg("--graph") + .arg("knowledge") + .arg("-e") + .arg("query q { Person { id } }"), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("already a single graph"), + "expected --graph-on-explicit-store rejection; got: {stderr}" + ); +} + +#[test] +fn query_by_name_against_a_store_needs_a_server() { + // RFC-011 D3: by-name (catalog) invocation is served-only — the catalog is + // server-owned, so a bare `--store` has nothing to resolve the name + // against. The ad-hoc lane (`-e`/`--query`) is the local alternative. + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + let output = output_failure( + cli() + .arg("query") + .arg("find_people") + .arg("--store") + .arg(&graph), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("needs a server"), + "expected a served-only by-name error; got: {stderr}" + ); +} + +#[test] +fn optimize_with_remote_target_errors_storage_plane() { + // RFC-010 Slice 1: a maintenance verb pointed at a remote URI fails loudly + // and declaratively (was: whatever Omnigraph::open said about an https URI). + let output = output_failure(cli().arg("optimize").arg("https://graph.example.invalid")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("`optimize` is a direct (storage-native) command and needs direct storage access") + && stderr.contains("remote server"), + "direct remote-target message not found; got: {stderr}" + ); +} + #[test] fn repair_json_reports_noop_on_clean_graph() { let temp = tempdir().unwrap(); @@ -412,10 +529,9 @@ query list_people() { #[test] fn deprecated_read_and_change_subcommands_emit_warnings() { - // Both subcommands require `--query`/`--query-string`/`--alias`, so - // invoking them with no args will exit non-zero. That's fine -- - // we only care that the deprecation warning is printed before the - // argument-required error. + // Both subcommands require `--query`/`--query-string`, so invoking them + // with no args will exit non-zero. That's fine -- we only care that the + // deprecation warning is printed before the argument-required error. let output = cli().arg("read").output().unwrap(); let stderr = String::from_utf8(output.stderr).unwrap(); assert!( @@ -483,13 +599,15 @@ query list_people() { } #[test] -fn query_lint_can_resolve_graph_and_query_from_config() { +fn query_lint_can_resolve_graph_from_store_scope() { + // RFC-011: lint resolves its graph target through `--store` (the direct + // scope), not omnigraph.yaml's cli.graph; the .gq path is plain cwd-relative. let temp = tempdir().unwrap(); let graph = graph_path(temp.path()); - let config_path = temp.path().join("omnigraph.yaml"); init_graph(&graph); + let query_path = temp.path().join("queries.gq"); write_query_file( - &temp.path().join("queries.gq"), + &query_path, r#" query list_people() { match { $p: Person } @@ -497,16 +615,15 @@ query list_people() { } "#, ); - write_config(&config_path, &local_yaml_config(&graph)); let output = output_success( cli() .arg("query") .arg("lint") .arg("--query") - .arg("queries.gq") - .arg("--config") - .arg(&config_path) + .arg(&query_path) + .arg("--store") + .arg(&graph) .arg("--json"), ); let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); @@ -542,8 +659,12 @@ query list_people() { .arg("http://127.0.0.1:8080"), ); let stderr = String::from_utf8_lossy(&output.stderr); + // RFC-010/011: the direct (storage-native) verbs share one declared message + // (was: "query lint is only supported against local graph URIs …"). assert!( - stderr.contains("query lint is only supported against local graph URIs in this milestone") + stderr.contains("`lint` is a direct (storage-native) command and needs direct storage access") + && stderr.contains("remote server"), + "direct remote-target message not found; got: {stderr}" ); } @@ -570,7 +691,9 @@ query list_people() { ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("query lint requires --schema or a resolvable graph target") + stderr.contains("lint requires --schema ") + || stderr.contains("no graph addressed"), + "expected a schema-or-graph-target requirement; got: {stderr}" ); } @@ -739,10 +862,10 @@ fn read_json_outputs_rows_for_named_query() { let output = output_success( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(&queries) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -756,6 +879,58 @@ fn read_json_outputs_rows_for_named_query() { assert_eq!(payload["rows"][0]["p.name"], "Alice"); } +#[test] +fn read_via_store_flag_and_profile_match_positional_uri() { + // RFC-011 Slice A: the new scope addressing (--store, and a --profile that + // binds a store) drives a read identically to the legacy positional URI — + // the scope layer is additive, not a behavior change. + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + load_fixture(&graph); + let queries = fixture("test.gq"); + + let read_rows = |cmd: &mut Command| -> Value { + let output = output_success( + cmd.arg("--query") + .arg(&queries) + .arg("get_person") + .arg("--params") + .arg(r#"{"name":"Alice"}"#) + .arg("--json"), + ); + serde_json::from_slice(&output.stdout).unwrap() + }; + + // Baseline: --store names the graph. + let baseline = read_rows(cli().arg("query").arg("--store").arg(&graph)); + assert_eq!(baseline["rows"][0]["p.name"], "Alice"); + + // --store names the same graph directly. + let via_store = read_rows(cli().arg("query").arg("--store").arg(&graph)); + assert_eq!(via_store["rows"], baseline["rows"]); + + // A profile binding that store, selected with --profile (no positional). + let home = temp.path().join("op-home"); + std::fs::create_dir_all(&home).unwrap(); + std::fs::write( + home.join("config.yaml"), + format!( + "profiles:\n local:\n store: '{}'\n", + graph.to_string_lossy() + ), + ) + .unwrap(); + let via_profile = read_rows( + cli() + .env("OMNIGRAPH_HOME", &home) + .arg("query") + .arg("--profile") + .arg("local"), + ); + assert_eq!(via_profile["rows"], baseline["rows"]); +} + #[test] fn export_jsonl_outputs_source_rows_for_selected_branch_and_type() { let temp = tempdir().unwrap(); @@ -815,43 +990,38 @@ fn export_jsonl_outputs_source_rows_for_selected_branch_and_type() { ); } +// RFC-011: `policy validate|test|explain` source the Cedar bundle from a +// converged cluster's applied policies (`--cluster ` + `--graph `), +// not omnigraph.yaml's policy.file. + #[test] -fn policy_validate_accepts_valid_policy_file() { - let temp = tempdir().unwrap(); - let (config, _) = write_policy_config_fixture(temp.path()); +fn policy_validate_accepts_cluster_bundle() { + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_YAML)); let output = output_success( cli() .arg("policy") .arg("validate") - .arg("--config") - .arg(&config), + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge"), ); let stdout = stdout_string(&output); assert!(stdout.contains("policy valid:")); - assert!(stdout.contains("policy.yaml")); assert!(stdout.contains("[2 actors]")); } #[test] -fn policy_validate_fails_for_invalid_policy_file() { - let temp = tempdir().unwrap(); - let config = temp.path().join("omnigraph.yaml"); - let policy = temp.path().join("policy.yaml"); - fs::write( - &config, - r#" -project: - name: policy-test-graph -policy: - file: ./policy.yaml -"#, - ) - .unwrap(); - fs::write( - &policy, - r#" +fn policy_validate_fails_for_invalid_cluster_bundle() { + // The cluster does not validate a policy bundle's internal rules, so an + // applied-but-malformed bundle reaches `policy validate`, which compiles it + // and surfaces the error (here: a duplicate rule id). + let cluster = converged_loaded_cluster( + "knowledge", + Some( + r#" version: 1 groups: team: [act-andrew] @@ -867,26 +1037,42 @@ rules: actions: [export] branch_scope: any "#, - ) - .unwrap(); + ), + ); let output = output_failure( cli() .arg("policy") .arg("validate") - .arg("--config") - .arg(&config), + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge"), ); let stderr = String::from_utf8(output.stderr).unwrap(); - assert!(stderr.contains("duplicate policy rule id")); + assert!( + stderr.contains("duplicate policy rule id"), + "expected a duplicate-rule error; got: {stderr}" + ); } #[test] -fn policy_test_runs_declarative_cases() { - let temp = tempdir().unwrap(); - let (config, _) = write_policy_config_fixture(temp.path()); +fn policy_test_runs_declarative_cases_against_cluster_bundle() { + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_YAML)); + let tests = cluster.path().join("policy.tests.yaml"); + fs::write(&tests, POLICY_TESTS_YAML).unwrap(); - let output = output_success(cli().arg("policy").arg("test").arg("--config").arg(&config)); + let output = output_success( + cli() + .arg("policy") + .arg("test") + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge") + .arg("--tests") + .arg(&tests), + ); let stdout = stdout_string(&output); assert!(stdout.contains("policy tests passed: 2 cases")); @@ -894,15 +1080,16 @@ fn policy_test_runs_declarative_cases() { #[test] fn policy_explain_reports_decision_and_matched_rule() { - let temp = tempdir().unwrap(); - let (config, _) = write_policy_config_fixture(temp.path()); + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_YAML)); let allow = output_success( cli() .arg("policy") .arg("explain") - .arg("--config") - .arg(&config) + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge") .arg("--actor") .arg("act-andrew") .arg("--action") @@ -918,8 +1105,10 @@ fn policy_explain_reports_decision_and_matched_rule() { cli() .arg("policy") .arg("explain") - .arg("--config") - .arg(&config) + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge") .arg("--actor") .arg("act-bruno") .arg("--action") @@ -933,22 +1122,26 @@ fn policy_explain_reports_decision_and_matched_rule() { } #[test] -fn read_can_resolve_uri_from_config() { +fn read_resolves_uri_from_default_store_scope() { + // RFC-011: a zero-flag read resolves its graph from `defaults.store` in the + // operator config (the local-dev default scope) — no omnigraph.yaml. let temp = tempdir().unwrap(); let graph = graph_path(temp.path()); - let config = temp.path().join("omnigraph.yaml"); init_graph(&graph); load_fixture(&graph); - write_config(&config, &local_yaml_config(&graph)); + let home = tempdir().unwrap(); + std::fs::write( + home.path().join("config.yaml"), + format!("defaults:\n store: {}\n", graph.to_string_lossy()), + ) + .unwrap(); let output = output_success( cli() + .env("OMNIGRAPH_HOME", home.path()) .arg("read") - .arg("--config") - .arg(&config) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -968,10 +1161,10 @@ fn read_csv_format_outputs_header_and_row_values() { let output = output_success( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -1005,10 +1198,10 @@ fn read_uses_operator_default_output_format() { command .env("OMNIGRAPH_HOME", operator_home.path()) .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#); @@ -1040,10 +1233,10 @@ fn read_jsonl_format_outputs_metadata_header_first() { let output = output_success( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -1075,6 +1268,7 @@ query insert_person($name: String, $age: I32) { let output = output_success( cli() .arg("change") + .arg("--store") .arg(&graph) .arg("--query") .arg(&mutation_file) @@ -1091,10 +1285,10 @@ query insert_person($name: String, $age: I32) { let verify = output_success( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Eve"}"#) @@ -1106,13 +1300,13 @@ query insert_person($name: String, $age: I32) { } #[test] -fn change_can_resolve_uri_and_branch_from_config() { +fn change_resolves_uri_and_default_branch_from_store_scope() { + // RFC-011: a mutate resolves its graph from `--store` and defaults the + // branch to main (no omnigraph.yaml cli.graph / cli.branch). let temp = tempdir().unwrap(); let graph = graph_path(temp.path()); - let config = temp.path().join("omnigraph.yaml"); init_graph(&graph); load_fixture(&graph); - write_config(&config, &local_yaml_config(&graph)); let mutation_file = temp.path().join("config-mutations.gq"); write_query_file( &mutation_file, @@ -1126,8 +1320,8 @@ query insert_person($name: String, $age: I32) { let output = output_success( cli() .arg("change") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(&graph) .arg("--query") .arg(&mutation_file) .arg("--params") @@ -1149,6 +1343,7 @@ fn read_requires_name_for_multi_query_files() { let output = output_failure( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(fixture("test.gq")), @@ -1167,6 +1362,7 @@ fn read_supports_inline_query_string() { let output = output_success( cli() .arg("read") + .arg("--store") .arg(&repo) .arg("-e") .arg("query find($name: String) { match { $p: Person { name: $name } } return { $p.name, $p.age } }") @@ -1180,6 +1376,49 @@ fn read_supports_inline_query_string() { assert_eq!(payload["rows"][0]["p.name"], "Alice"); } +#[test] +fn positional_http_uri_on_a_data_verb_is_rejected() { + // RFC-011: a `--store` http(s):// URL no longer dispatches to a remote + // server — that requires `--server `. + let output = output_failure( + cli() + .arg("query") + .arg("--store") + .arg("http://127.0.0.1:1") + .arg("-e") + .arg("query q() { match { $p: Person { } } return { $p } }"), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("must be addressed with `--server `"), + "expected store-remote rejection; got: {stderr}" + ); +} + +#[test] +fn as_on_a_served_write_is_rejected() { + // RFC-011: a served write resolves the actor from the bearer token, so --as + // cannot set identity. It errors while building the remote client — before + // any HTTP call, so no server is needed. + let output = output_failure( + cli() + .arg("mutate") + .arg("--server") + .arg("http://127.0.0.1:1") + .arg("--as") + .arg("act-nope") + .arg("-e") + .arg("query add($name: String) { insert Person { name: $name } }") + .arg("--params") + .arg(r#"{"name":"X"}"#), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("`--as` is not allowed on a served write"), + "expected --as-served rejection; got: {stderr}" + ); +} + #[test] fn change_supports_inline_query_string() { let temp = tempdir().unwrap(); @@ -1190,6 +1429,7 @@ fn change_supports_inline_query_string() { let output = output_success( cli() .arg("change") + .arg("--store") .arg(&repo) .arg("--query-string") .arg("query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }") @@ -1204,6 +1444,7 @@ fn change_supports_inline_query_string() { let verify = output_success( cli() .arg("read") + .arg("--store") .arg(&repo) .arg("-e") .arg("query find($name: String) { match { $p: Person { name: $name } } return { $p.name } }") @@ -1225,6 +1466,7 @@ fn read_rejects_query_string_combined_with_query() { let output = output_failure( cli() .arg("read") + .arg("--store") .arg(&repo) .arg("--query") .arg(fixture("test.gq")) @@ -1245,7 +1487,7 @@ fn read_rejects_empty_query_string() { init_graph(&repo); load_fixture(&repo); - let output = output_failure(cli().arg("read").arg(&repo).arg("-e").arg("")); + let output = output_failure(cli().arg("read").arg("--store").arg(&repo).arg("-e").arg("")); let stderr = String::from_utf8(output.stderr).unwrap(); assert!( stderr.contains("must not be empty"), @@ -1373,6 +1615,160 @@ fn branch_delete_rejects_main() { assert!(stderr.contains("cannot delete branch 'main'")); } +// ── RFC-011 Decision 9: write diagnostics + non-local destructive-confirm ── + +#[test] +fn write_echoes_resolved_target_to_stderr() { + // Every write echoes its resolved target + access path to stderr; --json + // (stdout) is unaffected. A local load → "(direct, local)". + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + let data = fixture("test.jsonl"); + let output = output_success( + cli() + .arg("load") + .arg("--mode") + .arg("append") + .arg("--data") + .arg(&data) + .arg(&graph) + .arg("--json"), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains("omnigraph load →") && stderr.contains("(direct, local)"), + "missing write-target echo; stderr: {stderr}" + ); + // stdout still parses as JSON — the echo went to stderr. + let _: Value = serde_json::from_slice(&output.stdout).unwrap(); +} + +#[test] +fn quiet_suppresses_the_write_target_echo() { + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + let data = fixture("test.jsonl"); + let output = output_success( + cli() + .arg("--quiet") + .arg("load") + .arg("--mode") + .arg("append") + .arg("--data") + .arg(&data) + .arg(&graph), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + !stderr.contains("omnigraph load →"), + "--quiet should suppress the echo; stderr: {stderr}" + ); +} + +#[test] +fn branch_delete_against_non_local_scope_refuses_without_yes() { + // No bucket needed: the confirm gate fires before the graph is opened. + let output = output_failure( + cli() + .arg("branch") + .arg("delete") + .arg("--store") + .arg("s3://fake-bucket/g.omni") + .arg("feature") + .arg("--json"), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains("refusing destructive `branch delete`") && stderr.contains("--yes"), + "expected a non-local destructive refusal; stderr: {stderr}" + ); +} + +#[test] +fn branch_delete_against_non_local_scope_passes_gate_with_yes() { + // With --yes the gate is bypassed; the command then fails for an unrelated + // reason (the fake bucket can't be opened), so the refusal must be ABSENT. + let output = output_failure( + cli() + .arg("branch") + .arg("delete") + .arg("--store") + .arg("s3://fake-bucket/g.omni") + .arg("feature") + .arg("--yes") + .arg("--json"), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + !stderr.contains("refusing destructive"), + "--yes should bypass the confirm gate; stderr: {stderr}" + ); +} + +#[test] +fn overwrite_load_against_non_local_scope_refuses_without_yes() { + let output = output_failure( + cli() + .arg("load") + .arg("--mode") + .arg("overwrite") + .arg("--data") + .arg(fixture("test.jsonl")) + .arg("--store") + .arg("s3://fake-bucket/g.omni") + .arg("--json"), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains("refusing destructive `load --mode overwrite`"), + "expected a non-local overwrite refusal; stderr: {stderr}" + ); +} + +#[test] +fn cleanup_against_non_local_scope_refuses_without_yes() { + // Past the --confirm preview gate, a non-local cleanup still needs --yes. + let output = output_failure( + cli() + .arg("cleanup") + .arg("--store") + .arg("s3://fake-bucket/g.omni") + .arg("--keep") + .arg("5") + .arg("--confirm") + .arg("--json"), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains("refusing destructive `cleanup`"), + "expected a non-local cleanup refusal; stderr: {stderr}" + ); +} + +#[test] +fn cleanup_against_local_scope_executes_with_confirm() { + // Local cleanup needs no --yes; --confirm alone executes (and echoes). + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_graph(&graph); + load_fixture(&graph); + let output = output_success( + cli() + .arg("cleanup") + .arg("--keep") + .arg("1") + .arg("--confirm") + .arg(&graph) + .arg("--json"), + ); + let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!(payload["tables"].as_array().is_some(), "{payload}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("omnigraph cleanup →"), "stderr: {stderr}"); +} + #[test] fn branch_merge_defaults_target_to_main() { let temp = tempdir().unwrap(); @@ -1522,19 +1918,17 @@ fn snapshot_json_returns_manifest_version_and_tables() { } #[test] -fn snapshot_can_resolve_uri_from_config() { +fn snapshot_resolves_uri_from_store_scope() { let temp = tempdir().unwrap(); let graph = graph_path(temp.path()); - let config = temp.path().join("omnigraph.yaml"); init_graph(&graph); load_fixture(&graph); - write_config(&config, &local_yaml_config(&graph)); let output = output_success( cli() .arg("snapshot") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(&graph) .arg("--json"), ); let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); @@ -1675,3 +2069,162 @@ fn cli_fails_for_invalid_merge_requests() { .contains("distinct source and target") ); } + +/// RFC-011 Decision 8: `profile list` / `profile show` inspect the operator +/// config's profiles read-only. Hermetic via OMNIGRAPH_HOME. +fn profile_home() -> tempfile::TempDir { + let home = tempdir().unwrap(); + std::fs::write( + home.path().join("config.yaml"), + "operator:\n actor: act-andrew\n\ + defaults:\n output: json\n server: prod\n default_graph: knowledge\n\ + servers:\n prod:\n url: https://graph.example.com\n\ + clusters:\n brain:\n root: s3://acme/clusters/brain\n\ + profiles:\n\ + \x20 staging:\n server: prod\n default_graph: kb\n\ + \x20 brain-admin:\n cluster: brain\n\ + \x20 localdev:\n store: file:///data/dev.omni\n\ + \x20 broken:\n server: a\n store: b\n", + ) + .unwrap(); + home +} + +#[test] +fn profile_list_names_each_profile_with_its_binding_and_marks_active() { + let home = profile_home(); + let out = output_success( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .env("OMNIGRAPH_PROFILE", "staging") + .arg("profile") + .arg("list"), + ); + let stdout = stdout_string(&out); + assert!(stdout.contains("staging (active)"), "{stdout}"); + assert!(stdout.contains("server: prod"), "{stdout}"); + assert!(stdout.contains("cluster: brain"), "{stdout}"); + assert!(stdout.contains("store: file:///data/dev.omni"), "{stdout}"); + // A malformed (two-scope) profile is reported, not a hard failure. + assert!(stdout.contains("broken") && stdout.contains("invalid:"), "{stdout}"); +} + +#[test] +fn profile_list_json_shape() { + let home = profile_home(); + let out = output_success( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .arg("profile") + .arg("list") + .arg("--json"), + ); + let items: Value = serde_json::from_slice(&out.stdout).unwrap(); + let brain = items + .as_array() + .unwrap() + .iter() + .find(|p| p["name"] == "brain-admin") + .unwrap(); + assert_eq!(brain["binding"], "cluster: brain"); + assert_eq!(brain["scope_kind"], "cluster"); + assert_eq!(brain["target"], "brain"); + assert_eq!(brain["valid"], true); + assert!(brain["error"].is_null()); + assert_eq!(brain["active"], false); + let broken = items + .as_array() + .unwrap() + .iter() + .find(|p| p["name"] == "broken") + .unwrap(); + assert_eq!(broken["scope_kind"], "invalid"); + assert_eq!(broken["valid"], false); + assert!(broken["target"].is_null()); + assert!( + broken["error"] + .as_str() + .unwrap() + .contains("profile 'broken'") + ); +} + +#[test] +fn profile_show_resolves_named_scope_endpoints() { + let home = profile_home(); + // A cluster profile resolves its root. + let cluster = output_success( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .arg("profile") + .arg("show") + .arg("brain-admin"), + ); + let cs = stdout_string(&cluster); + assert!(cs.contains("scope: cluster brain"), "{cs}"); + assert!(cs.contains("endpoint: s3://acme/clusters/brain"), "{cs}"); + + // A store profile shows its URI as the endpoint. + let store = output_success( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .arg("profile") + .arg("show") + .arg("localdev") + .arg("--json"), + ); + let detail: Value = serde_json::from_slice(&store.stdout).unwrap(); + assert_eq!(detail["scope_kind"], "store"); + assert_eq!(detail["endpoint"], "file:///data/dev.omni"); +} + +#[test] +fn profile_show_without_name_falls_back_to_flat_defaults() { + let home = profile_home(); + let out = output_success( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .arg("profile") + .arg("show") + .arg("--json"), + ); + let detail: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(detail["name"], "(defaults)"); + assert_eq!(detail["scope_kind"], "server"); + assert_eq!(detail["endpoint"], "https://graph.example.com"); + assert_eq!(detail["default_graph"], "knowledge"); +} + +#[test] +fn profile_show_without_name_uses_active_env_profile() { + let home = profile_home(); + let out = output_success( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .env("OMNIGRAPH_PROFILE", "brain-admin") + .arg("profile") + .arg("show") + .arg("--json"), + ); + let detail: Value = serde_json::from_slice(&out.stdout).unwrap(); + // No name arg, but $OMNIGRAPH_PROFILE selects brain-admin (not the flat defaults). + assert_eq!(detail["name"], "brain-admin"); + assert_eq!(detail["scope_kind"], "cluster"); + assert_eq!(detail["endpoint"], "s3://acme/clusters/brain"); + // output_format renders as the canonical lowercase value name. + assert_eq!(detail["output_format"], "json"); +} + +#[test] +fn profile_show_unknown_name_errors() { + let home = profile_home(); + let out = output_failure( + cli() + .env("OMNIGRAPH_HOME", home.path()) + .arg("profile") + .arg("show") + .arg("nope"), + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("unknown profile 'nope'"), "{stderr}"); +} diff --git a/crates/omnigraph-cli/tests/cli_queries.rs b/crates/omnigraph-cli/tests/cli_queries.rs index 8a1e553..92f7879 100644 --- a/crates/omnigraph-cli/tests/cli_queries.rs +++ b/crates/omnigraph-cli/tests/cli_queries.rs @@ -2,7 +2,6 @@ //! Moved verbatim from tests/cli.rs in the modularization. -use serde_json::Value; use tempfile::tempdir; mod support; @@ -57,227 +56,172 @@ query list_people() { assert_eq!(stdout_string(&lint_output), stdout_string(&check_output)); } +// Legacy `omnigraph.yaml` `aliases:` invoked via the `--alias` flag were +// removed in RFC-011 D4 — operator aliases now live under `omnigraph alias +// ` (the happy path is covered by system_local's operator-alias e2e). +// The legacy file-alias path has no CLI entry point. + #[test] -fn read_alias_from_yaml_config_runs_with_kv_output() { - let temp = tempdir().unwrap(); - let graph = graph_path(temp.path()); - let config = temp.path().join("omnigraph.yaml"); - let query = temp.path().join("aliases.gq"); - init_graph(&graph); - load_fixture(&graph); - write_query_file( - &query, - &std::fs::read_to_string(fixture("test.gq")).unwrap(), +fn alias_flag_is_removed_from_query() { + // RFC-011 D4: `--alias` no longer exists on query/mutate; use `alias `. + let output = output_failure(cli().arg("query").arg("--alias").arg("who")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("unexpected argument") && stderr.contains("--alias"), + "expected clap to reject --alias on query; got: {stderr}" ); - write_config( - &config, - &format!( - "{}aliases:\n owner:\n command: read\n query: aliases.gq\n name: get_person\n args: [name]\n format: kv\n", - local_yaml_config(&graph) - ), - ); - - let output = output_success( - cli() - .arg("read") - .arg("--config") - .arg(&config) - .arg("--alias") - .arg("owner") - .arg("Alice"), - ); - let stdout = stdout_string(&output); - - assert!(stdout.contains("row 1")); - assert!(stdout.contains("p.name: Alice")); } #[test] -fn read_alias_uses_alias_target_without_cli_default_and_accepts_url_like_arg() { - let temp = tempdir().unwrap(); - let graph = graph_path(temp.path()); - let config = temp.path().join("omnigraph.yaml"); - let query = temp.path().join("aliases.gq"); - let data = temp.path().join("url-like.jsonl"); - init_graph(&graph); - write_jsonl( - &data, - r#"{"type":"Person","data":{"name":"https://example.com","age":30}}"#, - ); - output_success( +fn alias_unknown_name_errors_listing_defined() { + // Hermetic: an unknown alias fails before any network, listing defined ones. + let home = tempdir().unwrap(); + std::fs::write( + home.path().join("config.yaml"), + "servers:\n dev:\n url: https://x\naliases:\n who:\n server: dev\n query: find_person\n", + ) + .unwrap(); + let output = output_failure( cli() - .arg("load") - .arg("--mode") - .arg("overwrite") - .arg("--data") - .arg(&data) - .arg(&graph), + .env("OMNIGRAPH_HOME", home.path()) + .arg("alias") + .arg("nope"), ); - write_query_file( - &query, - &std::fs::read_to_string(fixture("test.gq")).unwrap(), + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("unknown alias 'nope'") && stderr.contains("who"), + "expected an unknown-alias error listing defined aliases; got: {stderr}" ); - write_config( - &config, - &format!( - "graphs:\n local:\n uri: '{}'\nquery:\n roots:\n - .\npolicy: {{}}\naliases:\n owner:\n command: read\n query: aliases.gq\n name: get_person\n args: [name]\n graph: local\n format: kv\n", - graph.to_string_lossy() - ), - ); - - let output = output_success( - cli() - .arg("read") - .arg("--config") - .arg(&config) - .arg("--alias") - .arg("owner") - .arg("https://example.com"), - ); - let stdout = stdout_string(&output); - - assert!(stdout.contains("row 1")); - assert!(stdout.contains("p.name: https://example.com")); } #[test] -fn change_alias_from_yaml_config_persists_changes() { - let temp = tempdir().unwrap(); - let graph = graph_path(temp.path()); - let config = temp.path().join("omnigraph.yaml"); - let query = temp.path().join("mutations.gq"); - init_graph(&graph); - load_fixture(&graph); - write_query_file( - &query, - r#" -query insert_person($name: String, $age: I32) { - insert Person { name: $name, age: $age } +fn alias_rejects_global_scope_flags_that_the_binding_owns() { + for (flag, value) in [ + ("--server", "dev"), + ("--graph", "local"), + ("--store", "file:///tmp/graph.omni"), + ("--cluster", "."), + ("--profile", "prod"), + ("--as", "act-op"), + ] { + let output = output_failure(cli().arg(flag).arg(value).arg("alias").arg("who")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("`alias` uses the server, graph, and stored query") + && stderr.contains(flag), + "expected {flag} to be rejected by the alias binding guard; got: {stderr}" + ); + } } -"#, + +#[test] +fn queries_and_policy_wrong_server_scope_points_at_cluster_scope() { + let output = output_failure(cli().arg("--server").arg("prod").arg("queries").arg("list")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("pass --cluster ") && !stderr.contains("pass --config "), + "queries should point at --cluster, not --config; got: {stderr}" ); - write_config( - &config, - &format!( - "{}aliases:\n add_person:\n command: change\n query: mutations.gq\n name: insert_person\n args: [name, age]\n", - local_yaml_config(&graph) + + let output = output_failure( + cli() + .arg("--server") + .arg("prod") + .arg("policy") + .arg("validate"), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("pass --cluster ") && !stderr.contains("pass --config "), + "policy should point at --cluster, not --config; got: {stderr}" + ); +} + +// RFC-011: `queries validate`/`list` source the registry + schemas from a +// converged cluster's applied state (`--cluster `), not omnigraph.yaml. + +/// Build a converged single-graph cluster (id `knowledge`) with one stored +/// query. `query_block` is the YAML under the graph's `queries:` key. +fn converged_cluster_with_query(query_file: &str, query_src: &str, query_block: &str) -> tempfile::TempDir { + let temp = tempdir().unwrap(); + let dir = temp.path(); + std::fs::copy(fixture("test.pg"), dir.join("graph.pg")).unwrap(); + write_query_file(&dir.join(query_file), query_src); + std::fs::write( + dir.join("cluster.yaml"), + format!( + "version: 1\nmetadata:\n name: sys\nstate:\n backend: cluster\n lock: true\n\ + graphs:\n knowledge:\n schema: ./graph.pg\n queries:\n{query_block}" ), - ); - - let output = output_success( - cli() - .arg("change") - .arg("--config") - .arg(&config) - .arg("--alias") - .arg("add_person") - .arg("Eve") - .arg("29") - .arg("--json"), - ); - let payload: Value = serde_json::from_slice(&output.stdout).unwrap(); - assert_eq!(payload["affected_nodes"], 1); - - let verify = output_success( - cli() - .arg("read") - .arg(&graph) - .arg("--query") - .arg(fixture("test.gq")) - .arg("--name") - .arg("get_person") - .arg("--params") - .arg(r#"{"name":"Eve"}"#) - .arg("--json"), - ); - let verify_payload: Value = serde_json::from_slice(&verify.stdout).unwrap(); - assert_eq!(verify_payload["row_count"], 1); + ) + .unwrap(); + output_success(cli().arg("cluster").arg("import").arg("--config").arg(dir)); + output_success(cli().arg("cluster").arg("apply").arg("--config").arg(dir)); + temp } #[test] fn queries_validate_exits_zero_on_clean_registry() { - let graph = SystemGraph::loaded(); - graph.write_query( + let cluster = converged_cluster_with_query( "find_person.gq", "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - &queries_test_config( - &graph.path().to_string_lossy(), - "find_person", - "find_person.gq", - ), + " find_person:\n file: ./find_person.gq\n", ); let output = output_success( cli() .arg("queries") .arg("validate") - .arg("--config") - .arg(&config), + .arg("--cluster") + .arg(cluster.path()), ); let stdout = stdout_string(&output); assert!(stdout.contains("OK"), "stdout:\n{stdout}"); } #[test] -fn queries_validate_exits_nonzero_on_type_broken_query() { - let graph = SystemGraph::loaded(); - // `Widget` is not in the fixture schema. - graph.write_query( - "ghost.gq", +fn cluster_import_rejects_a_type_broken_query() { + // In the cluster model a stored query is type-checked at the cluster + // boundary (import/apply), so a broken query can never reach the applied + // state `queries validate` reads — the gate is upstream. `Widget` is not in + // the fixture schema, so import must reject it, naming the query. + let temp = tempdir().unwrap(); + let dir = temp.path(); + std::fs::copy(fixture("test.pg"), dir.join("graph.pg")).unwrap(); + write_query_file( + &dir.join("ghost.gq"), "query ghost() { match { $w: Widget } return { $w.name } }", ); - let config = graph.write_config( - "omnigraph.yaml", - &queries_test_config(&graph.path().to_string_lossy(), "ghost", "ghost.gq"), + std::fs::write( + dir.join("cluster.yaml"), + "version: 1\nmetadata:\n name: sys\nstate:\n backend: cluster\n lock: true\n\ + graphs:\n knowledge:\n schema: ./graph.pg\n queries:\n ghost:\n file: ./ghost.gq\n", + ) + .unwrap(); + let output = output_failure(cli().arg("cluster").arg("import").arg("--config").arg(dir)); + let combined = format!( + "{}{}", + stdout_string(&output), + String::from_utf8_lossy(&output.stderr) ); - let output = output_failure( - cli() - .arg("queries") - .arg("validate") - .arg("--config") - .arg(&config), - ); - let stdout = stdout_string(&output); assert!( - stdout.contains("ghost"), - "validation should name the broken query; stdout:\n{stdout}" + combined.contains("ghost"), + "cluster import must reject the broken query, naming it; got:\n{combined}" ); } #[test] fn queries_list_prints_registered_query() { - let graph = SystemGraph::loaded(); - graph.write_query( + let cluster = converged_cluster_with_query( "find_person.gq", "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - // Exposed with an explicit tool name so the list shows the MCP suffix. - let config = graph.write_config( - "omnigraph.yaml", - &format!( - concat!( - "graphs:\n", - " local:\n", - " uri: '{}'\n", - " queries:\n", - " find_person:\n", - " file: ./find_person.gq\n", - " mcp: {{ expose: true, tool_name: lookup_person }}\n", - "cli:\n", - " graph: local\n", - "policy: {{}}\n", - ), - graph.path().to_string_lossy().replace('\'', "''") - ), + " find_person:\n file: ./find_person.gq\n", ); let output = output_success( cli() .arg("queries") .arg("list") - .arg("--config") - .arg(&config), + .arg("--cluster") + .arg(cluster.path()), ); let stdout = stdout_string(&output); assert!(stdout.contains("find_person"), "stdout:\n{stdout}"); @@ -285,251 +229,37 @@ fn queries_list_prints_registered_query() { stdout.contains("$name: String"), "list should show typed params; stdout:\n{stdout}" ); - assert!( - stdout.contains("[mcp: lookup_person]"), - "list should show the MCP tool name for exposed queries; stdout:\n{stdout}" - ); } #[test] -fn queries_list_requires_graph_selection_for_per_graph_only_registries() { - let graph = SystemGraph::loaded(); - graph.write_query( - "find_person.gq", - "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - &format!( - concat!( - "graphs:\n", - " local:\n", - " uri: '{}'\n", - " queries:\n", - " find_person:\n", - " file: ./find_person.gq\n", - "policy: {{}}\n", - ), - graph.path().to_string_lossy().replace('\'', "''") - ), - ); - - let output = output_failure( - cli() - .arg("queries") - .arg("list") - .arg("--config") - .arg(&config), - ); +fn queries_validate_requires_a_cluster() { + // RFC-011: with no --cluster (and no cluster profile), the command errors + // loudly rather than reading any omnigraph.yaml. + let output = output_failure(cli().arg("queries").arg("validate")); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("local") && stderr.contains("--target local"), - "error must name the graph and give a concrete selection hint; stderr:\n{stderr}" + stderr.contains("needs a cluster") || stderr.contains("--cluster"), + "queries validate must require a cluster; stderr:\n{stderr}" ); } #[test] -fn queries_list_without_graph_selection_lists_top_level_registry() { - let graph = SystemGraph::loaded(); - graph.write_query( - "top_find.gq", - "query top_find($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - concat!( - "queries:\n", - " top_find:\n", - " file: ./top_find.gq\n", - "policy: {}\n", - ), - ); - - let output = output_success( - cli() - .arg("queries") - .arg("list") - .arg("--config") - .arg(&config), - ); - let stdout = stdout_string(&output); - assert!(stdout.contains("top_find"), "stdout:\n{stdout}"); -} - -#[test] -fn queries_list_unknown_target_errors() { - // `queries list` opens no graph URI, so unknown-graph validation can't ride - // along on URI resolution the way it does for every other command. An - // unknown `--target` must still error (naming the graph) instead of - // silently falling back to the top-level registry and showing the wrong - // (or empty) catalog. - let graph = SystemGraph::loaded(); - graph.write_query( - "find_person.gq", - "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - &queries_test_config( - &graph.path().to_string_lossy(), - "find_person", - "find_person.gq", - ), - ); - let output = output_failure( - cli() - .arg("queries") - .arg("list") - .arg("--target") - .arg("nonexistent") - .arg("--config") - .arg(&config), - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("nonexistent"), - "error must name the unknown graph; stderr:\n{stderr}" - ); -} - -#[test] -fn queries_commands_reject_named_graph_with_populated_top_level_block() { - // A named graph (here via `cli.graph`) uses its own `graphs.` block, - // so a populated top-level `queries:` block would be silently ignored — a - // config the server REFUSES to boot. `queries validate`/`list` must reject - // it too (matching boot) instead of validating/listing the per-graph block - // and giving a false green. - let graph = SystemGraph::loaded(); - graph.write_query( - "find_person.gq", - "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - &format!( - concat!( - "graphs:\n", - " local:\n", - " uri: '{}'\n", - " queries:\n", - " find_person:\n", - " file: ./find_person.gq\n", - "cli:\n", - " graph: local\n", - "queries:\n", // populated top-level block: the coherence violation - " legacy:\n", - " file: ./legacy.gq\n", - "policy: {{}}\n", - ), - graph.path().to_string_lossy().replace('\'', "''") - ), - ); - // Both resolve `local` from cli.graph (no positional URI), so both must - // error and name the graph + the ignored block — like server boot does. - for sub in ["validate", "list"] { - let output = output_failure(cli().arg("queries").arg(sub).arg("--config").arg(&config)); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("local") && stderr.contains("queries"), - "`queries {sub}` must reject a named graph with a populated top-level block; stderr:\n{stderr}" - ); - } -} - -#[test] -fn queries_validate_exits_nonzero_on_duplicate_tool_name() { - // Two exposed queries claiming one MCP tool name is a load-time - // collision — `queries validate` must fail (offline, before the engine - // opens) and name both queries plus the contested tool. - let graph = SystemGraph::loaded(); - graph.write_query( - "a.gq", - "query a() { match { $p: Person } return { $p.name } }", - ); - graph.write_query( - "b.gq", - "query b() { match { $p: Person } return { $p.name } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - &format!( - concat!( - "graphs:\n", - " local:\n", - " uri: '{}'\n", - " queries:\n", - " a:\n", - " file: ./a.gq\n", - " mcp: {{ expose: true, tool_name: dup }}\n", - " b:\n", - " file: ./b.gq\n", - " mcp: {{ expose: true, tool_name: dup }}\n", - "cli:\n", - " graph: local\n", - "policy: {{}}\n", - ), - graph.path().to_string_lossy().replace('\'', "''") - ), - ); - let output = output_failure( - cli() - .arg("queries") - .arg("validate") - .arg("--config") - .arg(&config), - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("dup") && stderr.contains("'a'") && stderr.contains("'b'"), - "duplicate tool name should be reported naming both queries; stderr:\n{stderr}" - ); -} - -#[test] -fn queries_validate_positional_uri_ignores_default_graph() { - // A positional URI is anonymous → the schema AND the registry both come - // from top-level, even when `cli.graph` names a graph whose per-graph - // queries would fail. Pins that the URI and registry can't diverge. - let graph = SystemGraph::loaded(); - graph.write_query( - "clean.gq", - "query clean($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - // `Widget` is not in the fixture schema — the default graph's per-graph - // query would break validate if it were (wrongly) selected. - graph.write_query( - "broken.gq", - "query broken() { match { $w: Widget } return { $w.name } }", - ); - let config = graph.write_config( - "omnigraph.yaml", - concat!( - "cli:\n graph: prod\n", - "graphs:\n", - " prod:\n", - " uri: /nonexistent-prod.omni\n", - " queries:\n", - " broken:\n", - " file: ./broken.gq\n", - "queries:\n", - " clean:\n", - " file: ./clean.gq\n", - "policy: {}\n", - ), - ); - // Positional URI = the real loaded graph; selection is anonymous, so the - // CLEAN top-level registry validates (not prod's broken one). +fn queries_validate_graph_filter_selects_one_graph() { + // A multi-graph cluster: validate scoped to `knowledge` type-checks only + // that graph's registry, ignoring `engineering`'s. + let temp = tempdir().unwrap(); + let dir = temp.path(); + write_multi_graph_cluster_fixture(dir); + output_success(cli().arg("cluster").arg("import").arg("--config").arg(dir)); + output_success(cli().arg("cluster").arg("apply").arg("--config").arg(dir)); let output = output_success( cli() .arg("queries") .arg("validate") - .arg(graph.path()) - .arg("--config") - .arg(&config), - ); - let stdout = stdout_string(&output); - assert!( - stdout.contains("OK"), - "positional URI must validate the top-level registry, not the cli.graph default; stdout:\n{stdout}" + .arg("--cluster") + .arg(dir) + .arg("--graph") + .arg("knowledge"), ); + assert!(stdout_string(&output).contains("OK")); } diff --git a/crates/omnigraph-cli/tests/cli_schema_config.rs b/crates/omnigraph-cli/tests/cli_schema_config.rs index 710c856..5577aa8 100644 --- a/crates/omnigraph-cli/tests/cli_schema_config.rs +++ b/crates/omnigraph-cli/tests/cli_schema_config.rs @@ -24,6 +24,38 @@ fn version_command_prints_current_cli_version() { ); } +#[test] +fn help_groups_commands_by_capability() { + // RFC-010 Slice 2 / RFC-011 Slice B: `--help` clusters commands (declaration + // order in the Command enum) and explains the capability each needs in an + // after_help legend. Pinned lightly — the legend phrase + the cluster + // ordering — to avoid brittle full-text assertions on clap's help body. + let output = output_success(cli().arg("--help")); + let stdout = stdout_string(&output); + + assert!( + stdout.contains("COMMANDS BY CAPABILITY"), + "capability legend (after_help) missing from --help:\n{stdout}" + ); + + // The Commands list precedes the legend, so first occurrences sit in the + // list and must appear in order: an `any` data verb, then a `direct` verb, + // then the `control` verb. + let pos = |needle: &str| { + stdout + .find(needle) + .unwrap_or_else(|| panic!("'{needle}' not found in --help:\n{stdout}")) + }; + assert!( + pos("query") < pos("optimize"), + "data (any) commands should be listed before direct commands" + ); + assert!( + pos("optimize") < pos("cluster"), + "direct commands should be listed before the control command" + ); +} + #[test] fn init_creates_graph_successfully_on_missing_local_directory() { let temp = tempdir().unwrap(); @@ -72,6 +104,28 @@ fn schema_plan_json_reports_supported_additive_change() { assert_eq!(payload["steps"][0]["property_name"], "nickname"); } +#[test] +fn schema_plan_with_server_flag_errors_wrong_plane() { + // RFC-010 Slice 1: `schema plan` is storage-plane while `schema show/apply` + // are data-plane — the guard rejects --server on plan with the per-subcommand + // label (proving command_plane/command_label descend into the nested enum). + let output = output_failure( + cli() + .arg("schema") + .arg("plan") + .arg("--schema") + .arg(fixture("test.pg")) + .arg("--server") + .arg("prod"), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("`schema plan` is a direct (storage-native) command") + && stderr.contains("Pass a storage URI."), + "schema plan wrong-capability message not found; got: {stderr}" + ); +} + #[test] fn schema_plan_json_reports_unsupported_type_change() { let temp = tempdir().unwrap(); @@ -280,7 +334,13 @@ fn schema_apply_json_adds_index_for_existing_property() { let dataset = snapshot.open("node:Person").await.unwrap(); dataset.load_indices().await.unwrap().len() }); - assert!(after_index_count > before_index_count); + // iss-848: `schema apply` records the `@index` intent but defers the physical + // index build (materialized later by ensure_indices/optimize; on this empty + // table nothing builds anyway). So the physical index count is unchanged. + assert_eq!( + after_index_count, before_index_count, + "schema apply records @index intent but defers the physical build (iss-848)" + ); } #[test] @@ -486,163 +546,18 @@ fn graphs_subcommand_help_lists_list_only() { #[test] fn graphs_list_against_local_uri_errors_with_remote_only_message() { + // RFC-011: `graphs list` is served-only; a `--store` (local) address has no + // enumeration endpoint, so it fails loudly pointing at a server / cluster. let output = output_failure( cli() .arg("graphs") .arg("list") - .arg("--uri") + .arg("--store") .arg("/tmp/local"), ); let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!( - stderr.contains("remote multi-graph server URL"), - "expected 'remote multi-graph server URL' rejection in stderr; got:\n{stderr}" + stderr.contains("remote multi-graph server"), + "expected a remote-server rejection in stderr; got:\n{stderr}" ); } - -/// RFC-008 stage 1: loading a legacy omnigraph.yaml emits the per-key -/// deprecation block (the migration map applied to THIS file), suppressible -/// via OMNIGRAPH_SUPPRESS_YAML_DEPRECATION. -#[test] -fn legacy_config_load_warns_per_key_and_suppression_silences() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "cli:\n actor: act-x\ngraphs:\n g:\n uri: /tmp/never-opened\n", - ) - .unwrap(); - - // `graphs list --json` loads the config and exits without touching the - // graph URI. - let output = cli() - .current_dir(temp.path()) - .arg("graphs") - .arg("list") - .arg("--json") - .output() - .unwrap(); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("deprecated (RFC-008)") && stderr.contains("`cli.actor` -> `operator.actor`"), - "{stderr}" - ); - assert!(stderr.contains("config migrate"), "{stderr}"); - - let output = cli() - .current_dir(temp.path()) - .env("OMNIGRAPH_SUPPRESS_YAML_DEPRECATION", "1") - .arg("graphs") - .arg("list") - .arg("--json") - .output() - .unwrap(); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(!stderr.contains("deprecated (RFC-008)"), "{stderr}"); -} - -/// RFC-008 stage 2: `config migrate` proposes the split read-only, applies -/// it with --write (operator merge never clobbers; cluster.yaml emitted), -/// and a second --write is idempotent. -#[test] -fn config_migrate_splits_legacy_config() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n prod:\n uri: https://graph.example.com\n bearer_token_env: PROD_TOKEN\ncli:\n actor: act-me\n output_format: json\npolicy:\n file: ./top.policy.yaml\n", - ) - .unwrap(); - let operator_home = tempfile::tempdir().unwrap(); - fs::write( - operator_home.path().join("config.yaml"), - "operator:\n actor: act-existing\n", - ) - .unwrap(); - - // Read-only proposal: names both halves, writes nothing. - let output = cli() - .current_dir(temp.path()) - .env("OMNIGRAPH_HOME", operator_home.path()) - .env("OMNIGRAPH_SUPPRESS_YAML_DEPRECATION", "1") - .arg("config") - .arg("migrate") - .output() - .unwrap(); - assert!(output.status.success(), "{output:?}"); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!(stdout.contains("team half -> cluster.yaml"), "{stdout}"); - assert!(stdout.contains("operator.actor: act-me"), "{stdout}"); - assert!(stdout.contains("omnigraph login prod"), "{stdout}"); - assert!(!temp.path().join("cluster.yaml").exists()); - - // --write: cluster.yaml lands; the existing operator actor is KEPT. - let output = cli() - .current_dir(temp.path()) - .env("OMNIGRAPH_HOME", operator_home.path()) - .env("OMNIGRAPH_SUPPRESS_YAML_DEPRECATION", "1") - .arg("config") - .arg("migrate") - .arg("--write") - .output() - .unwrap(); - assert!(output.status.success(), "{output:?}"); - let cluster = fs::read_to_string(temp.path().join("cluster.yaml")).unwrap(); - assert!(cluster.contains("version: 1") && cluster.contains(" prod:"), "{cluster}"); - let operator_text = - fs::read_to_string(operator_home.path().join("config.yaml")).unwrap(); - assert!(operator_text.contains("act-existing"), "{operator_text}"); - assert!(!operator_text.contains("act-me"), "existing keys win: {operator_text}"); - assert!(operator_text.contains("output: json"), "{operator_text}"); - assert!( - operator_text.contains("url: https://graph.example.com"), - "{operator_text}" - ); - - // Second --write: cluster.yaml exists -> proposal file, no clobber. - let output = cli() - .current_dir(temp.path()) - .env("OMNIGRAPH_HOME", operator_home.path()) - .env("OMNIGRAPH_SUPPRESS_YAML_DEPRECATION", "1") - .arg("config") - .arg("migrate") - .arg("--write") - .output() - .unwrap(); - assert!(output.status.success(), "{output:?}"); - assert!(temp.path().join("cluster.yaml.proposed").exists()); -} - -/// RFC-008 stage 4: OMNIGRAPH_NO_LEGACY_CONFIG refuses a present legacy -/// file (pointing at config migrate) but changes nothing on migrated -/// setups with no file. -#[test] -fn strict_mode_refuses_legacy_file_but_not_its_absence() { - let temp = tempdir().unwrap(); - fs::write(temp.path().join("omnigraph.yaml"), "cli:\n actor: a\n").unwrap(); - let output = cli() - .current_dir(temp.path()) - .env("OMNIGRAPH_NO_LEGACY_CONFIG", "1") - .arg("graphs") - .arg("list") - .arg("--json") - .output() - .unwrap(); - assert!(!output.status.success()); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("OMNIGRAPH_NO_LEGACY_CONFIG") && stderr.contains("config migrate"), - "{stderr}" - ); - - // Migrated setup (no file): strict mode is a no-op — a config-loading - // command that tolerates empty defaults succeeds. - let clean = tempdir().unwrap(); - let output = cli() - .current_dir(clean.path()) - .env("OMNIGRAPH_NO_LEGACY_CONFIG", "1") - .arg("queries") - .arg("list") - .arg("--json") - .output() - .unwrap(); - assert!(output.status.success(), "{output:?}"); -} diff --git a/crates/omnigraph-cli/tests/parity_matrix.rs b/crates/omnigraph-cli/tests/parity_matrix.rs new file mode 100644 index 0000000..e46f064 --- /dev/null +++ b/crates/omnigraph-cli/tests/parity_matrix.rs @@ -0,0 +1,285 @@ +//! RFC-009 Phase 1 — the embedded/remote parity referee. +//! +//! For every CLI verb with an `is_remote` fork, run the identical +//! invocation against (a) the local graph directly and (b) a spawned +//! server on a twin copy of the same graph, with the SAME actor on both +//! arms (local `--as act-parity`; remote bearer token resolving to +//! `act-parity`). Scrub the declared-volatile allowlist +//! (`support::scrub_volatile` — ids, wall-clock, transport locations); +//! everything else must match exactly. +//! +//! This test PINS behavior; it does not idealize it. Genuine divergences +//! discovered here are recorded in `KNOWN_DIVERGENCES` below (and filed), +//! never silently repaired — repairs are Phase 3's job, gated by this +//! referee staying green through the refactor. + +use tempfile::TempDir; + +mod support; +use support::*; + +/// Divergences between the arms that exist today, pinned as expectations. +/// Removing an entry requires the corresponding behavior change to be a +/// deliberate, release-noted decision (RFC-009 Compatibility). +const KNOWN_DIVERGENCES: &[&str] = &[ + // populated by the rows below as they are written +]; + +/// One matched setup per row: twin graphs + the parity Cedar bundle on the +/// served arm. The local (`--store`) arm carries no policy (RFC-011); the +/// bundle is permissive for `act-parity`, so the arms still agree. +struct Parity { + _temp: TempDir, + local: std::path::PathBuf, + server: TestServer, +} + +fn parity() -> Parity { + let (temp, local, remote) = twin_graphs(); + // RFC-011 cluster-only: the remote arm is served from a converged + // cluster directory (one graph, id `parity`), seeded with the same + // fixture data as the local twin. + let cluster_dir = parity_configs(temp.path(), &local, &remote); + let server = spawn_server_with_cluster_env( + &cluster_dir, + &[( + "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", + r#"{"act-parity":"parity-tok"}"#, + )], + ); + Parity { + _temp: temp, + local, + server, + } +} + +impl Parity { + fn run(&self, args: &[&str]) -> (std::process::Output, std::process::Output) { + run_both(&self.local, &self.server.base_url, args) + } +} + +fn assert_parity(verb: &str, local: &std::process::Output, remote: &std::process::Output) { + assert_eq!( + local.status.code(), + remote.status.code(), + "{verb}: exit codes diverge\nlocal: {local:?}\nremote: {remote:?}" + ); + if local.status.success() { + let local_json = scrubbed_json(local); + let remote_json = scrubbed_json(remote); + assert_eq!( + local_json, remote_json, + "{verb}: scrubbed JSON diverges (left=local, right=remote)" + ); + } +} + +#[test] +fn parity_query() { + let p = parity(); + let query = fixture("test.gq"); + let (l, r) = p.run(&[ + "query", + "--query", + query.to_str().unwrap(), + "get_person", + "--params", + r#"{"name":"Alice"}"#, + "--json", + ], + ); + assert_parity("query", &l, &r); +} + +#[test] +fn parity_schema_show() { + let p = parity(); + let (l, r) = p.run(&["schema", "show", "--json"]); + assert_parity("schema show", &l, &r); +} + +#[test] +fn parity_snapshot() { + let p = parity(); + let (l, r) = p.run(&["snapshot", "--json"]); + assert_parity("snapshot", &l, &r); +} + +#[test] +fn parity_branch_list() { + let p = parity(); + let (l, r) = p.run(&["branch", "list", "--json"]); + assert_parity("branch list", &l, &r); +} + +#[test] +fn parity_commit_list() { + let p = parity(); + let (l, r) = p.run(&["commit", "list", "--json"]); + assert_parity("commit list", &l, &r); +} + +#[test] +fn parity_mutate() { + let p = parity(); + let (l, r) = p.run(&[ + "mutate", + "-e", + "query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }", + "--params", + r#"{"name":"Parity","age":7}"#, + "--json", + ], + ); + assert_parity("mutate", &l, &r); +} + +#[test] +fn parity_branch_create_delete() { + let p = parity(); + let (l, r) = p.run(&["branch", "create", "--from", "main", "parity-branch", "--json"], + ); + assert_parity("branch create", &l, &r); + // `branch delete` is destructive: the served (remote) arm is non-local and + // requires consent (RFC-011 Decision 9), so the row passes `--yes` to test + // the operation itself, not the safety gate. The local arm ignores `--yes`. + let (l, r) = p.run(&["branch", "delete", "parity-branch", "--yes", "--json"], + ); + assert_parity("branch delete", &l, &r); +} + +#[test] +fn parity_branch_merge() { + let p = parity(); + let (l, r) = p.run(&["branch", "create", "--from", "main", "feature", "--json"], + ); + assert_parity("branch create (merge setup)", &l, &r); + let (l, r) = p.run(&["branch", "merge", "feature", "--into", "main", "--json"], + ); + assert_parity("branch merge", &l, &r); +} + +#[test] +fn parity_load() { + let p = parity(); + let data = p.local.parent().unwrap().join("rows.jsonl"); + std::fs::write( + &data, + "{\"type\":\"Person\",\"data\":{\"name\":\"Loaded\",\"age\":1}}\n", + ) + .unwrap(); + let (l, r) = p.run(&[ + "load", + "--mode", + "merge", + "--data", + data.to_str().unwrap(), + "--json", + ], + ); + assert_parity("load", &l, &r); +} + +#[test] +fn parity_export() { + let p = parity(); + let (l, r) = p.run(&["export"]); + // export emits a JSONL STREAM, not a single `--json` document, so the + // scrubbed-single-doc `assert_parity` doesn't apply — compare line-wise. + // The twin graphs are byte-copies of one loaded fixture, so rows carry + // identical ids/versions and need no scrubbing; sort the lines so any + // cross-arm row-ordering difference doesn't masquerade as a divergence. + assert_eq!( + l.status.code(), + r.status.code(), + "export: exit codes diverge\nlocal {l:?}\nremote {r:?}" + ); + assert!(l.status.success(), "export local arm failed: {l:?}"); + let mut local_lines: Vec<&str> = std::str::from_utf8(&l.stdout).unwrap().lines().collect(); + let mut remote_lines: Vec<&str> = std::str::from_utf8(&r.stdout).unwrap().lines().collect(); + assert!( + !local_lines.is_empty(), + "export produced no rows — the parity check would be vacuous" + ); + local_lines.sort_unstable(); + remote_lines.sort_unstable(); + assert_eq!( + local_lines, remote_lines, + "export: JSONL streams diverge (left=local, right=remote)" + ); +} + +// ---- error parity: exit codes must match for shared failure cases ---- + +#[test] +fn parity_errors_share_exit_codes() { + let p = parity(); + + // unknown branch on merge + let (l, r) = p.run(&["branch", "merge", "no-such-branch", "--into", "main", "--json"], + ); + assert_eq!( + (l.status.success(), r.status.success()), + (false, false), + "merge of unknown branch must fail on both arms\nlocal {l:?}\nremote {r:?}" + ); + + // unknown query name in the source + let query = fixture("test.gq"); + let (l, r) = p.run(&[ + "query", + "--query", + query.to_str().unwrap(), + "no_such_query", + "--json", + ], + ); + assert_eq!( + (l.status.success(), r.status.success()), + (false, false), + "unknown query name must fail on both arms\nlocal {l:?}\nremote {r:?}" + ); + + // Discovery (parity HOLDS, behavior surprising): an inline query run + // with a declared-but-unbound param does NOT error on either arm — it + // returns every row (the filter drops), while the stored-query invoke + // path hard-errors 'parameter not provided'. Pinned here as agreeing + // behavior; the cross-path asymmetry is filed separately. + let (l, r) = p.run(&[ + "query", + "--query", + query.to_str().unwrap(), + "get_person", + "--json", + ], + ); + assert_eq!( + (l.status.success(), r.status.success()), + (true, true), + "unbound-param inline query currently SUCCEEDS on both arms (matches-all)" + ); +} + +// ---- documented exclusions (not bugs; the Phase 4 capability table) ---- +// +// - `graphs list`: server-only today; becomes Both-capability when the +// embedded arm enumerates the cluster catalog (RFC-009 open Q3, answered). +// - `ingest`: deprecated alias of load; its remote arm rides the deprecated +// /ingest route. The canonical `load` verb targets `/load` (RFC-009 Phase 5, +// landed) — `parity_load` exercises it on the remote arm. +// - `init`, `optimize`, `repair`, `cleanup`, `cluster *`: storage-plane by +// design (must work with the server down); Phase 4 declares this. +#[allow(dead_code)] +const EXCLUSIONS_DOCUMENTED: () = (); + +#[test] +fn known_divergences_ledger_is_current() { + // The ledger exists so removals are deliberate: an empty list with all + // rows green means the arms agree everywhere the matrix looks. + assert!( + KNOWN_DIVERGENCES.is_empty(), + "divergences are pinned: {KNOWN_DIVERGENCES:?}" + ); +} diff --git a/crates/omnigraph-cli/tests/support/mod.rs b/crates/omnigraph-cli/tests/support/mod.rs index b11e94d..ff6a5d4 100644 --- a/crates/omnigraph-cli/tests/support/mod.rs +++ b/crates/omnigraph-cli/tests/support/mod.rs @@ -339,6 +339,63 @@ impl SystemGraph { } } +/// A converged cluster directory the server can boot from (`--cluster`), +/// serving one graph seeded with the standard fixture. Holds the temp dir +/// alive for the test's lifetime. +pub struct ClusterFixture { + _temp: TempDir, + dir: PathBuf, +} + +impl ClusterFixture { + pub fn path(&self) -> &Path { + &self.dir + } +} + +/// Build a converged cluster (RFC-011 cluster-only serving) with a single +/// graph `graph_id`, seeded with the `test.jsonl` fixture so reads return +/// data. When `policy_yaml` is `Some`, the bundle is bound to the graph +/// scope. The server boots from the returned path via `--cluster`. +pub fn converged_loaded_cluster(graph_id: &str, policy_yaml: Option<&str>) -> ClusterFixture { + let temp = tempdir().unwrap(); + let dir = temp.path().to_path_buf(); + fs::copy(fixture("test.pg"), dir.join("graph.pg")).unwrap(); + + let policy_block = match policy_yaml { + Some(source) => { + fs::write(dir.join("graph.policy.yaml"), source).unwrap(); + format!( + "policies:\n graph:\n file: ./graph.policy.yaml\n applies_to: [{graph_id}]\n" + ) + } + None => String::new(), + }; + fs::write( + dir.join("cluster.yaml"), + format!( + "version: 1\nmetadata:\n name: sys\nstate:\n backend: cluster\n lock: true\ngraphs:\n {graph_id}:\n schema: ./graph.pg\n{policy_block}" + ), + ) + .unwrap(); + + output_success(cli().arg("cluster").arg("import").arg("--config").arg(&dir)); + output_success(cli().arg("cluster").arg("apply").arg("--config").arg(&dir)); + + let served_root = dir.join("graphs").join(format!("{graph_id}.omni")); + output_success( + cli() + .arg("load") + .arg("--data") + .arg(fixture("test.jsonl")) + .arg("--mode") + .arg("overwrite") + .arg(&served_root), + ); + + ClusterFixture { _temp: temp, dir } +} + // ---- helpers moved from the monolithic tests/cli.rs ---- #[allow(unused_imports)] use lance::Dataset; @@ -688,3 +745,239 @@ pub fn queries_test_config(graph_uri: &str, entry: &str, gq_file: &str) -> Strin graph_uri.replace('\'', "''") ) } + +// ---- RFC-009 Phase 1: parity-matrix harness ---- + +/// Twin graphs for embedded-vs-remote comparison: the same loaded fixture +/// copied to two roots, so write verbs can run once per arm on identical +/// state. Returns (tempdir-guard, local_graph, remote_graph). +pub fn twin_graphs() -> (TempDir, PathBuf, PathBuf) { + let temp = tempdir().unwrap(); + let seed = temp.path().join("seed"); + fs::create_dir_all(&seed).unwrap(); + let graph = seed.join("server.omni"); + init_graph(&graph); + load_fixture(&graph); + let local = temp.path().join("local.omni"); + let remote = temp.path().join("remote.omni"); + copy_dir(&graph, &local); + copy_dir(&graph, &remote); + (temp, local, remote) +} + +pub fn copy_dir(from: &Path, to: &Path) { + fs::create_dir_all(to).unwrap(); + for entry in fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + fs::copy(entry.path(), &target).unwrap(); + } + } +} + +/// Scrub declared-volatile fields (RFC-009 Phase 1 allowlist) so the rest +/// of the JSON must match exactly. Key-based, recursive; both arms get the +/// same placeholders. Everything NOT listed here is contract. +pub fn scrub_volatile(value: &mut serde_json::Value) { + const VOLATILE_KEYS: &[&str] = &[ + // identity-bearing per-instance values + "commit_id", "id", "parent_id", "merge_parent_id", "snapshot", + // wall-clock + "committed_at", "created_at", "timestamp", + // transport / location + "uri", "path", + ]; + match value { + serde_json::Value::Object(map) => { + for (key, val) in map.iter_mut() { + if VOLATILE_KEYS.contains(&key.as_str()) && !val.is_null() { + *val = serde_json::Value::String(format!("")); + } else { + scrub_volatile(val); + } + } + } + serde_json::Value::Array(items) => { + for item in items { + scrub_volatile(item); + } + } + _ => {} + } +} + +pub const PARITY_ACTOR: &str = "act-parity"; +pub const PARITY_TOKEN: &str = "parity-tok"; + +/// Identical Cedar bundle for BOTH arms — like-for-like enforcement is part +/// of the parity contract (a bare local arm is permissive while a +/// tokens-only server is default-deny; comparing those would measure +/// configuration, not the fork). +pub fn parity_policy_yaml() -> String { + r#"version: 1 +groups: + parity: ["act-parity"] +protected_branches: [] +rules: + - id: reads + allow: + actors: { group: parity } + actions: [read, export, invoke_query] + - id: read-scope + allow: + actors: { group: parity } + actions: [read, export] + branch_scope: any + - id: writes + allow: + actors: { group: parity } + actions: [change] + branch_scope: any + - id: branching + allow: + actors: { group: parity } + actions: [schema_apply, branch_create, branch_delete, branch_merge] + target_branch_scope: any +"# + .to_string() +} + +/// The graph id the parity cluster serves the remote arm under. The +/// remote arm addresses it with `--graph PARITY_GRAPH_ID` (RFC-011: the +/// server is cluster-only, so a graph selector is required). +pub const PARITY_GRAPH_ID: &str = "parity"; + +/// Build the remote arm's configuration (RFC-011 cluster-only server). +/// +/// The remote arm is served from a converged cluster directory whose single +/// graph (id `parity`) carries the parity Cedar bundle (bound to the graph +/// scope). The cluster's derived graph root (`/graphs/parity.omni`) is +/// seeded with the SAME fixture data as the local twin so the two arms compare +/// like-for-like. The local (`--store`) arm carries no Cedar policy (RFC-011), +/// which is fine because the parity bundle is permissive for `act-parity`. +/// +/// `local_graph` is overwritten with a byte-for-byte copy of the cluster's +/// seeded served graph so identity-bearing values that are NOT scrubbed +/// (e.g. `graph_commit_id`, edge `id`s in export) match across the arms — +/// the served graph is the source of truth and the local twin mirrors it. +/// +/// Returns the `cluster_dir`. The caller spawns the server with `--cluster`. +pub fn parity_configs(root: &Path, local_graph: &Path, _remote_graph: &Path) -> PathBuf { + let policy = root.join("parity.policy.yaml"); + fs::write(&policy, parity_policy_yaml()).unwrap(); + + // Remote arm: a cluster directory the server boots from. One graph + // (`parity`), schema = the shared fixture, policy bound to the graph. + let cluster_dir = root.join("parity-cluster"); + fs::create_dir_all(&cluster_dir).unwrap(); + fs::copy(fixture("test.pg"), cluster_dir.join("parity.pg")).unwrap(); + fs::copy(&policy, cluster_dir.join("parity.policy.yaml")).unwrap(); + fs::write( + cluster_dir.join("cluster.yaml"), + format!( + r#"version: 1 +metadata: + name: parity +state: + backend: cluster + lock: true +graphs: + {PARITY_GRAPH_ID}: + schema: ./parity.pg +policies: + parity: + file: ./parity.policy.yaml + applies_to: [{PARITY_GRAPH_ID}] +"# + ), + ) + .unwrap(); + + // Converge the cluster (creates the empty graph at the derived root), + // then seed it with the same fixture data the local twin holds. + output_success( + cli() + .arg("cluster") + .arg("import") + .arg("--config") + .arg(&cluster_dir), + ); + output_success( + cli() + .arg("cluster") + .arg("apply") + .arg("--config") + .arg(&cluster_dir), + ); + let served_root = cluster_dir + .join("graphs") + .join(format!("{PARITY_GRAPH_ID}.omni")); + output_success( + cli() + .arg("load") + .arg("--data") + .arg(fixture("test.jsonl")) + .arg("--mode") + .arg("overwrite") + .arg(&served_root), + ); + + // Mirror the seeded served graph into the local twin so both arms hold + // identical ULIDs / commit ids (the served graph is authoritative). + if local_graph.exists() { + fs::remove_dir_all(local_graph).unwrap(); + } + copy_dir(&served_root, local_graph); + + cluster_dir +} + +/// Run one CLI invocation per arm with identical verb args: locally against +/// `local_graph` (--as actor) and remotely against a server URL whose token +/// resolves to the same actor. Returns raw Outputs for exit-code + JSON +/// comparison by the caller. +pub fn run_both( + local_graph: &Path, + server_url: &str, + args: &[&str], +) -> (std::process::Output, std::process::Output) { + // Address both arms with GLOBAL flags (`--store` / `--server`) appended after + // the verb + its args, so the address is placed correctly regardless of + // subcommand nesting (a positional graph only works for top-level verbs; + // `schema show ` etc. need the global flag). Local = embedded store, + // remote = served. RFC-011: a direct (`--store`) write carries no Cedar + // policy — the parity policy is permissive for `act-parity` on the served + // arm, so the two arms still agree. + let mut local = cli(); + local + .args(args) + .arg("--store") + .arg(local_graph) + .arg("--as") + .arg(PARITY_ACTOR); + let local_out = local.output().unwrap(); + + let mut remote = cli(); + remote + .env("OMNIGRAPH_BEARER_TOKEN", PARITY_TOKEN) + .args(args) + .arg("--server") + .arg(server_url) + // RFC-011: the parity server is cluster-only (multi-graph), so the + // remote arm must name the graph it addresses. + .arg("--graph") + .arg(PARITY_GRAPH_ID); + let remote_out = remote.output().unwrap(); + (local_out, remote_out) +} + +/// Parse, scrub, and pretty-print for diffable assertion messages. +pub fn scrubbed_json(output: &std::process::Output) -> String { + let mut value: serde_json::Value = serde_json::from_slice(&output.stdout) + .unwrap_or_else(|e| panic!("non-JSON stdout ({e}): {output:?}")); + scrub_volatile(&mut value); + serde_json::to_string_pretty(&value).unwrap() +} diff --git a/crates/omnigraph-cli/tests/system_local.rs b/crates/omnigraph-cli/tests/system_local.rs index 28ed7a3..9b3701e 100644 --- a/crates/omnigraph-cli/tests/system_local.rs +++ b/crates/omnigraph-cli/tests/system_local.rs @@ -3,6 +3,7 @@ mod support; use std::env; use std::fs; +use omnigraph::db::Omnigraph; use reqwest::blocking::Client; use serde_json::Value; @@ -62,53 +63,6 @@ cases: expect: allow "#; -fn yaml_string(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - -fn local_policy_config(graph: &SystemGraph) -> String { - format!( - "\ -project: - name: policy-e2e-local -graphs: - local: - uri: {} - policy: - file: ./policy.yaml -cli: - graph: local - branch: main -query: - roots: - - . -", - yaml_string(&graph.path().to_string_lossy()) - ) -} - -fn local_policy_server_graph_config(graph: &SystemGraph) -> String { - format!( - "\ -project: - name: policy-e2e-local -graphs: - local: - uri: {} - policy: - file: ./policy.yaml -server: - graph: local -cli: - branch: main -query: - roots: - - . -", - yaml_string(&graph.path().to_string_lossy()) - ) -} - fn insert_person_query(graph: &SystemGraph, name: &str) -> std::path::PathBuf { graph.write_query( name, @@ -231,10 +185,10 @@ fn local_cli_end_to_end_init_load_read_change_read_flow() { let read_before = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -246,6 +200,7 @@ fn local_cli_end_to_end_init_load_read_change_read_flow() { let change_payload = parse_stdout_json(&output_success( cli() .arg("change") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(&mutation_file) @@ -259,10 +214,10 @@ fn local_cli_end_to_end_init_load_read_change_read_flow() { let read_after = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Eve"}"#) @@ -277,6 +232,7 @@ fn local_cli_end_to_end_init_load_read_change_read_flow() { let inline_change = parse_stdout_json(&output_success( cli() .arg("change") + .arg("--store") .arg(graph.path()) .arg("-e") .arg("query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }") @@ -291,6 +247,7 @@ fn local_cli_end_to_end_init_load_read_change_read_flow() { let inline_read = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query-string") .arg("query find($name: String) { match { $p: Person { name: $name } } return { $p.name, $p.age } }") @@ -322,6 +279,7 @@ fn local_cli_end_to_end_branch_change_merge_flow() { let change_payload = parse_stdout_json(&output_success( cli() .arg("change") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(&mutation_file) @@ -337,10 +295,10 @@ fn local_cli_end_to_end_branch_change_merge_flow() { let feature_read = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--branch") .arg("feature") @@ -365,10 +323,10 @@ fn local_cli_end_to_end_branch_change_merge_flow() { let main_read = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Zoe"}"#) @@ -435,10 +393,10 @@ fn local_cli_ingest_creates_review_branch_and_keeps_it_readable() { let zoe = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--branch") .arg("feature-ingest") @@ -452,10 +410,10 @@ fn local_cli_ingest_creates_review_branch_and_keeps_it_readable() { let bob = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--branch") .arg("feature-ingest") @@ -629,10 +587,10 @@ fn local_cli_export_round_trips_full_branch_graph() { let eve = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(&imported_graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Eve"}"#) @@ -644,10 +602,10 @@ fn local_cli_export_round_trips_full_branch_graph() { let friends = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(&imported_graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("friends_of") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -665,31 +623,9 @@ fn local_cli_s3_end_to_end_init_load_read_flow() { let temp = tempfile::tempdir().unwrap(); let query_root = temp.path(); - let config = query_root.join("omnigraph.yaml"); let query = query_root.join("test.gq"); fs::copy(fixture("test.gq"), &query).unwrap(); - write_config( - &config, - &format!( - "\ -graphs: - rustfs: - uri: '{}' -cli: - graph: rustfs - branch: main -query: - roots: - - . -policy: {{}} -", - graph_uri - ), - ); - // current_dir matters: `init` scaffolds an omnigraph.yaml into its cwd, - // and without this it pollutes the crate dir, breaking unrelated tests - // (anything resolving a graph target from the cwd config). output_success( cli() .current_dir(query_root) @@ -704,20 +640,25 @@ policy: {{}} .arg("load") .arg("--mode") .arg("overwrite") + // `--yes` clears the RFC-011 Decision 9 destructive-write + // confirmation: `--mode overwrite` against a non-local (s3://) + // target is refused without it. + .arg("--yes") .arg("--data") .arg(fixture("test.jsonl")) .arg(&graph_uri), ); + // RFC-011: the graph is addressed by `--store `; the `.gq` path is + // resolved cwd-relative (no omnigraph.yaml `query.roots`). let read = parse_stdout_json(&output_success( cli() .current_dir(query_root) .arg("read") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(&graph_uri) .arg("--query") .arg("test.gq") - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -730,8 +671,8 @@ policy: {{}} cli() .current_dir(query_root) .arg("snapshot") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(&graph_uri) .arg("--json"), )); assert!(snapshot["tables"].is_array()); @@ -779,6 +720,7 @@ fn local_cli_failed_change_keeps_target_state_unchanged() { let output = output_failure( cli() .arg("change") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(&mutation_file) @@ -791,10 +733,10 @@ fn local_cli_failed_change_keeps_target_state_unchanged() { let friends_payload = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("friends_of") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -806,36 +748,22 @@ fn local_cli_failed_change_keeps_target_state_unchanged() { } #[test] -fn local_cli_resolves_relative_query_against_config_base_dir() { +fn local_cli_resolves_relative_query_cwd_relative() { + // RFC-011: omnigraph.yaml `query.roots` search is gone — a `--query` + // path is resolved plainly relative to the process cwd. This pins that + // a bare relative `.gq` filename resolves against `.current_dir`, and + // that the file actually read is the cwd-local one (a same-named query + // elsewhere with different columns is never picked up). let graph = SystemGraph::loaded(); let root = graph.path().parent().unwrap(); - let config_dir = root.join("config"); - let query_dir = config_dir.join("queries"); - let ambient_dir = root.join("ambient"); - fs::create_dir_all(&query_dir).unwrap(); - fs::create_dir_all(&ambient_dir).unwrap(); + let cwd_dir = root.join("cwd"); + let other_dir = root.join("other"); + fs::create_dir_all(&cwd_dir).unwrap(); + fs::create_dir_all(&other_dir).unwrap(); - let config = config_dir.join("omnigraph.yaml"); - write_config( - &config, - &format!( - "\ -graphs: - local: - uri: '{}' -cli: - graph: local - branch: main -query: - roots: - - queries -policy: {{}} -", - graph.path().display() - ), - ); + // The query in the cwd projects (age, name). write_query_file( - &query_dir.join("local.gq"), + &cwd_dir.join("local.gq"), r#" query get_person($name: String) { match { @@ -845,8 +773,10 @@ query get_person($name: String) { } "#, ); + // A same-named query elsewhere projects only (name): if cwd-relative + // resolution regressed and picked this up, the columns assert fails. write_query_file( - &ambient_dir.join("local.gq"), + &other_dir.join("local.gq"), r#" query get_person($name: String) { match { @@ -859,13 +789,12 @@ query get_person($name: String) { let payload = parse_stdout_json(&output_success( cli() - .current_dir(&ambient_dir) + .current_dir(&cwd_dir) .arg("read") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(graph.path()) .arg("--query") .arg("local.gq") - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -974,10 +903,10 @@ query get_task($slug: String) { let filtered = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(&queries) - .arg("--name") .arg("due_with_tag") .arg("--params") .arg(r#"{"deadline":"2026-04-02T00:00:00Z","tag":"launch"}"#) @@ -999,10 +928,10 @@ query get_task($slug: String) { let insert_payload = parse_stdout_json(&output_success( cli() .arg("change") + .arg("--store") .arg(&graph) .arg("--query") .arg(&queries) - .arg("--name") .arg("insert_task") .arg("--params") .arg( @@ -1015,10 +944,10 @@ query get_task($slug: String) { let update_payload = parse_stdout_json(&output_success( cli() .arg("change") + .arg("--store") .arg(&graph) .arg("--query") .arg(&queries) - .arg("--name") .arg("update_task") .arg("--params") .arg(r#"{"slug":"gamma","due_at":"2026-04-04T10:45:00Z","tags":["embed","released"],"scores":[13,21],"active_days":["2026-04-04","2026-04-05"]}"#) @@ -1029,10 +958,10 @@ query get_task($slug: String) { let gamma = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(&queries) - .arg("--name") .arg("get_task") .arg("--params") .arg(r#"{"slug":"gamma"}"#) @@ -1111,11 +1040,16 @@ query vector_search($q: String) { let result = parse_stdout_json(&output_success( cli() + // Stored vectors above were produced with gemini-embedding-2-preview; + // pin the query-time embedder to the same provider/model so the + // auto-embedded `$q` lands in the same vector space. + .env("OMNIGRAPH_EMBED_PROVIDER", "gemini") + .env("OMNIGRAPH_EMBED_MODEL", "gemini-embedding-2-preview") .arg("read") + .arg("--store") .arg(&graph) .arg("--query") .arg(&queries) - .arg("--name") .arg("vector_search") .arg("--params") .arg(r#"{"q":"alpha"}"#) @@ -1136,122 +1070,145 @@ query vector_search($q: String) { #[test] fn local_cli_policy_tooling_is_end_to_end() { - // Sanity check for the read-only policy CLI surfaces. These don't - // mutate the graph; they parse and evaluate the effective policy for - // named graph selections, including per-graph policy files. - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - let server_graph_config = graph.write_config( - "omnigraph-policy-server.yaml", - &local_policy_server_graph_config(&graph), + // RFC-011: the read-only policy CLI surfaces source the bundle from a + // cluster's applied policies (`--cluster ` + `--graph `), not + // from an omnigraph.yaml `graphs:` map. These don't mutate the graph; + // they parse and evaluate the effective bundle bound to the graph. + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + // `policy test` has no per-bundle tests file in the cluster model, so + // the cases are supplied explicitly via `--tests`. + let tests_file = cluster.path().join("policy.tests.yaml"); + fs::write(&tests_file, POLICY_E2E_TESTS_YAML).unwrap(); + + let validate = output_success( + cli() + .arg("policy") + .arg("validate") + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge"), ); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - graph.write_config("policy.tests.yaml", POLICY_E2E_TESTS_YAML); + assert!(stdout_string(&validate).contains("policy valid:")); - for config in [&config, &server_graph_config] { - let validate = output_success( - cli() - .arg("policy") - .arg("validate") - .arg("--config") - .arg(config), - ); - assert!(stdout_string(&validate).contains("policy valid:")); + let tests = output_success( + cli() + .arg("policy") + .arg("test") + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge") + .arg("--tests") + .arg(&tests_file), + ); + assert!(stdout_string(&tests).contains("policy tests passed: 2 cases")); - let tests = output_success(cli().arg("policy").arg("test").arg("--config").arg(config)); - assert!(stdout_string(&tests).contains("policy tests passed: 2 cases")); - - let explain = output_success( - cli() - .arg("policy") - .arg("explain") - .arg("--config") - .arg(config) - .arg("--actor") - .arg("act-bruno") - .arg("--action") - .arg("change") - .arg("--branch") - .arg("main"), - ); - let explain_stdout = stdout_string(&explain); - assert!(explain_stdout.contains("decision: deny")); - assert!(explain_stdout.contains("branch: main")); - } + let explain = output_success( + cli() + .arg("policy") + .arg("explain") + .arg("--cluster") + .arg(cluster.path()) + .arg("--graph") + .arg("knowledge") + .arg("--actor") + .arg("act-bruno") + .arg("--action") + .arg("change") + .arg("--branch") + .arg("main"), + ); + let explain_stdout = stdout_string(&explain); + assert!(explain_stdout.contains("decision: deny")); + assert!(explain_stdout.contains("branch: main")); } +/// Token→actor map for the served-policy tests: the bearer tokens the +/// cluster server resolves to `act-bruno` / `act-ragnor`. +const POLICY_TOKENS_JSON: &str = r#"{"act-bruno":"bruno-tok","act-ragnor":"ragnor-tok"}"#; + #[test] fn local_cli_change_enforces_engine_layer_policy() { - // Asserts MR-722 PR #4: when the selected graph has a configured - // policy file, the CLI loads PolicyEngine into Omnigraph and every - // direct-engine write hits `enforce(action, scope, actor)` — identical - // to what the HTTP server gets, regardless of transport. + // RFC-011: a CLI direct-store write carries NO policy — policy lives in + // the cluster/server. So engine-layer policy on a direct write no longer + // exists; this test asserts the faithful migration: the SERVER enforces + // the bundle bound to the served graph, addressed via `--server --graph` + // with a bearer token that resolves to the actor. // // Three cases, each discriminating: // - // 1. Policy installed, no actor source (no `cli.actor` in config, - // no `--as` flag) → engine-layer footgun guard fires; CLI exits - // non-zero with a "no actor" message. Silent bypass is the bug - // PR #4 prevents. - // 2. Policy installed, `--as act-bruno`, change on main → Cedar - // denies (bruno can change unprotected branches; main is - // protected). CLI exits non-zero with a "denied" message. - // 3. Policy installed, `--as act-ragnor`, change on main → - // Cedar permits (admins-write rule). Write succeeds and the - // inserted row is readable. - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - let mutation_file = insert_person_query(&graph, "system-local-policy-change.gq"); - - // Case 1: policy configured, no actor threaded → footgun guard. - let no_actor = output_failure( - cli() - .arg("change") - .arg("--config") - .arg(&config) - .arg("--query") - .arg(&mutation_file) - .arg("--params") - .arg(r#"{"name":"NoActorPerson","age":1}"#) - .arg("--json"), + // 1. No token → the server refuses (401, unauthenticated). The old + // embedded "no actor" footgun does not apply to the served path + // (the actor comes from the token), so this replaces it. + // 2. bruno token, change on protected main → Cedar denies (bruno can + // change unprotected branches; main is protected). Non-zero exit, + // "denied" surfaced from the server error body. + // 3. ragnor token, change on main → Cedar permits (admins-write). Write + // succeeds and the inserted row is readable. + if skip_system_e2e("local_cli_change_enforces_engine_layer_policy") { + return; + } + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[("OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", POLICY_TOKENS_JSON)], ); - let no_actor_stderr = String::from_utf8_lossy(&no_actor.stderr); + let insert = + "query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }"; + + // Case 1: no token → the server refuses before any policy check. + let no_token = cli() + .arg("change") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("-e") + .arg(insert) + .arg("--params") + .arg(r#"{"name":"NoTokenPerson","age":1}"#) + .arg("--json") + .output() + .unwrap(); assert!( - no_actor_stderr.contains("no actor"), - "expected 'no actor' footgun message, got stderr: {no_actor_stderr}" + !no_token.status.success(), + "unauthenticated served write must be refused: {no_token:?}" ); - // Case 2: `--as act-bruno` against protected main → denied. - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("change") - .arg("--config") - .arg(&config) - .arg("--query") - .arg(&mutation_file) - .arg("--params") - .arg(r#"{"name":"BrunoOnMain","age":2}"#) - .arg("--json"), - ); + // Case 2: bruno token against protected main → denied by the server. + let denied = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") + .arg("change") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("-e") + .arg(insert) + .arg("--params") + .arg(r#"{"name":"BrunoOnMain","age":2}"#) + .arg("--json") + .output() + .unwrap(); + assert!(!denied.status.success(), "bruno/main must be denied"); let denied_stderr = String::from_utf8_lossy(&denied.stderr); assert!( denied_stderr.contains("denied"), "expected 'denied' message for bruno/main, got stderr: {denied_stderr}" ); - // Case 3: `--as act-ragnor` against main → permitted by admins-write. + // Case 3: ragnor token against main → permitted by admins-write. let allowed = parse_stdout_json(&output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("change") - .arg("--config") - .arg(&config) - .arg("--query") - .arg(&mutation_file) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("-e") + .arg(insert) .arg("--params") .arg(r#"{"name":"RagnorOnMain","age":3}"#) .arg("--json"), @@ -1261,14 +1218,19 @@ fn local_cli_change_enforces_engine_layer_policy() { assert_eq!(allowed["actor_id"], "act-ragnor"); // Verify the row landed — proves the write actually committed, not - // just that enforce returned Ok and silently dropped the work. + // just that enforce returned Ok and silently dropped the work. The read + // uses the bruno token: POLICY_E2E_YAML grants `read` to the `team` + // group (bruno), while admins (ragnor) get write-only rules. let verify = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") .arg("read") - .arg(graph.path()) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"RagnorOnMain"}"#) @@ -1279,27 +1241,30 @@ fn local_cli_change_enforces_engine_layer_policy() { } #[test] -fn local_cli_positional_uri_does_not_inherit_default_graph_policy() { +fn local_cli_direct_store_write_is_unpoliced_regardless_of_actor() { + // RFC-011: a direct (`--store`) write carries no Cedar policy at all — + // policy lives in the cluster/server. So a write that the SERVED path + // would deny (bruno changing protected main) succeeds on the direct + // path, regardless of the actor. This is the faithful replacement for + // the obsolete `..._positional_uri_does_not_inherit_default_graph_policy` + // premise: a positional/`--store` address has no policy to inherit. let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - let mutation_file = insert_person_query(&graph, "system-local-policy-positional.gq"); + let mutation_file = insert_person_query(&graph, "system-local-policy-direct.gq"); let allowed = parse_stdout_json(&output_success( cli() .arg("--as") .arg("act-bruno") .arg("change") - .arg("--config") - .arg(&config) - .arg("--uri") + .arg("--store") .arg(graph.path()) .arg("--query") .arg(&mutation_file) .arg("--params") - .arg(r#"{"name":"PositionalUriBruno","age":4}"#) + .arg(r#"{"name":"DirectStoreBruno","age":4}"#) .arg("--json"), )); + assert_eq!(allowed["branch"], "main"); assert_eq!(allowed["affected_nodes"], 1); assert_eq!(allowed["actor_id"], "act-bruno"); } @@ -1317,28 +1282,44 @@ fn local_cli_positional_uri_does_not_inherit_default_graph_policy() { #[test] fn local_cli_load_enforces_engine_layer_policy() { - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - let data = graph.write_jsonl( - "system-local-policy-load.jsonl", - r#"{"type":"Person","data":{"name":"LoadPolicy","age":11}}"#, + // RFC-011 served re-point: the server enforces the graph-bound bundle on + // a remote load. A load into protected main is a `change`: bruno + // (team-write-unprotected) is denied, ragnor (admins-write) is allowed. + if skip_system_e2e("local_cli_load_enforces_engine_layer_policy") { + return; + } + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[("OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", POLICY_TOKENS_JSON)], ); + let temp = tempfile::tempdir().unwrap(); + let data = temp.path().join("policy-load.jsonl"); + fs::write( + &data, + r#"{"type":"Person","data":{"name":"LoadPolicy","age":11}}"#, + ) + .unwrap(); // act-bruno: change-on-protected is denied (team-write-unprotected only). - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("load") - .arg("--mode") - .arg("overwrite") - .arg("--config") - .arg(&config) - .arg("--data") - .arg(&data) - .arg("--json"), - ); + let denied = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") + .arg("load") + .arg("--mode") + .arg("overwrite") + // `--yes` clears the RFC-011 Decision 9 destructive-write confirmation + // so the policy check (not the confirmation refusal) is what denies. + .arg("--yes") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("--data") + .arg(&data) + .arg("--json") + .output() + .unwrap(); + assert!(!denied.status.success(), "bruno/main load must be denied"); let stderr = String::from_utf8_lossy(&denied.stderr); assert!( stderr.contains("denied"), @@ -1348,13 +1329,15 @@ fn local_cli_load_enforces_engine_layer_policy() { // act-ragnor: admins-write rule permits change anywhere. let allowed = parse_stdout_json(&output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("load") .arg("--mode") .arg("overwrite") - .arg("--config") - .arg(&config) + .arg("--yes") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("--data") .arg(&data) .arg("--json"), @@ -1365,47 +1348,55 @@ fn local_cli_load_enforces_engine_layer_policy() { #[test] fn local_cli_ingest_enforces_engine_layer_policy() { - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - let data = graph.write_jsonl( - "system-local-policy-ingest.jsonl", + // RFC-011 served re-point: ingest into a new branch requires both + // BranchCreate and Change. Bruno has change-unprotected only (no + // branch-ops) — either gate denies. Ragnor has admins-write + + // admins-branch-ops — both fire as ingest creates the branch + loads. + if skip_system_e2e("local_cli_ingest_enforces_engine_layer_policy") { + return; + } + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[("OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", POLICY_TOKENS_JSON)], + ); + let temp = tempfile::tempdir().unwrap(); + let data = temp.path().join("policy-ingest.jsonl"); + fs::write( + &data, r#"{"type":"Person","data":{"name":"IngestPolicy","age":12}}"#, - ); + ) + .unwrap(); - // act-bruno: ingest into a new branch requires both BranchCreate and - // Change. Bruno has change-unprotected only, and the implicit - // branch_create fires first when the target branch doesn't exist. - // Either gate is enough to deny — assert denial without pinning - // which one fires first. - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("ingest") - .arg("--config") - .arg(&config) - .arg("--data") - .arg(&data) - .arg("--branch") - .arg("policy-ingest-feature") - .arg("--json"), - ); + let denied = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") + .arg("ingest") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("--data") + .arg(&data) + .arg("--branch") + .arg("policy-ingest-feature") + .arg("--json") + .output() + .unwrap(); + assert!(!denied.status.success(), "bruno ingest must be denied"); let stderr = String::from_utf8_lossy(&denied.stderr); assert!( stderr.contains("denied"), "expected 'denied' for bruno ingest, got: {stderr}" ); - // act-ragnor: admins-write covers Change, admins-branch-ops covers - // BranchCreate. Both fire as ingest creates the branch + loads. let allowed = parse_stdout_json(&output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("ingest") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("--data") .arg(&data) .arg("--branch") @@ -1416,130 +1407,33 @@ fn local_cli_ingest_enforces_engine_layer_policy() { assert_eq!(allowed["branch_created"], true); } -#[test] -fn local_cli_schema_apply_enforces_engine_layer_policy() { - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - - // Additive: add a nullable property; SDK-compatible with the fixture - // schema. Uses the schema-apply scope (TargetBranch("main")). - let new_schema = std::fs::read_to_string(fixture("test.pg")) - .unwrap() - .replace( - " age: I32?\n}", - " age: I32?\n nickname: String?\n}", - ); - let schema_path = graph.path().join("policy-additive.pg"); - std::fs::write(&schema_path, &new_schema).unwrap(); - - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("schema") - .arg("apply") - .arg("--config") - .arg(&config) - .arg("--schema") - .arg(&schema_path) - .arg("--json"), - ); - let stderr = String::from_utf8_lossy(&denied.stderr); - assert!( - stderr.contains("denied"), - "expected 'denied' for bruno schema apply, got: {stderr}" - ); - - let allowed = parse_stdout_json(&output_success( - cli() - .arg("--as") - .arg("act-ragnor") - .arg("schema") - .arg("apply") - .arg("--config") - .arg(&config) - .arg("--schema") - .arg(&schema_path) - .arg("--json"), - )); - assert_eq!(allowed["applied"], true); -} - -#[test] -fn local_cli_schema_apply_rejects_stored_query_breakage_before_publish() { - let graph = SystemGraph::loaded(); - graph.write_query( - "stored-find-person.gq", - "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }", - ); - let config = graph.write_config( - "omnigraph-stored-query-schema.yaml", - &format!( - "\ -graphs: - local: - uri: {} - queries: - find_person: - file: ./stored-find-person.gq -cli: - graph: local - branch: main -query: - roots: - - . -policy: {{}} -", - yaml_string(&graph.path().to_string_lossy()) - ), - ); - let renamed_schema = std::fs::read_to_string(fixture("test.pg")) - .unwrap() - .replace("age: I32?", "years: I32? @rename_from(\"age\")"); - let schema_path = graph.write_file("stored-query-breaks.pg", &renamed_schema); - - let rejected = output_failure( - cli() - .arg("schema") - .arg("apply") - .arg("--config") - .arg(&config) - .arg("--schema") - .arg(&schema_path) - .arg("--json"), - ); - let stderr = String::from_utf8_lossy(&rejected.stderr); - assert!( - stderr.contains("find_person") && stderr.contains("schema check"), - "schema apply should reject the stored-query breakage before publish; stderr: {stderr}" - ); - - let schema = stdout_string(&output_success( - cli().arg("schema").arg("show").arg("--config").arg(&config), - )); - assert!(schema.contains("age: I32?")); - assert!(!schema.contains("years: I32?")); -} - #[test] fn local_cli_branch_create_enforces_engine_layer_policy() { - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); - - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("branch") - .arg("create") - .arg("--config") - .arg(&config) - .arg("--from") - .arg("main") - .arg("bruno-feature"), + // RFC-011 served re-point: bruno has no branch-ops rule → denied; + // ragnor has admins-branch-ops → allowed. + if skip_system_e2e("local_cli_branch_create_enforces_engine_layer_policy") { + return; + } + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[("OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", POLICY_TOKENS_JSON)], ); + + let denied = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") + .arg("branch") + .arg("create") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("--from") + .arg("main") + .arg("bruno-feature") + .output() + .unwrap(); + assert!(!denied.status.success(), "bruno branch create must be denied"); let stderr = String::from_utf8_lossy(&denied.stderr); assert!( stderr.contains("denied"), @@ -1548,12 +1442,13 @@ fn local_cli_branch_create_enforces_engine_layer_policy() { output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("--from") .arg("main") .arg("ragnor-feature"), @@ -1562,34 +1457,47 @@ fn local_cli_branch_create_enforces_engine_layer_policy() { #[test] fn local_cli_branch_delete_enforces_engine_layer_policy() { - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); + // RFC-011 served re-point: bruno has no branch-ops rule → denied; + // ragnor has admins-branch-ops → allowed. + if skip_system_e2e("local_cli_branch_delete_enforces_engine_layer_policy") { + return; + } + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[("OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", POLICY_TOKENS_JSON)], + ); // Pre-create the branch as ragnor so there's something to delete. output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("--from") .arg("main") .arg("doomed"), ); - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("branch") - .arg("delete") - .arg("--config") - .arg(&config) - .arg("doomed"), - ); + // `--yes` clears the RFC-011 Decision 9 destructive-write confirmation so + // the policy check (not the confirmation refusal) is what denies. + let denied = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") + .arg("branch") + .arg("delete") + .arg("--yes") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("doomed") + .output() + .unwrap(); + assert!(!denied.status.success(), "bruno branch delete must be denied"); let stderr = String::from_utf8_lossy(&denied.stderr); assert!( stderr.contains("denied"), @@ -1598,48 +1506,61 @@ fn local_cli_branch_delete_enforces_engine_layer_policy() { output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("branch") .arg("delete") - .arg("--config") - .arg(&config) + .arg("--yes") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("doomed"), ); } #[test] fn local_cli_branch_merge_enforces_engine_layer_policy() { - let graph = SystemGraph::loaded(); - let config = graph.write_config("omnigraph-policy.yaml", &local_policy_config(&graph)); - graph.write_config("policy.yaml", POLICY_E2E_YAML); + // RFC-011 served re-point: merging into protected main needs + // branch_merge with target_branch_scope protected. bruno has no such + // rule → denied; ragnor has admins-promote → allowed. + if skip_system_e2e("local_cli_branch_merge_enforces_engine_layer_policy") { + return; + } + let cluster = converged_loaded_cluster("knowledge", Some(POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[("OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", POLICY_TOKENS_JSON)], + ); // Pre-create a feature branch as ragnor (admins-branch-ops covers it). output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("--from") .arg("main") .arg("merge-feature"), ); - let denied = output_failure( - cli() - .arg("--as") - .arg("act-bruno") - .arg("branch") - .arg("merge") - .arg("--config") - .arg(&config) - .arg("merge-feature") - .arg("--into") - .arg("main"), - ); + let denied = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "bruno-tok") + .arg("branch") + .arg("merge") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") + .arg("merge-feature") + .arg("--into") + .arg("main") + .output() + .unwrap(); + assert!(!denied.status.success(), "bruno branch merge must be denied"); let stderr = String::from_utf8_lossy(&denied.stderr); assert!( stderr.contains("denied"), @@ -1648,68 +1569,56 @@ fn local_cli_branch_merge_enforces_engine_layer_policy() { output_success( cli() - .arg("--as") - .arg("act-ragnor") + .env("OMNIGRAPH_BEARER_TOKEN", "ragnor-tok") .arg("branch") .arg("merge") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg("knowledge") .arg("merge-feature") .arg("--into") .arg("main"), ); } -// ─── MR-722 PR A: cli.actor config-only precedence ──────────────────────── +// ─── RFC-011: operator.actor cascade ────────────────────────────────────── // -// The change-writer test above uses `--as` directly. These two tests -// pin the precedence rule that `main.rs::resolve_cli_actor` implements: -// `--as` flag > `cli.actor` from `omnigraph.yaml` > None. +// The CLI actor chain is `--as` > `operator.actor` (in the operator config +// at $OMNIGRAPH_HOME/config.yaml) > none. These two tests pin that order on +// a direct (`--store`) write. RFC-011 makes direct-store writes unpoliced, +// so the assertion is on which `actor_id` the write records, not on a Cedar +// allow/deny — the actor still has to be resolved correctly and stamped onto +// the commit. -fn local_policy_config_with_actor(graph: &SystemGraph, actor: &str) -> String { - // Mirrors `local_policy_config` but adds `cli.actor` so the - // config-only precedence path is exercised. The `cli:` block - // already has `graph` and `branch`; appending `actor` here. - format!( - "\ -project: - name: policy-e2e-local -graphs: - local: - uri: {} - policy: - file: ./policy.yaml -cli: - graph: local - branch: main - actor: {} -query: - roots: - - . -", - yaml_string(&graph.path().to_string_lossy()), - actor, +/// An operator config (`$OMNIGRAPH_HOME/config.yaml`) carrying just +/// `operator.actor`. Pointing OMNIGRAPH_HOME at the holding dir makes the +/// CLI read it as the operator layer. +fn operator_home_with_actor(actor: &str) -> tempfile::TempDir { + let home = tempfile::tempdir().unwrap(); + fs::write( + home.path().join("config.yaml"), + format!("operator:\n actor: {actor}\n"), ) + .unwrap(); + home } #[test] fn local_cli_actor_from_config_used_when_no_flag() { - // cli.actor: act-ragnor in omnigraph.yaml, no --as flag → change - // permitted via admins-write rule. Proves the config-only path - // works; previously the only proof was structural. + // operator.actor: act-ragnor in the operator config, no --as flag → + // the write records act-ragnor. Proves the operator-layer actor source + // is consulted when `--as` is absent. let graph = SystemGraph::loaded(); - let config = graph.write_config( - "omnigraph-policy.yaml", - &local_policy_config_with_actor(&graph, "act-ragnor"), - ); - graph.write_config("policy.yaml", POLICY_E2E_YAML); + let home = operator_home_with_actor("act-ragnor"); let mutation_file = insert_person_query(&graph, "system-local-cli-actor.gq"); let allowed = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_HOME", home.path()) .arg("change") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(graph.path()) .arg("--query") .arg(&mutation_file) .arg("--params") @@ -1722,35 +1631,30 @@ fn local_cli_actor_from_config_used_when_no_flag() { #[test] fn local_cli_actor_flag_overrides_config_actor() { - // cli.actor: act-ragnor in config + --as act-bruno on CLI → change - // denied. Flag wins per the precedence rule. Without this test, a - // future change that reverses precedence would ride through silently. + // operator.actor: act-ragnor in the config + --as act-bruno on the CLI → + // the write records act-bruno. The flag wins per the precedence rule. + // Without this test, a future change that reverses precedence would ride + // through silently. let graph = SystemGraph::loaded(); - let config = graph.write_config( - "omnigraph-policy.yaml", - &local_policy_config_with_actor(&graph, "act-ragnor"), - ); - graph.write_config("policy.yaml", POLICY_E2E_YAML); + let home = operator_home_with_actor("act-ragnor"); let mutation_file = insert_person_query(&graph, "system-local-cli-actor-override.gq"); - let denied = output_failure( + let overridden = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_HOME", home.path()) .arg("--as") .arg("act-bruno") .arg("change") - .arg("--config") - .arg(&config) + .arg("--store") + .arg(graph.path()) .arg("--query") .arg(&mutation_file) .arg("--params") .arg(r#"{"name":"OverrideEve","age":19}"#) .arg("--json"), - ); - let stderr = String::from_utf8_lossy(&denied.stderr); - assert!( - stderr.contains("denied"), - "expected 'denied' when --as overrides config to bruno, got: {stderr}" - ); + )); + assert_eq!(overridden["affected_nodes"], 1); + assert_eq!(overridden["actor_id"], "act-bruno"); } /// Phase 5 (RFC-005): "applied means serving" — converge a cluster with the @@ -2048,22 +1952,16 @@ fn local_cluster_full_lifecycle_declare_serve_evolve_delete() { } // Out-of-band drift: the live graph evolves behind the cluster's back; - // refresh observes it, apply converges it back to the declared schema. - std::fs::write( - dir.join("rogue.pg"), - "\nnode Person {\n name: String @key\n bio: String?\n rogue: String?\n}\n", - ) - .unwrap(); - let output = cli() - .arg("schema") - .arg("apply") - .arg(dir.join("graphs/knowledge.omni")) - .arg("--schema") - .arg(dir.join("rogue.pg")) - .arg("--json") - .output() - .unwrap(); - assert!(output.status.success(), "out-of-band schema apply failed"); + // refresh observes it, apply converges it back to the declared schema. RFC-011 + // D10 makes the CLI `schema apply` refuse a cluster-managed graph, so a true + // bypass is a direct engine apply against the storage root. + let rogue_pg = "\nnode Person {\n name: String @key\n bio: String?\n rogue: String?\n}\n"; + tokio::runtime::Runtime::new().unwrap().block_on(async { + let db = Omnigraph::open(dir.join("graphs/knowledge.omni").to_string_lossy().as_ref()) + .await + .unwrap(); + db.apply_schema(rogue_pg).await.unwrap(); + }); let refresh = cluster_cli(dir, &["refresh"]); assert_eq!( refresh["resource_statuses"]["schema.knowledge"]["status"], @@ -2316,9 +2214,12 @@ fn cluster_server_boot_ignores_local_config_in_cwd() { /// 3), and `logout` revokes. #[test] fn local_cli_keyed_credentials_authenticate_url_matched_server() { - let graph = SystemGraph::loaded(); - let server = spawn_server_with_env( - graph.path(), + // RFC-011 cluster-only: the server boots from a converged cluster + // serving the fixture graph under id `local`; tokens-only boot is + // default-deny, which still permits `read`. + let cluster = converged_loaded_cluster("local", None); + let server = spawn_server_with_cluster_env( + cluster.path(), &[("OMNIGRAPH_SERVER_BEARER_TOKEN", "secret-tok")], ); let operator_home = tempfile::tempdir().unwrap(); @@ -2339,10 +2240,12 @@ fn local_cli_keyed_credentials_authenticate_url_matched_server() { } command .arg("read") + .arg("--server") .arg(&server.base_url) + .arg("--graph") + .arg("local") .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -2429,26 +2332,45 @@ fn local_cli_keyed_credentials_authenticate_url_matched_server() { /// stored queries) end to end, with the keyed credential from PR 2. #[test] fn local_cli_operator_alias_and_server_flag_invoke_stored_query() { - let graph = SystemGraph::loaded(); - graph.write_query( - "stored-find-person.gq", + // RFC-011 cluster-only: build a converged cluster serving graph `local` + // with a stored query `find_person` and a per-graph policy granting the + // operator invoke_query + read (invoke_query is policy-gated — anti-probing + // 404 without the grant). + let cluster = tempfile::tempdir().unwrap(); + fs::copy(fixture("test.pg"), cluster.path().join("local.pg")).unwrap(); + fs::write( + cluster.path().join("find-person.gq"), "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.name } }", + ) + .unwrap(); + fs::write( + cluster.path().join("insert-person.gq"), + "query insert_person($name: String) { insert Person { name: $name, age: 41 } }", + ) + .unwrap(); + fs::write( + cluster.path().join("graph.policy.yaml"), + "version: 1\ngroups:\n ops: [\"act-op\"]\nprotected_branches: [main]\nrules:\n - id: allow-invoke\n allow:\n actors: { group: ops }\n actions: [invoke_query]\n - id: allow-read\n allow:\n actors: { group: ops }\n actions: [read]\n branch_scope: any\n - id: allow-change\n allow:\n actors: { group: ops }\n actions: [change]\n branch_scope: any\n", + ) + .unwrap(); + fs::write( + cluster.path().join("cluster.yaml"), + "version: 1\nmetadata:\n name: alias-sys\nstate:\n backend: cluster\n lock: true\ngraphs:\n local:\n schema: ./local.pg\n queries:\n find_person:\n file: ./find-person.gq\n insert_person:\n file: ./insert-person.gq\npolicies:\n graph:\n file: ./graph.policy.yaml\n applies_to: [local]\n", + ) + .unwrap(); + output_success(cli().arg("cluster").arg("import").arg("--config").arg(cluster.path())); + output_success(cli().arg("cluster").arg("apply").arg("--config").arg(cluster.path())); + output_success( + cli() + .arg("load") + .arg("--data") + .arg(fixture("test.jsonl")) + .arg("--mode") + .arg("overwrite") + .arg(cluster.path().join("graphs").join("local.omni")), ); - // invoke_query is policy-gated (anti-probing 404 without the grant), - // so the server gets a per-graph bundle granting it to the operator. - graph.write_file( - "graph.policy.yaml", - "version: 1\ngroups:\n ops: [\"act-op\"]\nprotected_branches: [main]\nrules:\n - id: allow-invoke\n allow:\n actors: { group: ops }\n actions: [invoke_query]\n - id: allow-read\n allow:\n actors: { group: ops }\n actions: [read]\n branch_scope: any\n", - ); - let config = graph.write_config( - "omnigraph-server.yaml", - &format!( - "graphs:\n local:\n uri: {}\n policy:\n file: ./graph.policy.yaml\n queries:\n find_person:\n file: ./stored-find-person.gq\n", - yaml_string(&graph.path().to_string_lossy()) - ), - ); - let server = spawn_server_with_config_env( - &config, + let server = spawn_server_with_cluster_env( + cluster.path(), &[( "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", r#"{"act-op":"srv-tok"}"#, @@ -2459,7 +2381,7 @@ fn local_cli_operator_alias_and_server_flag_invoke_stored_query() { fs::write( operator_home.path().join("config.yaml"), format!( - "servers:\n dev:\n url: {}\naliases:\n who:\n server: dev\n graph: local\n query: find_person\n args: [name]\n", + "servers:\n dev:\n url: {}\naliases:\n who:\n server: dev\n graph: local\n query: find_person\n args: [name]\n create_person:\n server: dev\n graph: local\n query: insert_person\n args: [name]\n", server.base_url ), ) @@ -2479,12 +2401,11 @@ fn local_cli_operator_alias_and_server_flag_invoke_stored_query() { .unwrap(); } - // The operator alias: name + positional arg, nothing else — server, + // The operator alias (RFC-011 D4): `alias [args]` — server, // graph, stored query, and token all resolve from the operator layer. let output = cli() .env("OMNIGRAPH_HOME", operator_home.path()) - .arg("query") - .arg("--alias") + .arg("alias") .arg("who") .arg("Alice") .arg("--json") @@ -2497,6 +2418,46 @@ fn local_cli_operator_alias_and_server_flag_invoke_stored_query() { let payload: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); assert_eq!(payload["rows"][0]["p.name"], "Alice", "{payload}"); + // Operator aliases are read-only conveniences: a binding to a stored + // mutation must be rejected before the server executes it. + let output = cli() + .env("OMNIGRAPH_HOME", operator_home.path()) + .arg("alias") + .arg("create_person") + .arg("AliasGuardPerson") + .output() + .unwrap(); + assert!(!output.status.success(), "mutation alias must fail"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("'insert_person' is a mutation") + && stderr.contains("omnigraph mutate insert_person"), + "expected mutation-kind mismatch; got: {stderr}" + ); + let output = cli() + .env("OMNIGRAPH_HOME", operator_home.path()) + .arg("query") + .arg("find_person") + .arg("--server") + .arg("dev") + .arg("--graph") + .arg("local") + .arg("--params") + .arg(r#"{"name":"AliasGuardPerson"}"#) + .arg("--json") + .output() + .unwrap(); + assert!( + output.status.success(), + "post-alias read should succeed: {output:?}" + ); + let payload: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + payload["rows"].as_array().unwrap().len(), + 0, + "mutation alias must not insert AliasGuardPerson: {payload}" + ); + // --server/--graph: the same stored query via explicit targeting. let output = cli() .env("OMNIGRAPH_HOME", operator_home.path()) @@ -2514,6 +2475,45 @@ fn local_cli_operator_alias_and_server_flag_invoke_stored_query() { .unwrap(); assert!(output.status.success(), "{output:?}"); + // RFC-011 D3: invoke the STORED query by name (catalog lane, served-only). + // No `-e`/`--query` — the positional `find_person` is the catalog name. + let output = cli() + .env("OMNIGRAPH_HOME", operator_home.path()) + .arg("query") + .arg("find_person") + .arg("--server") + .arg("dev") + .arg("--graph") + .arg("local") + .arg("--params") + .arg(r#"{"name":"Alice"}"#) + .arg("--json") + .output() + .unwrap(); + assert!(output.status.success(), "by-name catalog invocation: {output:?}"); + let payload: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(payload["rows"][0]["p.name"], "Alice", "{payload}"); + + // The verb asserts kind: `mutate ` is rejected by the server. + let output = cli() + .env("OMNIGRAPH_HOME", operator_home.path()) + .arg("mutate") + .arg("find_person") + .arg("--server") + .arg("dev") + .arg("--graph") + .arg("local") + .arg("--params") + .arg(r#"{"name":"Alice"}"#) + .output() + .unwrap(); + assert!(!output.status.success(), "mutate on a read query must fail"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("'find_person' is a read — use omnigraph query find_person"), + "expected a kind-mismatch error; got: {stderr}" + ); + // Unknown --server errors listing what IS defined. let output = cli() .env("OMNIGRAPH_HOME", operator_home.path()) @@ -2528,10 +2528,14 @@ fn local_cli_operator_alias_and_server_flag_invoke_stored_query() { let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("unknown server 'nope'") && stderr.contains("dev"), "{stderr}"); - // --server is exclusive with a positional URI. + // --server is exclusive with --store (two ways to address the graph). + // (RFC-011 D3: there is no positional URI anymore — the positional is a + // query name — so the double-addressing contradiction now surfaces between + // the two scope primitives.) let output = cli() .env("OMNIGRAPH_HOME", operator_home.path()) .arg("query") + .arg("--store") .arg(&server.base_url) .arg("--server") .arg("dev") diff --git a/crates/omnigraph-cli/tests/system_remote.rs b/crates/omnigraph-cli/tests/system_remote.rs index 95a53e7..19f460e 100644 --- a/crates/omnigraph-cli/tests/system_remote.rs +++ b/crates/omnigraph-cli/tests/system_remote.rs @@ -8,6 +8,14 @@ use serde_json::json; use support::*; +/// Graph id every served test addresses (`--server --graph GRAPH_ID`). +/// RFC-011: the server is cluster-only, so a graph selector is always required +/// — even for a single-graph cluster. +const GRAPH_ID: &str = "knowledge"; + +/// Graph-bound Cedar bundle for the policy-flavored remote tests. `act-bruno` +/// (team) reads + writes unprotected branches; `act-ragnor` (admins) merges +/// into protected `main`. const REMOTE_POLICY_E2E_YAML: &str = r#" version: 1 groups: @@ -37,6 +45,8 @@ rules: target_branch_scope: protected "#; +/// Server-scoped bundle granting `act-admin` the `graph_list` action so +/// `GET /graphs` succeeds. const GRAPH_LIST_SERVER_POLICY_YAML: &str = r#" version: 1 groups: @@ -48,61 +58,24 @@ rules: actions: [graph_list] "#; -fn yaml_string(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - -fn remote_policy_server_config(graph: &SystemGraph) -> String { - format!( - "\ -project: - name: remote-policy-e2e -graphs: - local: - uri: {} - policy: - file: ./policy.yaml -server: - graph: local -", - yaml_string(&graph.path().to_string_lossy()) - ) -} - -fn remote_policy_client_config(url: &str) -> String { - format!( - "\ -graphs: - dev: - uri: {} - bearer_token_env: POLICY_TEST_TOKEN -cli: - graph: dev - branch: main -query: - roots: - - . -auth: - env_file: ./.env.omni -", - yaml_string(url) - ) -} - #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_server_and_cli_end_to_end_flow() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let mutation_file = graph.write_query( - "system-remote-change.gq", + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + // The served graph's storage root — used for embedded-side cross checks. + let served_root = cluster.path().join("graphs").join(format!("{GRAPH_ID}.omni")); + let temp = tempfile::tempdir().unwrap(); + let mutation_file = temp.path().join("system-remote-change.gq"); + fs::write( + &mutation_file, r#" query insert_person($name: String, $age: I32) { insert Person { name: $name, age: $age } } "#, - ); + ) + .unwrap(); let client = Client::new(); let health = client @@ -116,13 +89,15 @@ query insert_person($name: String, $age: I32) { assert_eq!(health["status"], "ok"); let local_snapshot = parse_stdout_json(&output_success( - cli().arg("snapshot").arg(graph.path()).arg("--json"), + cli().arg("snapshot").arg(&served_root).arg("--json"), )); let snapshot = parse_stdout_json(&output_success( cli() .arg("snapshot") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--json"), )); assert_eq!(snapshot["branch"], "main"); @@ -131,10 +106,10 @@ query insert_person($name: String, $age: I32) { let local_read = parse_stdout_json(&output_success( cli() .arg("read") - .arg(graph.path()) + .arg("--store") + .arg(&served_root) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -143,11 +118,12 @@ query insert_person($name: String, $age: I32) { let read_payload = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -157,11 +133,15 @@ query insert_person($name: String, $age: I32) { assert_eq!(read_payload["row_count"], 1); assert_eq!(read_payload["rows"][0]["p.name"], "Alice"); + // Served write: no `--as` (the server resolves the actor; here the server + // is `--unauthenticated`, so the actor is the server default). let change_payload = parse_stdout_json(&output_success( cli() .arg("change") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&mutation_file) .arg("--params") @@ -172,7 +152,7 @@ query insert_person($name: String, $age: I32) { let query_source = fs::read_to_string(fixture("test.gq")).unwrap(); let http_read = client - .post(format!("{}/read", server.base_url)) + .post(format!("{}/graphs/{GRAPH_ID}/read", server.base_url)) .json(&json!({ "branch": "main", "query_source": query_source, @@ -191,10 +171,10 @@ query insert_person($name: String, $age: I32) { let local_verify = parse_stdout_json(&output_success( cli() .arg("read") - .arg(graph.path()) + .arg("--store") + .arg(&served_root) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Mina"}"#) @@ -203,15 +183,16 @@ query insert_person($name: String, $age: I32) { assert_eq!(local_verify["row_count"], 1); assert_eq!(local_verify["rows"][0]["p.name"], "Mina"); - // CLI `-e` over the HTTP transport (--config points at remote server). - // Confirms inline source survives the remote-execution path identically - // to file-based queries, and exercises `POST /query` end-to-end via the - // change-then-read round trip we just established. + // CLI inline source over the HTTP transport (--server). Confirms inline + // source survives the remote-execution path identically to file-based + // queries. let inline_remote_read = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("-e") .arg("query find($name: String) { match { $p: Person { name: $name } } return { $p.name, $p.age } }") .arg("--params") @@ -224,8 +205,10 @@ query insert_person($name: String, $age: I32) { let inline_remote_change = parse_stdout_json(&output_success( cli() .arg("change") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query-string") .arg("query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }") .arg("--params") @@ -234,10 +217,9 @@ query insert_person($name: String, $age: I32) { )); assert_eq!(inline_remote_change["affected_nodes"], 1); - // `POST /query` happy path directly: a hand-rolled HTTP body using the - // new clean field names. + // `POST /graphs/{id}/query` happy path directly. let http_query = client - .post(format!("{}/query", server.base_url)) + .post(format!("{}/graphs/{GRAPH_ID}/query", server.base_url)) .json(&json!({ "branch": "main", "query": "query find($name: String) { match { $p: Person { name: $name } } return { $p.name } }", @@ -252,9 +234,9 @@ query insert_person($name: String, $age: I32) { assert_eq!(http_query["row_count"], 1); assert_eq!(http_query["rows"][0]["p.name"], "Inline"); - // `POST /query` rejects mutations with 400. + // `POST /graphs/{id}/query` rejects mutations with 400. let http_query_mutation = client - .post(format!("{}/query", server.base_url)) + .post(format!("{}/graphs/{GRAPH_ID}/query", server.base_url)) .json(&json!({ "branch": "main", "query": "query bad($name: String, $age: I32) { insert Person { name: $name, age: $age } }", @@ -263,32 +245,33 @@ query insert_person($name: String, $age: I32) { .send() .unwrap(); assert_eq!(http_query_mutation.status(), reqwest::StatusCode::BAD_REQUEST); - - // `run publish` / `run list` removed. Direct-to-target writes - // already landed via the change call above; the commit graph is now - // the audit surface (verified separately by `commit list`). } #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_schema_apply_via_cli_updates_graph() { - let graph = SystemGraph::initialized(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let next_schema = graph.write_file( - "next.pg", - &fs::read_to_string(fixture("test.pg")).unwrap().replace( + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let served_root = cluster.path().join("graphs").join(format!("{GRAPH_ID}.omni")); + let temp = tempfile::tempdir().unwrap(); + let next_schema = temp.path().join("next.pg"); + fs::write( + &next_schema, + fs::read_to_string(fixture("test.pg")).unwrap().replace( " age: I32?\n}", " age: I32?\n nickname: String?\n}", ), - ); + ) + .unwrap(); let payload = parse_stdout_json(&output_success( cli() .arg("schema") .arg("apply") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--schema") .arg(&next_schema) .arg("--json"), @@ -297,7 +280,7 @@ fn remote_schema_apply_via_cli_updates_graph() { let db = tokio::runtime::Runtime::new() .unwrap() - .block_on(Omnigraph::open(graph.path().to_string_lossy().as_ref())) + .block_on(Omnigraph::open(served_root.to_string_lossy().as_ref())) .unwrap(); assert!( db.catalog().node_types["Person"] @@ -309,74 +292,95 @@ fn remote_schema_apply_via_cli_updates_graph() { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_schema_apply_rejects_unsupported_plan() { - let graph = SystemGraph::initialized(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let breaking_schema = graph.write_file( - "breaking.pg", - &fs::read_to_string(fixture("test.pg")) + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let temp = tempfile::tempdir().unwrap(); + let breaking_schema = temp.path().join("breaking.pg"); + fs::write( + &breaking_schema, + fs::read_to_string(fixture("test.pg")) .unwrap() .replace("age: I32?", "age: I64?"), - ); + ) + .unwrap(); let output = output_failure( cli() .arg("schema") .arg("apply") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--schema") .arg(&breaking_schema), ); let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stderr.contains("changing property type")); + assert!( + stderr.contains("changing property type"), + "expected unsupported-plan error, got: {stderr}" + ); } #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_schema_apply_rejects_when_non_main_branch_exists() { - let graph = SystemGraph::initialized(); + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + + // Create a non-main branch over the served path so the schema-apply + // single-branch precondition fails. output_success( cli() .arg("branch") .arg("create") + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--from") .arg("main") - .arg("--uri") - .arg(graph.path()) .arg("feature"), ); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let next_schema = graph.write_file( - "next.pg", - &fs::read_to_string(fixture("test.pg")).unwrap().replace( + + let temp = tempfile::tempdir().unwrap(); + let next_schema = temp.path().join("next.pg"); + fs::write( + &next_schema, + fs::read_to_string(fixture("test.pg")).unwrap().replace( " age: I32?\n}", " age: I32?\n nickname: String?\n}", ), - ); + ) + .unwrap(); let output = output_failure( cli() .arg("schema") .arg("apply") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--schema") .arg(&next_schema), ); let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stderr.contains("schema apply requires a graph with only main")); + assert!( + stderr.contains("schema apply requires a graph with only main"), + "expected single-branch precondition error, got: {stderr}" + ); } #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_read_preserves_projection_order_in_json_and_csv() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let ordered_query = graph.write_query( - "ordered-remote.gq", + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let temp = tempfile::tempdir().unwrap(); + let ordered_query = temp.path().join("ordered-remote.gq"); + fs::write( + &ordered_query, r#" query ordered_person($name: String) { match { @@ -385,16 +389,18 @@ query ordered_person($name: String) { return { $p.age, $p.name } } "#, - ); + ) + .unwrap(); let json_payload = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&ordered_query) - .arg("--name") .arg("ordered_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -411,11 +417,12 @@ query ordered_person($name: String) { let csv = stdout_string(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&ordered_query) - .arg("--name") .arg("ordered_person") .arg("--params") .arg(r#"{"name":"Alice"}"#) @@ -430,24 +437,28 @@ query ordered_person($name: String) { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_branch_create_list_merge_flow() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let mutation_file = graph.write_query( - "system-remote-branch-change.gq", + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let temp = tempfile::tempdir().unwrap(); + let mutation_file = temp.path().join("system-remote-branch-change.gq"); + fs::write( + &mutation_file, r#" query insert_person($name: String, $age: I32) { insert Person { name: $name, age: $age } } "#, - ); + ) + .unwrap(); let initial = parse_stdout_json(&output_success( cli() .arg("branch") .arg("list") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--json"), )); assert_eq!(initial["branches"], json!(["main"])); @@ -456,8 +467,10 @@ query insert_person($name: String, $age: I32) { cli() .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--from") .arg("main") .arg("feature") @@ -470,8 +483,10 @@ query insert_person($name: String, $age: I32) { cli() .arg("branch") .arg("list") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--json"), )); assert_eq!(listed["branches"], json!(["feature", "main"])); @@ -479,8 +494,10 @@ query insert_person($name: String, $age: I32) { let changed = parse_stdout_json(&output_success( cli() .arg("change") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&mutation_file) .arg("--branch") @@ -496,8 +513,10 @@ query insert_person($name: String, $age: I32) { cli() .arg("branch") .arg("merge") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("feature") .arg("--into") .arg("main") @@ -510,11 +529,12 @@ query insert_person($name: String, $age: I32) { let verify = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Zoe"}"#) @@ -527,16 +547,17 @@ query insert_person($name: String, $age: I32) { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_branch_delete_removes_branch() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); parse_stdout_json(&output_success( cli() .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--from") .arg("main") .arg("feature") @@ -547,9 +568,13 @@ fn remote_branch_delete_removes_branch() { cli() .arg("branch") .arg("delete") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("feature") + // Served target is non-local → destructive-confirm gate (RFC-011 D9). + .arg("--yes") .arg("--json"), )); assert_eq!(deleted["name"], "feature"); @@ -558,8 +583,10 @@ fn remote_branch_delete_removes_branch() { cli() .arg("branch") .arg("list") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--json"), )); assert_eq!(listed["branches"], json!(["main"])); @@ -568,11 +595,12 @@ fn remote_branch_delete_removes_branch() { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_export_round_trips_full_branch_graph() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let mutation_file = graph.write_query( - "system-remote-export-change.gq", + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let temp = tempfile::tempdir().unwrap(); + let mutation_file = temp.path().join("system-remote-export-change.gq"); + fs::write( + &mutation_file, r#" query insert_person($name: String, $age: I32) { insert Person { name: $name, age: $age } @@ -582,14 +610,17 @@ query add_friend($from: String, $to: String) { insert Knows { from: $from, to: $to } } "#, - ); + ) + .unwrap(); output_success( cli() .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--from") .arg("main") .arg("feature"), @@ -598,11 +629,12 @@ query add_friend($from: String, $to: String) { output_success( cli() .arg("change") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&mutation_file) - .arg("--name") .arg("insert_person") .arg("--branch") .arg("feature") @@ -613,11 +645,12 @@ query add_friend($from: String, $to: String) { output_success( cli() .arg("change") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&mutation_file) - .arg("--name") .arg("add_friend") .arg("--branch") .arg("feature") @@ -629,18 +662,17 @@ query add_friend($from: String, $to: String) { let exported = stdout_string(&output_success( cli() .arg("export") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--branch") .arg("feature") .arg("--jsonl"), )); - let export_path = graph.write_jsonl("system-remote-exported.jsonl", &exported); - let imported_graph = graph - .path() - .parent() - .unwrap() - .join("imported-remote-export.omni"); + let export_path = temp.path().join("system-remote-exported.jsonl"); + fs::write(&export_path, &exported).unwrap(); + let imported_graph = temp.path().join("imported-remote-export.omni"); output_success( cli() @@ -684,10 +716,10 @@ query add_friend($from: String, $to: String) { let eve = parse_stdout_json(&output_success( cli() .arg("read") + .arg("--store") .arg(&imported_graph) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"Eve"}"#) @@ -700,20 +732,24 @@ query add_friend($from: String, $to: String) { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_ingest_creates_review_branch_and_keeps_it_readable() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let ingest_data = graph.write_jsonl( - "system-remote-ingest.jsonl", + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let temp = tempfile::tempdir().unwrap(); + let ingest_data = temp.path().join("system-remote-ingest.jsonl"); + fs::write( + &ingest_data, r#"{"type":"Person","data":{"name":"Zoe","age":33}} {"type":"Person","data":{"name":"Bob","age":26}}"#, - ); + ) + .unwrap(); let ingest_payload = parse_stdout_json(&output_success( cli() .arg("ingest") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--data") .arg(&ingest_data) .arg("--branch") @@ -730,8 +766,10 @@ fn remote_ingest_creates_review_branch_and_keeps_it_readable() { let feature_snapshot = parse_stdout_json(&output_success( cli() .arg("snapshot") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--branch") .arg("feature-ingest") .arg("--json"), @@ -741,11 +779,12 @@ fn remote_ingest_creates_review_branch_and_keeps_it_readable() { let zoe = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--branch") .arg("feature-ingest") @@ -763,20 +802,24 @@ fn remote_ingest_creates_review_branch_and_keeps_it_readable() { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_load_round_trips_and_requires_from_for_new_branches() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); - let extra = graph.write_jsonl( - "system-remote-load.jsonl", + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); + let temp = tempfile::tempdir().unwrap(); + let extra = temp.path().join("system-remote-load.jsonl"); + fs::write( + &extra, r#"{"type":"Person","data":{"name":"Zoe","age":33}}"#, - ); + ) + .unwrap(); // Missing branch without --from: refused remotely, nothing created. let failure = output_failure( cli() .arg("load") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--mode") .arg("merge") .arg("--data") @@ -793,8 +836,10 @@ fn remote_load_round_trips_and_requires_from_for_new_branches() { let payload = parse_stdout_json(&output_success( cli() .arg("load") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--mode") .arg("merge") .arg("--data") @@ -813,8 +858,10 @@ fn remote_load_round_trips_and_requires_from_for_new_branches() { let snapshot = parse_stdout_json(&output_success( cli() .arg("snapshot") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--branch") .arg("feature-load") .arg("--json"), @@ -825,32 +872,38 @@ fn remote_load_round_trips_and_requires_from_for_new_branches() { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_ingest_reuses_existing_branch_and_merges_updates() { - let graph = SystemGraph::loaded(); - let server = graph.spawn_server(); - let config = graph.write_config("omnigraph.yaml", &remote_yaml_config(&server.base_url)); + let cluster = converged_loaded_cluster(GRAPH_ID, None); + let server = spawn_server_with_cluster(cluster.path()); output_success( cli() .arg("branch") .arg("create") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--from") .arg("main") .arg("feature-ingest"), ); - let ingest_data = graph.write_jsonl( - "system-remote-ingest-merge.jsonl", + let temp = tempfile::tempdir().unwrap(); + let ingest_data = temp.path().join("system-remote-ingest-merge.jsonl"); + fs::write( + &ingest_data, r#"{"type":"Person","data":{"name":"Bob","age":26}} {"type":"Person","data":{"name":"Zoe","age":33}}"#, - ); + ) + .unwrap(); let ingest_payload = parse_stdout_json(&output_success( cli() .arg("ingest") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--data") .arg(&ingest_data) .arg("--branch") @@ -869,11 +922,12 @@ fn remote_ingest_reuses_existing_branch_and_merges_updates() { let bob = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--branch") .arg("feature-ingest") @@ -887,11 +941,12 @@ fn remote_ingest_reuses_existing_branch_and_merges_updates() { let zoe = parse_stdout_json(&output_success( cli() .arg("read") - .arg("--config") - .arg(&config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--branch") .arg("feature-ingest") @@ -906,45 +961,51 @@ fn remote_ingest_reuses_existing_branch_and_merges_updates() { #[test] #[ignore = "requires loopback socket permissions in sandboxed runners"] fn remote_policy_enforces_branch_first_cli_workflow() { - let graph = SystemGraph::loaded(); - let server_config = - graph.write_config("server-policy.yaml", &remote_policy_server_config(&graph)); - graph.write_config("policy.yaml", REMOTE_POLICY_E2E_YAML); - let server = graph.spawn_server_with_config_env( - &server_config, + // Served policy enforcement: the cluster binds REMOTE_POLICY_E2E_YAML to the + // graph, and the server maps bearer tokens to actors. The actor is resolved + // from the token (no `--as` on served writes). + let cluster = converged_loaded_cluster(GRAPH_ID, Some(REMOTE_POLICY_E2E_YAML)); + let server = spawn_server_with_cluster_env( + cluster.path(), &[( "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", r#"{"act-bruno":"team-token","act-ragnor":"admin-token"}"#, )], ); - let client_config = graph.write_config( - "omnigraph-policy.yaml", - &remote_policy_client_config(&server.base_url), - ); - graph.write_config(".env.omni", "POLICY_TEST_TOKEN=team-token\n"); - let mutation_file = graph.write_query( - "system-remote-policy-change.gq", + let temp = tempfile::tempdir().unwrap(); + let mutation_file = temp.path().join("system-remote-policy-change.gq"); + fs::write( + &mutation_file, r#" query insert_person($name: String, $age: I32) { insert Person { name: $name, age: $age } } "#, - ); + ) + .unwrap(); + // Reads are granted to the team group (bruno). let snapshot = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "team-token") .arg("snapshot") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--json"), )); assert_eq!(snapshot["branch"], "main"); + // bruno cannot change protected main (team-write-unprotected only). let denied_main_change = output_failure( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "team-token") .arg("change") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&mutation_file) .arg("--params") @@ -952,14 +1013,23 @@ query insert_person($name: String, $age: I32) { .arg("--json"), ); let denied_main_stderr = String::from_utf8(denied_main_change.stderr).unwrap(); - assert!(denied_main_stderr.contains("policy denied action 'change' on branch 'main'")); + assert!( + denied_main_stderr.contains("denied") + && denied_main_stderr.contains("change") + && denied_main_stderr.contains("main"), + "expected change-on-main denial, got: {denied_main_stderr}" + ); + // bruno can create an unprotected branch. let created = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "team-token") .arg("branch") .arg("create") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--from") .arg("main") .arg("feature") @@ -967,11 +1037,15 @@ query insert_person($name: String, $age: I32) { )); assert_eq!(created["name"], "feature"); + // bruno can change the unprotected branch; actor resolves from the token. let changed = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "team-token") .arg("change") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(&mutation_file) .arg("--branch") @@ -982,28 +1056,39 @@ query insert_person($name: String, $age: I32) { )); assert_eq!(changed["branch"], "feature"); assert_eq!(changed["affected_nodes"], 1); + assert_eq!(changed["actor_id"], "act-bruno"); + // bruno cannot merge into protected main (admins-promote only). let denied_merge = output_failure( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "team-token") .arg("branch") .arg("merge") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("feature") .arg("--into") .arg("main") .arg("--json"), ); let denied_merge_stderr = String::from_utf8(denied_merge.stderr).unwrap(); - assert!(denied_merge_stderr.contains("policy denied action 'branch_merge'")); + assert!( + denied_merge_stderr.contains("denied") && denied_merge_stderr.contains("branch_merge"), + "expected branch_merge denial, got: {denied_merge_stderr}" + ); + // ragnor (admins) can promote into protected main. let merged = parse_stdout_json(&output_success( cli() - .env("POLICY_TEST_TOKEN", "admin-token") + .env("OMNIGRAPH_BEARER_TOKEN", "admin-token") .arg("branch") .arg("merge") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("feature") .arg("--into") .arg("main") @@ -1013,12 +1098,14 @@ query insert_person($name: String, $age: I32) { let verify = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "team-token") .arg("read") - .arg("--config") - .arg(&client_config) + .arg("--server") + .arg(&server.base_url) + .arg("--graph") + .arg(GRAPH_ID) .arg("--query") .arg(fixture("test.gq")) - .arg("--name") .arg("get_person") .arg("--params") .arg(r#"{"name":"PolicyRemote"}"#) @@ -1030,13 +1117,16 @@ query insert_person($name: String, $age: I32) { // ─── MR-668 PR 8 — omnigraph graphs list end-to-end ──────────────────────── -/// Multi-graph server + CLI `omnigraph graphs list` end-to-end. +/// Multi-graph server + CLI `omnigraph graphs list` end-to-end (RFC-011 +/// cluster-only serving). /// /// Steps: -/// 1. Init a graph `alpha` on disk and write an `omnigraph.yaml` -/// whose `graphs:` map references it. -/// 2. Spawn the server with `--config `. -/// 3. `omnigraph graphs list` — expect to see `alpha`. +/// 1. Build a converged cluster serving one graph `alpha` with a +/// server-scoped policy granting `act-admin` the `graph_list` action. +/// 2. Spawn the server with `--cluster` + a bearer-token map. +/// 3. `omnigraph graphs list --server ` (admin token) — expect `alpha`. +/// 4. Addressing the server via `--server ` with NO `--graph` errors and +/// lists the candidate graphs (RFC-011 D7). /// /// Ignored by default — spawning servers needs loopback socket /// permissions some sandboxes lack. @@ -1044,86 +1134,33 @@ query insert_person($name: String, $age: I32) { #[ignore = "requires loopback socket permissions in sandboxed runners"] fn graphs_list_against_multi_graph_server() { let cfg_dir = tempfile::tempdir().unwrap(); - let schema_path = fixture("test.pg"); - - // Init `alpha` on disk. - let alpha_uri = cfg_dir.path().join("alpha.omni"); - tokio::runtime::Runtime::new().unwrap().block_on(async { - Omnigraph::init( - alpha_uri.to_str().unwrap(), - &fs::read_to_string(&schema_path).unwrap(), - ) - .await - .unwrap(); - }); - + let dir = cfg_dir.path(); + fs::copy(fixture("test.pg"), dir.join("alpha.pg")).unwrap(); + fs::write(dir.join("server.policy.yaml"), GRAPH_LIST_SERVER_POLICY_YAML).unwrap(); fs::write( - cfg_dir.path().join("server-policy.yaml"), - GRAPH_LIST_SERVER_POLICY_YAML, + dir.join("cluster.yaml"), + "version: 1\nmetadata:\n name: sys\nstate:\n backend: cluster\n lock: true\ngraphs:\n alpha:\n schema: ./alpha.pg\npolicies:\n server:\n file: ./server.policy.yaml\n applies_to: [cluster]\n", ) .unwrap(); + output_success(cli().arg("cluster").arg("import").arg("--config").arg(dir)); + output_success(cli().arg("cluster").arg("apply").arg("--config").arg(dir)); - // Server config with `graphs:` map and no `server.graph` selector - // — multi mode (rule 4 of the inference matrix). `GET /graphs` is a - // server-scoped action, so the success path needs an explicit server - // policy and bearer token. - let server_config_path = cfg_dir.path().join("omnigraph.yaml"); - fs::write( - &server_config_path, - format!( - "\ -server: - policy: - file: ./server-policy.yaml -graphs: - alpha: - uri: {} -", - yaml_string(&alpha_uri.to_string_lossy()) - ), - ) - .unwrap(); - - let server = spawn_server_with_config_env( - &server_config_path, + let server = spawn_server_with_cluster_env( + dir, &[( "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", r#"{"act-admin":"admin-token"}"#, )], ); - // Client config — the CLI's `--target dev` resolves to `server.base_url`. - let client_config_path = cfg_dir.path().join("client.yaml"); - fs::write( - &client_config_path, - format!( - "\ -graphs: - dev: - uri: {} - bearer_token_env: GRAPH_LIST_TOKEN -cli: - graph: dev -auth: - env_file: ./.env.omni -", - yaml_string(&server.base_url) - ), - ) - .unwrap(); - fs::write( - cfg_dir.path().join(".env.omni"), - "GRAPH_LIST_TOKEN=admin-token\n", - ) - .unwrap(); - // `graphs list` lists `alpha`. let payload = parse_stdout_json(&output_success( cli() + .env("OMNIGRAPH_BEARER_TOKEN", "admin-token") .arg("graphs") .arg("list") - .arg("--config") - .arg(&client_config_path) + .arg("--server") + .arg(&server.base_url) .arg("--json"), )); let ids: Vec<&str> = payload["graphs"] @@ -1134,5 +1171,27 @@ auth: .collect(); assert_eq!(ids, vec!["alpha"]); + // RFC-011 D7: addressing the multi-graph server via `--server ` with no + // `--graph` errors and lists the candidate graphs (the resolver probes + // GET /graphs; the default-env token authorizes it). + let no_graph = cli() + .env("OMNIGRAPH_BEARER_TOKEN", "admin-token") + .arg("query") + .arg("--server") + .arg(&server.base_url) + .arg("-e") + .arg("query q { match { $p: Person { name: \"x\" } } return { $p.name } }") + .output() + .unwrap(); + assert!( + !no_graph.status.success(), + "multi-graph server with no --graph must error" + ); + let stderr = String::from_utf8_lossy(&no_graph.stderr); + assert!( + stderr.contains("alpha") && stderr.contains("--graph "), + "expected a candidate-listing error naming alpha; got: {stderr}" + ); + drop(server); } diff --git a/crates/omnigraph-cluster/src/config.rs b/crates/omnigraph-cluster/src/config.rs index acc954d..d0e0edd 100644 --- a/crates/omnigraph-cluster/src/config.rs +++ b/crates/omnigraph-cluster/src/config.rs @@ -42,7 +42,12 @@ pub(crate) fn resolve_query_decls( return ( map.iter() .map(|(name, config)| { - (name.clone(), QueryConfig { file: config.file.clone() }) + ( + name.clone(), + QueryConfig { + file: config.file.clone(), + }, + ) }) .collect(), BTreeMap::new(), @@ -66,7 +71,10 @@ pub(crate) fn resolve_query_decls( diagnostics.push(Diagnostic::error( "query_dir_unreadable", format!("graphs.{graph_id}.queries"), - format!("could not list query directory '{}': {err}", resolved.display()), + format!( + "could not list query directory '{}': {err}", + resolved.display() + ), )); continue; } @@ -76,7 +84,10 @@ pub(crate) fn resolve_query_decls( diagnostics.push(Diagnostic::warning( "query_dir_empty", format!("graphs.{graph_id}.queries"), - format!("query directory '{}' contains no .gq files", resolved.display()), + format!( + "query directory '{}' contains no .gq files", + resolved.display() + ), )); } for path in entries { @@ -132,7 +143,12 @@ pub(crate) fn resolve_query_decls( continue; } origin.insert(name.clone(), declared.clone()); - registry.insert(name, QueryConfig { file: declared.clone() }); + registry.insert( + name, + QueryConfig { + file: declared.clone(), + }, + ); } contents.insert(declared, source); } @@ -269,8 +285,6 @@ pub(crate) fn validate_cluster_header( } } - - pub(crate) fn state_resource_digests(state: &ClusterState) -> BTreeMap { state .applied_revision @@ -295,7 +309,6 @@ pub(crate) fn initial_import_state(desired: &DesiredCluster) -> ClusterState { } } - pub(crate) async fn observe_declared_graphs( desired: &DesiredCluster, backend: &ClusterStore, @@ -350,19 +363,28 @@ pub(crate) async fn observe_declared_graphs( StateResource { digest: observation.schema_digest.clone(), applies_to: None, + embedding_provider: None, + embedding_profile: None, }, ); let query_digests = state_query_digests_for_graph(state, &graph.id); + let embedding_provider = state_graph_embedding_provider(state, &graph.id); + let embedding_provider_digest = + state_embedding_provider_digest(state, embedding_provider.as_deref()); let graph_digest_value = graph_digest( &graph.id, Some(&observation.schema_digest), Some(&query_digests), + embedding_provider.as_deref(), + embedding_provider_digest.as_ref(), ); state.applied_revision.resources.insert( graph_address.clone(), StateResource { digest: graph_digest_value, applies_to: None, + embedding_provider, + embedding_profile: None, }, ); state.observations.insert( @@ -499,7 +521,6 @@ pub(crate) fn graph_observation_json(observation: GraphObservationJson<'_>) -> s }) } - pub(crate) fn load_desired(config_dir: &Path) -> LoadOutcome { let parsed = parse_cluster_config(config_dir); let config_dir = parsed.config_dir; @@ -519,6 +540,35 @@ pub(crate) fn load_desired(config_dir: &Path) -> LoadOutcome { let mut dependencies = BTreeSet::new(); let mut graph_query_digests: BTreeMap> = BTreeMap::new(); let mut graph_schema_digests: BTreeMap = BTreeMap::new(); + let mut graph_embedding_providers: BTreeMap = BTreeMap::new(); + let mut embedding_provider_digests: BTreeMap = BTreeMap::new(); + let mut embedding_providers: BTreeMap = BTreeMap::new(); + + for (provider_name, profile) in &raw.providers.embedding { + validate_id( + "embedding provider name", + &format!("providers.embedding.{provider_name}"), + provider_name, + &mut diagnostics, + ); + let address = embedding_provider_address(provider_name); + profile.validate( + format!("providers.embedding.{provider_name}"), + &mut diagnostics, + ); + let digest = embedding_provider_digest(profile); + embedding_provider_digests.insert(address.clone(), digest.clone()); + embedding_providers.insert(address.clone(), profile.clone()); + resources.insert( + address.clone(), + ResourceSummary { + address, + kind: "embedding_provider".to_string(), + digest, + path: None, + }, + ); + } for (graph_id, graph) in &raw.graphs { validate_id( @@ -533,6 +583,35 @@ pub(crate) fn load_desired(config_dir: &Path) -> LoadOutcome { from: schema_address.clone(), to: graph_address.clone(), }); + if let Some(provider_ref) = graph.embedding_provider.as_deref() { + match normalize_embedding_provider_target(provider_ref) { + EmbeddingProviderTarget::Provider(provider_name) => { + let provider_address = embedding_provider_address(&provider_name); + if raw.providers.embedding.contains_key(&provider_name) { + dependencies.insert(Dependency { + from: graph_address.clone(), + to: provider_address.clone(), + }); + graph_embedding_providers.insert(graph_id.clone(), provider_address); + } else { + diagnostics.push(Diagnostic::error( + "dangling_embedding_provider_reference", + format!("graphs.{graph_id}.embedding_provider"), + format!( + "graph references embedding provider `{provider_name}`, but no providers.embedding.{provider_name} profile is declared" + ), + )); + } + } + EmbeddingProviderTarget::WrongKind(kind) => diagnostics.push(Diagnostic::error( + "wrong_kind_reference", + format!("graphs.{graph_id}.embedding_provider"), + format!( + "embedding_provider expects a providers.embedding ref or bare provider name, got `{kind}`" + ), + )), + } + } let schema_path = resolve_config_path(&config_dir, &graph.schema); let schema_source = match fs::read_to_string(&schema_path) { @@ -646,10 +725,15 @@ pub(crate) fn load_desired(config_dir: &Path) -> LoadOutcome { } for graph_id in raw.graphs.keys() { + let embedding_provider = graph_embedding_providers.get(graph_id); + let embedding_provider_digest = + embedding_provider.and_then(|address| embedding_provider_digests.get(address)); let digest = graph_digest( graph_id, graph_schema_digests.get(graph_id), graph_query_digests.get(graph_id), + embedding_provider.map(String::as_str), + embedding_provider_digest, ); resources.insert( graph_address(graph_id), @@ -754,6 +838,7 @@ pub(crate) fn load_desired(config_dir: &Path) -> LoadOutcome { .get(graph_id) .cloned() .unwrap_or_default(), + embedding_provider: graph_embedding_providers.get(graph_id).cloned(), }) .collect(); let config_digest = desired_config_digest(&raw, &resource_digests); @@ -769,6 +854,7 @@ pub(crate) fn load_desired(config_dir: &Path) -> LoadOutcome { resources: resource_list, dependencies, policy_bindings, + embedding_providers, }), diagnostics, config_dir, @@ -828,7 +914,6 @@ pub(crate) fn future_field_diagnostics(text: &str) -> Vec { let future_fields = [ "apply", "env_file", - "providers", "pipelines", "embeddings", "ui", @@ -882,6 +967,21 @@ pub(crate) fn normalize_policy_target(value: &str) -> PolicyTarget { } } +enum EmbeddingProviderTarget { + Provider(String), + WrongKind(String), +} + +fn normalize_embedding_provider_target(value: &str) -> EmbeddingProviderTarget { + if let Some(name) = value.strip_prefix("provider.embedding.") { + EmbeddingProviderTarget::Provider(name.to_string()) + } else if value.contains('.') { + EmbeddingProviderTarget::WrongKind(value.to_string()) + } else { + EmbeddingProviderTarget::Provider(value.to_string()) + } +} + pub(crate) fn graph_address(graph_id: &str) -> String { format!("graph.{graph_id}") } @@ -898,6 +998,10 @@ pub(crate) fn policy_address(policy_name: &str) -> String { format!("policy.{policy_name}") } +pub(crate) fn embedding_provider_address(provider_name: &str) -> String { + format!("provider.embedding.{provider_name}") +} + pub(crate) fn resolve_config_path(config_dir: &Path, path: &Path) -> PathBuf { if path.is_absolute() { path.to_path_buf() diff --git a/crates/omnigraph-cluster/src/diff.rs b/crates/omnigraph-cluster/src/diff.rs index 593b2fa..516a86e 100644 --- a/crates/omnigraph-cluster/src/diff.rs +++ b/crates/omnigraph-cluster/src/diff.rs @@ -152,7 +152,9 @@ pub(crate) fn approved_resources( let candidates: Vec<&ApprovalArtifact> = artifacts .iter() .map(|(_, artifact)| artifact) - .filter(|artifact| artifact.consumed_at.is_none() && artifact.resource == change.resource) + .filter(|artifact| { + artifact.consumed_at.is_none() && artifact.resource == change.resource + }) .collect(); if candidates.is_empty() { continue; @@ -181,6 +183,7 @@ pub(crate) enum ResourceKind { Schema(String), Query { graph: String, name: String }, Policy(String), + EmbeddingProvider(String), Unknown, } @@ -199,6 +202,8 @@ pub(crate) fn resource_kind(address: &str) -> ResourceKind { } } else if let Some(name) = address.strip_prefix("policy.") { ResourceKind::Policy(name.to_string()) + } else if let Some(name) = address.strip_prefix("provider.embedding.") { + ResourceKind::EmbeddingProvider(name.to_string()) } else { ResourceKind::Unknown } @@ -261,8 +266,7 @@ pub(crate) fn classify_changes( let (disposition, reason) = match resource_kind(&change.resource) { ResourceKind::Schema(graph) => match change.operation { PlanOperation::Create - if graph_creates.contains(&graph) - && !pending_recovery.contains(&graph) => + if graph_creates.contains(&graph) && !pending_recovery.contains(&graph) => { // Applied with the graph create — the init carries it. (ApplyDisposition::Applied, None) @@ -325,10 +329,7 @@ pub(crate) fn classify_changes( if pending_recovery.contains(&graph) { (ApplyDisposition::Blocked, Some("cluster_recovery_pending")) } else if schema_pending.contains(&graph) { - ( - ApplyDisposition::Blocked, - Some("dependency_not_applied"), - ) + (ApplyDisposition::Blocked, Some("dependency_not_applied")) } else { // A graph create in the same plan no longer blocks: // creates execute first in the same apply run. @@ -353,9 +354,8 @@ pub(crate) fn classify_changes( } } }, - ResourceKind::Unknown => { - (ApplyDisposition::Deferred, Some("apply_unsupported_kind")) - } + ResourceKind::EmbeddingProvider(_) => (ApplyDisposition::Applied, None), + ResourceKind::Unknown => (ApplyDisposition::Deferred, Some("apply_unsupported_kind")), }; change.disposition = Some(disposition); change.reason = reason.map(str::to_string); diff --git a/crates/omnigraph-cluster/src/lib.rs b/crates/omnigraph-cluster/src/lib.rs index 1422dad..1c4e4fc 100644 --- a/crates/omnigraph-cluster/src/lib.rs +++ b/crates/omnigraph-cluster/src/lib.rs @@ -20,18 +20,35 @@ use ulid::Ulid; pub mod failpoints; mod config; -mod types; mod diff; mod serve; -mod sweep; mod store; +mod sweep; +mod types; +use config::{ + QueriesDecl, future_field_diagnostics, graph_address, initial_import_state, load_desired, + normalize_policy_target, observe_declared_graphs, observe_live_graph, parse_cluster_config, + policy_address, preview_schema_migration, query_address, resolve_config_path, + resolve_query_decls, schema_address, state_resource_digests, validate_cluster_header, + validate_id, validate_query_source, +}; +use diff::{ + FailedGraphOrigin, ResourceKind, append_policy_binding_changes, approved_resources, + classify_changes, compute_approvals, compute_blast_radius, demote_dependents_of_failed_graphs, + diff_resources, resource_kind, +}; +pub use serve::{ + ServingGraph, ServingPolicy, ServingQuery, ServingSnapshot, cluster_graph_ids, + cluster_root_for_graph_uri, read_serving_snapshot, read_serving_snapshot_from_storage, + resolve_graph_storage_uri, +}; use store::{ClusterStore, StateLockGuard, StateSnapshot}; +use sweep::{ + mark_approvals_consumed, record_approval_consumed, sweep_recovery_sidecars, + tombstone_graph_subtree, warn_pending_recovery_sidecars, +}; pub use types::*; use types::*; -pub use serve::{ServingGraph, ServingPolicy, ServingQuery, ServingSnapshot, read_serving_snapshot, read_serving_snapshot_from_storage}; -use config::{QueriesDecl, observe_declared_graphs, validate_cluster_header, future_field_diagnostics, initial_import_state, observe_live_graph, preview_schema_migration, state_resource_digests, graph_address, policy_address, query_address, schema_address, load_desired, normalize_policy_target, parse_cluster_config, resolve_config_path, resolve_query_decls, validate_id, validate_query_source}; -use diff::{FailedGraphOrigin, ResourceKind, append_policy_binding_changes, approved_resources, classify_changes, compute_approvals, compute_blast_radius, demote_dependents_of_failed_graphs, diff_resources, resource_kind}; -use sweep::{mark_approvals_consumed, record_approval_consumed, sweep_recovery_sidecars, tombstone_graph_subtree, warn_pending_recovery_sidecars}; pub const CLUSTER_CONFIG_FILE: &str = "cluster.yaml"; pub const CLUSTER_GRAPHS_DIR: &str = "graphs"; @@ -44,10 +61,7 @@ pub const CLUSTER_APPROVALS_DIR: &str = "__cluster/approvals"; /// The store for a load outcome: the declared `storage:` root when present, /// the config directory itself otherwise. A bad root is a loud error. -fn store_for( - config_dir: &Path, - storage_root: Option<&str>, -) -> Result { +fn store_for(config_dir: &Path, storage_root: Option<&str>) -> Result { match storage_root { Some(root) => ClusterStore::for_storage_root(root), None => Ok(ClusterStore::for_config_dir(config_dir)), @@ -179,7 +193,12 @@ pub async fn plan_config_dir(config_dir: impl AsRef) -> PlanOutput { &desired.config_digest, &mut diagnostics, ); - classify_changes(&mut changes, &desired.dependencies, &BTreeSet::new(), &approved); + classify_changes( + &mut changes, + &desired.dependencies, + &BTreeSet::new(), + &approved, + ); // Embed real migration steps for schema updates so plan is a data-aware // preview; failures degrade to the digest diff with a warning. @@ -282,9 +301,7 @@ pub async fn apply_config_dir_with_options( ok: !has_errors(&diagnostics), config_dir, actor: actor_for_output.clone(), - desired_revision: DesiredRevision { - config_digest, - }, + desired_revision: DesiredRevision { config_digest }, state_observations: observations, changes, applied_count: 0, @@ -464,8 +481,7 @@ pub async fn apply_config_dir_with_options( failed_graphs.insert(graph_id.clone(), FailedGraphOrigin::GraphCreate); continue; } - let Some(desired_graph) = desired.graphs.iter().find(|graph| &graph.id == graph_id) - else { + let Some(desired_graph) = desired.graphs.iter().find(|graph| &graph.id == graph_id) else { continue; }; let graph_uri = backend.graph_root(graph_id); @@ -604,8 +620,7 @@ pub async fn apply_config_dir_with_options( failed_graphs.insert(graph_id.clone(), FailedGraphOrigin::SchemaApply); continue; } - let Some(desired_graph) = desired.graphs.iter().find(|graph| &graph.id == graph_id) - else { + let Some(desired_graph) = desired.graphs.iter().find(|graph| &graph.id == graph_id) else { continue; }; let graph_uri = backend.graph_root(graph_id); @@ -955,8 +970,10 @@ pub async fn apply_config_dir_with_options( .expect("create/update always carries an after digest"), // Policies record their applied bindings so the // ledger is serving-sufficient (RFC-005 §D3). - applies_to: desired - .policy_bindings + applies_to: desired.policy_bindings.get(&change.resource).cloned(), + embedding_provider: None, + embedding_profile: desired + .embedding_providers .get(&change.resource) .cloned(), }, @@ -964,7 +981,10 @@ pub async fn apply_config_dir_with_options( set_resource_status_applied(&mut new_state, &change.resource); } PlanOperation::Delete => { - new_state.applied_revision.resources.remove(&change.resource); + new_state + .applied_revision + .resources + .remove(&change.resource); new_state.resource_statuses.remove(&change.resource); } }, @@ -1219,7 +1239,6 @@ pub async fn approve_config_dir( } } - pub async fn status_config_dir(config_dir: impl AsRef) -> StatusOutput { let parsed = parse_cluster_config(config_dir.as_ref()); let mut diagnostics = parsed.diagnostics; @@ -1238,7 +1257,9 @@ pub async fn status_config_dir(config_dir: impl AsRef) -> StatusOutput { } }; let mut observations = backend.observations(); - backend.observe_lock(&mut observations, &mut diagnostics).await; + backend + .observe_lock(&mut observations, &mut diagnostics) + .await; warn_pending_recovery_sidecars(&parsed.config_dir, &mut diagnostics); let mut resource_digests = BTreeMap::new(); @@ -1254,9 +1275,7 @@ pub async fn status_config_dir(config_dir: impl AsRef) -> StatusOutput { // Read-only point-in-time catalog check: report the // findings as diagnostics; persisting Drifted statuses // is refresh's job. Status never writes state. - for (address, finding) in - verify_catalog_payloads(&backend, &state).await - { + for (address, finding) in verify_catalog_payloads(&backend, &state).await { diagnostics.push(payload_finding_diagnostic(&address, &finding)); } resource_digests = state_resource_digests(&state); @@ -1312,7 +1331,10 @@ pub async fn force_unlock_config_dir( if let Some(raw) = parsed.raw.as_ref() { let _settings = validate_cluster_header(raw, &mut diagnostics); if !has_errors(&diagnostics) { - match backend.force_unlock(lock_id.as_ref(), &mut observations).await { + match backend + .force_unlock(lock_id.as_ref(), &mut observations) + .await + { Ok(()) => lock_removed = true, Err(diagnostic) => diagnostics.push(diagnostic), } @@ -1380,7 +1402,10 @@ async fn sync_config_dir(config_dir: &Path, operation: StateSyncOperation) -> St let operation_label = state_sync_operation_label(operation); let _lock_guard = if desired.state_lock { - match backend.acquire_lock(operation_label, &mut observations).await { + match backend + .acquire_lock(operation_label, &mut observations) + .await + { Ok(guard) => Some(guard), Err(diagnostic) => { diagnostics.push(diagnostic); @@ -1542,7 +1567,10 @@ async fn sync_config_dir(config_dir: &Path, operation: StateSyncOperation) -> St state.state_revision = state.state_revision.saturating_add(1); } - match backend.write_state(&state, expected_cas.as_deref(), &mut observations).await { + match backend + .write_state(&state, expected_cas.as_deref(), &mut observations) + .await + { Ok(()) => { // Completed sweep sidecars are deleted only after their outcome // is durably recorded; on failure they stay and re-sweep. @@ -1569,9 +1597,6 @@ async fn sync_config_dir(config_dir: &Path, operation: StateSyncOperation) -> St } } - - - #[derive(Debug, PartialEq, Eq)] enum PayloadFinding { Missing, @@ -1650,7 +1675,10 @@ async fn write_resource_payload( Diagnostic::error( "resource_payload_write_error", resource, - format!("could not read resource source '{}': {err}", source.display()), + format!( + "could not read resource source '{}': {err}", + source.display() + ), ) })?; if sha256_hex(&bytes) != expected_digest { @@ -1692,7 +1720,11 @@ async fn write_resource_payload( fn recompute_state_graph_digests(state: &mut ClusterState, desired: &DesiredCluster) { for graph in &desired.graphs { let graph_address = graph_address(&graph.id); - if !state.applied_revision.resources.contains_key(&graph_address) { + if !state + .applied_revision + .resources + .contains_key(&graph_address) + { continue; } let schema_digest = state @@ -1701,11 +1733,26 @@ fn recompute_state_graph_digests(state: &mut ClusterState, desired: &DesiredClus .get(&schema_address(&graph.id)) .map(|resource| resource.digest.clone()); let query_digests = state_query_digests_for_graph(state, &graph.id); - let digest = graph_digest(&graph.id, schema_digest.as_ref(), Some(&query_digests)); - state - .applied_revision - .resources - .insert(graph_address, StateResource { digest, applies_to: None }); + let embedding_provider = graph.embedding_provider.as_deref(); + let embedding_provider_digest = embedding_provider + .and_then(|address| state.applied_revision.resources.get(address)) + .map(|resource| resource.digest.clone()); + let digest = graph_digest( + &graph.id, + schema_digest.as_ref(), + Some(&query_digests), + embedding_provider, + embedding_provider_digest.as_ref(), + ); + state.applied_revision.resources.insert( + graph_address, + StateResource { + digest, + applies_to: None, + embedding_provider: graph.embedding_provider.clone(), + embedding_profile: None, + }, + ); } } @@ -1773,7 +1820,6 @@ fn duplicate_key_diagnostics(text: &str) -> Vec { diagnostics } - fn strip_comment(line: &str) -> String { let mut in_single_quote = false; let mut in_double_quote = false; @@ -1796,7 +1842,6 @@ fn strip_comment(line: &str) -> String { line.to_string() } - fn state_query_digests_for_graph(state: &ClusterState, graph_id: &str) -> BTreeMap { let prefix = format!("query.{graph_id}."); state @@ -1811,6 +1856,23 @@ fn state_query_digests_for_graph(state: &ClusterState, graph_id: &str) -> BTreeM .collect() } +fn state_graph_embedding_provider(state: &ClusterState, graph_id: &str) -> Option { + state + .applied_revision + .resources + .get(&graph_address(graph_id)) + .and_then(|resource| resource.embedding_provider.clone()) +} + +fn state_embedding_provider_digest( + state: &ClusterState, + embedding_provider: Option<&str>, +) -> Option { + embedding_provider + .and_then(|address| state.applied_revision.resources.get(address)) + .map(|resource| resource.digest.clone()) +} + fn set_resource_status_applied(state: &mut ClusterState, address: &str) { state.resource_statuses.insert( address.to_string(), @@ -1843,6 +1905,8 @@ fn graph_digest( graph_id: &str, schema_digest: Option<&String>, query_digests: Option<&BTreeMap>, + embedding_provider: Option<&str>, + embedding_provider_digest: Option<&String>, ) -> String { let mut input = format!( "graph\0{graph_id}\0schema\0{}\0", @@ -1857,6 +1921,21 @@ fn graph_digest( input.push('\0'); } } + if let Some(provider) = embedding_provider { + input.push_str("embedding_provider\0"); + input.push_str(provider); + input.push('\0'); + input.push_str(embedding_provider_digest.map_or("", String::as_str)); + input.push('\0'); + } + sha256_hex(input.as_bytes()) +} + +fn embedding_provider_digest(profile: &EmbeddingProviderConfig) -> String { + let mut input = String::from("embedding-provider\0"); + let config_semantics = + serde_json::to_string(profile).expect("embedding provider config must serialize"); + input.push_str(&config_semantics); sha256_hex(input.as_bytes()) } @@ -1930,7 +2009,6 @@ fn display_path(path: &Path) -> String { path.display().to_string() } - #[cfg(test)] #[path = "tests.rs"] mod tests; diff --git a/crates/omnigraph-cluster/src/serve.rs b/crates/omnigraph-cluster/src/serve.rs index 4abd0bf..6f89e2d 100644 --- a/crates/omnigraph-cluster/src/serve.rs +++ b/crates/omnigraph-cluster/src/serve.rs @@ -8,6 +8,7 @@ use super::*; pub struct ServingGraph { pub graph_id: String, pub root: PathBuf, + pub embedding: Option, } /// One stored query: its graph binding, registry name, and verified source. @@ -79,6 +80,112 @@ pub async fn read_serving_snapshot_from_storage( read_snapshot_with_store(backend).await } +/// Cluster root for a graph **storage URI** of the cluster layout +/// (`/graphs/.omni`), if `` is actually a cluster (holds +/// `__cluster/state.json`); otherwise `None`. Used by the CLI to refuse +/// `init` into a cluster-managed location — graphs there are created by +/// `cluster apply`, not `init`. +/// +/// Cheap by construction: a URI that does not match the `/graphs/.omni` +/// shape returns `None` without any I/O, so ordinary `init` targets +/// (`./kb.omni`, `s3://bucket/kb.omni`) never probe storage. Works for +/// `file://` and `s3://` via the storage adapter. +pub async fn cluster_root_for_graph_uri(graph_uri: &str) -> Option { + let root = cluster_root_of_graph_layout(graph_uri)?; + let store = ClusterStore::for_storage_root(&root).ok()?; + store + .has_state() + .await + .then(|| store.display_root().to_string()) +} + +/// Resolve a graph's **storage URI** (`/graphs/.omni`) from a cluster's +/// applied state ledger — the lightweight path for storage-plane maintenance +/// (`optimize`/`repair`/`cleanup`). +/// +/// Unlike [`read_serving_snapshot`], this deliberately does NOT validate catalog +/// payloads or recovery readiness: maintenance only needs the derivable graph +/// root, and must not be blocked by an unrelated corrupt policy/query blob or a +/// pending recovery sweep — a degraded cluster is exactly when an operator +/// reaches for `repair`. It reads the state ledger, confirms the graph is in the +/// applied revision, and returns `graph_root(id)`. +/// +/// `cluster` is a config directory or a storage-root URI (`s3://…`, config-free), +/// mirroring the server's `--cluster` dispatch. +pub async fn resolve_graph_storage_uri(cluster: &str, graph_id: &str) -> Result { + let backend = open_cluster_backend(cluster)?; + let mut observations = backend.observations(); + let snapshot = backend.read_state(&mut observations).await?; + let state = snapshot.state.ok_or_else(|| missing_state_diagnostic(cluster))?; + let address = format!("graph.{graph_id}"); + if !state.applied_revision.resources.contains_key(&address) { + let applied = applied_graph_ids(&state); + return Err(Diagnostic::error( + "graph_not_applied", + address, + format!( + "graph `{graph_id}` is not applied in cluster `{cluster}` (applied graphs: [{}]); \ + declare it in cluster.yaml and run `cluster apply`, or check the id", + applied.join(", ") + ), + )); + } + Ok(backend.graph_root(graph_id)) +} + +/// List the graph ids applied in a cluster's served state (sorted). Reads the +/// ledger only — no catalog validation — like `resolve_graph_storage_uri`, so +/// it works on a degraded cluster. Used to enumerate candidates when no +/// `--graph` is selected (RFC-011 Decision 7). +pub async fn cluster_graph_ids(cluster: &str) -> Result, Diagnostic> { + let backend = open_cluster_backend(cluster)?; + let mut observations = backend.observations(); + let snapshot = backend.read_state(&mut observations).await?; + let state = snapshot.state.ok_or_else(|| missing_state_diagnostic(cluster))?; + Ok(applied_graph_ids(&state)) +} + +fn open_cluster_backend(cluster: &str) -> Result { + if cluster.contains("://") { + ClusterStore::for_storage_root(cluster) + } else { + Ok(ClusterStore::for_config_dir(Path::new(cluster))) + } +} + +fn missing_state_diagnostic(cluster: &str) -> Diagnostic { + Diagnostic::error( + "cluster_state_missing", + CLUSTER_STATE_FILE, + format!("cluster `{cluster}` has no applied state; run `cluster apply` first"), + ) +} + +fn applied_graph_ids(state: &crate::types::ClusterState) -> Vec { + let mut ids: Vec = state + .applied_revision + .resources + .keys() + .filter_map(|a| a.strip_prefix("graph.")) + .map(str::to_string) + .collect(); + ids.sort(); + ids +} + +/// Split `/graphs/.omni` → ``, gating on the exact cluster +/// graph-layout shape (a single `` segment, no nested path). `None` for +/// anything else — no I/O is done for non-cluster-shaped URIs. +fn cluster_root_of_graph_layout(graph_uri: &str) -> Option { + let trimmed = graph_uri.trim_end_matches('/'); + let rest = trimmed.strip_suffix(".omni")?; + let (root, id) = rest.rsplit_once("/graphs/")?; + if root.is_empty() || id.is_empty() || id.contains('/') { + return None; + } + Some(root.to_string()) +} + async fn read_snapshot_with_store( backend: ClusterStore, ) -> Result> { @@ -119,15 +226,73 @@ async fn read_snapshot_with_store( return Err(diagnostics); }; + let mut embedding_profiles: BTreeMap = BTreeMap::new(); + for (address, entry) in &state.applied_revision.resources { + if !matches!(resource_kind(address), ResourceKind::EmbeddingProvider(_)) { + continue; + } + let Some(profile) = entry.embedding_profile.clone() else { + diagnostics.push(Diagnostic::error( + "embedding_provider_profile_missing", + address.clone(), + "no applied embedding provider profile recorded; re-run `cluster apply` to backfill", + )); + continue; + }; + let actual_digest = embedding_provider_digest(&profile); + if actual_digest != entry.digest { + diagnostics.push(Diagnostic::error( + "embedding_provider_digest_mismatch", + address.clone(), + format!( + "applied embedding provider profile does not match its recorded digest (actual sha256:{actual_digest}); run `cluster refresh` then `cluster apply`, and restart" + ), + )); + continue; + } + embedding_profiles.insert(address.clone(), profile); + } + let mut graphs = Vec::new(); let mut queries = Vec::new(); let mut policies = Vec::new(); for (address, entry) in &state.applied_revision.resources { match resource_kind(address) { ResourceKind::Graph(graph_id) => { + let embedding = match entry.embedding_provider.as_deref() { + Some(provider_address) => match resource_kind(provider_address) { + ResourceKind::EmbeddingProvider(_) => { + match embedding_profiles.get(provider_address) { + Some(profile) => Some(profile.clone()), + None => { + diagnostics.push(Diagnostic::error( + "embedding_provider_missing", + address.clone(), + format!( + "graph references `{provider_address}`, but no applied embedding provider profile is available; re-run `cluster apply`" + ), + )); + None + } + } + } + _ => { + diagnostics.push(Diagnostic::error( + "wrong_kind_reference", + address.clone(), + format!( + "graph embedding_provider expects `provider.embedding.`, got `{provider_address}`" + ), + )); + None + } + }, + None => None, + }; graphs.push(ServingGraph { root: PathBuf::from(backend.graph_root(&graph_id)), graph_id, + embedding, }); } ResourceKind::Schema(_) => {} @@ -135,7 +300,10 @@ async fn read_snapshot_with_store( let ResourceKind::Query { graph, name } = &kind else { unreachable!() }; - match backend.read_verified_payload(&kind, &entry.digest, address).await { + match backend + .read_verified_payload(&kind, &entry.digest, address) + .await + { Ok(source) => queries.push(ServingQuery { graph_id: graph.clone(), name: name.clone(), @@ -156,7 +324,10 @@ async fn read_snapshot_with_store( )); continue; }; - match backend.read_verified_payload(&kind, &entry.digest, address).await { + match backend + .read_verified_payload(&kind, &entry.digest, address) + .await + { Ok(source) => policies.push(ServingPolicy { name: name.clone(), source, @@ -165,6 +336,7 @@ async fn read_snapshot_with_store( Err(diagnostic) => diagnostics.push(diagnostic), } } + ResourceKind::EmbeddingProvider(_) => {} ResourceKind::Unknown => {} } } @@ -186,3 +358,49 @@ async fn read_snapshot_with_store( }) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn graph_layout_gating_does_no_io_for_non_cluster_shapes() { + // Only `/graphs/.omni` matches; everything else is None. + assert_eq!( + cluster_root_of_graph_layout("/data/cluster/graphs/kb.omni").as_deref(), + Some("/data/cluster") + ); + assert_eq!( + cluster_root_of_graph_layout("s3://bucket/prefix/graphs/kb.omni").as_deref(), + Some("s3://bucket/prefix") + ); + assert_eq!(cluster_root_of_graph_layout("./kb.omni"), None); + assert_eq!(cluster_root_of_graph_layout("s3://bucket/kb.omni"), None); + // nested id under graphs/ is not the cluster layout + assert_eq!(cluster_root_of_graph_layout("/c/graphs/a/b.omni"), None); + // not a .omni graph + assert_eq!(cluster_root_of_graph_layout("/c/graphs/kb"), None); + } + + #[tokio::test] + async fn cluster_root_detected_only_when_state_ledger_present() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + std::fs::create_dir_all(root.join("graphs")).unwrap(); + let graph_uri = format!("{}/graphs/kb.omni", root.to_string_lossy()); + + // No __cluster/state.json yet → not a cluster. + assert_eq!(cluster_root_for_graph_uri(&graph_uri).await, None); + + // Lay down the state ledger → now it's a cluster-managed location. + std::fs::create_dir_all(root.join("__cluster")).unwrap(); + std::fs::write(root.join(CLUSTER_STATE_FILE), "{}").unwrap(); + let detected = cluster_root_for_graph_uri(&graph_uri).await; + assert!(detected.is_some(), "expected cluster root to be detected"); + + // A non-cluster-shaped target never probes and is always None. + assert_eq!( + cluster_root_for_graph_uri(&format!("{}/plain.omni", root.to_string_lossy())).await, + None + ); + } +} diff --git a/crates/omnigraph-cluster/src/store.rs b/crates/omnigraph-cluster/src/store.rs index 620df96..5129397 100644 --- a/crates/omnigraph-cluster/src/store.rs +++ b/crates/omnigraph-cluster/src/store.rs @@ -154,6 +154,21 @@ impl ClusterStore { } } + /// Display-form storage root (plain local path for `file://`, URI for S3). + pub(crate) fn display_root(&self) -> &str { + &self.display_root + } + + /// Whether this root holds the cluster state ledger (`__cluster/state.json`) + /// — i.e. is an actual cluster, not just any directory. Probed via the + /// adapter (`file://` or `s3://`), failures read as "not a cluster". + pub(crate) async fn has_state(&self) -> bool { + self.adapter + .exists(&self.uri(CLUSTER_STATE_FILE)) + .await + .unwrap_or(false) + } + /// `read_text_versioned`, returning None for a missing object (probed /// via `exists` — the engine error type doesn't discriminate NotFound). async fn read_versioned_opt(&self, uri: &str) -> Result, String> { diff --git a/crates/omnigraph-cluster/src/sweep.rs b/crates/omnigraph-cluster/src/sweep.rs index 7aecb01..6539cae 100644 --- a/crates/omnigraph-cluster/src/sweep.rs +++ b/crates/omnigraph-cluster/src/sweep.rs @@ -19,13 +19,29 @@ pub(crate) async fn sweep_recovery_sidecars( for (path, sidecar) in backend.list_recovery_sidecars(diagnostics).await { match sidecar.kind { RecoverySidecarKind::GraphCreate => { - sweep_graph_create_sidecar(backend, path, sidecar, state, diagnostics, &mut outcome).await; + sweep_graph_create_sidecar( + backend, + path, + sidecar, + state, + diagnostics, + &mut outcome, + ) + .await; } RecoverySidecarKind::SchemaApply => { sweep_schema_apply_sidecar(path, sidecar, state, diagnostics, &mut outcome).await; } RecoverySidecarKind::GraphDelete => { - sweep_graph_delete_sidecar(backend, path, sidecar, state, diagnostics, &mut outcome).await; + sweep_graph_delete_sidecar( + backend, + path, + sidecar, + state, + diagnostics, + &mut outcome, + ) + .await; } } } @@ -71,15 +87,30 @@ pub(crate) async fn sweep_graph_create_sidecar( StateResource { digest: live_digest.clone(), applies_to: None, + embedding_provider: None, + embedding_profile: None, }, ); let query_digests = state_query_digests_for_graph(state, &sidecar.graph_id); - let composite = - graph_digest(&sidecar.graph_id, Some(&live_digest), Some(&query_digests)); - state - .applied_revision - .resources - .insert(graph_address.clone(), StateResource { digest: composite, applies_to: None }); + let embedding_provider = state_graph_embedding_provider(state, &sidecar.graph_id); + let embedding_provider_digest = + state_embedding_provider_digest(state, embedding_provider.as_deref()); + let composite = graph_digest( + &sidecar.graph_id, + Some(&live_digest), + Some(&query_digests), + embedding_provider.as_deref(), + embedding_provider_digest.as_ref(), + ); + state.applied_revision.resources.insert( + graph_address.clone(), + StateResource { + digest: composite, + applies_to: None, + embedding_provider, + embedding_profile: None, + }, + ); set_resource_status_applied(state, &graph_address); set_resource_status_applied(state, &schema_addr); state.recovery_records.insert( @@ -200,14 +231,30 @@ pub(crate) async fn sweep_schema_apply_sidecar( StateResource { digest: live_digest.clone(), applies_to: None, + embedding_provider: None, + embedding_profile: None, }, ); let query_digests = state_query_digests_for_graph(state, &sidecar.graph_id); - let composite = graph_digest(&sidecar.graph_id, Some(&live_digest), Some(&query_digests)); - state - .applied_revision - .resources - .insert(graph_address.clone(), StateResource { digest: composite, applies_to: None }); + let embedding_provider = state_graph_embedding_provider(state, &sidecar.graph_id); + let embedding_provider_digest = + state_embedding_provider_digest(state, embedding_provider.as_deref()); + let composite = graph_digest( + &sidecar.graph_id, + Some(&live_digest), + Some(&query_digests), + embedding_provider.as_deref(), + embedding_provider_digest.as_ref(), + ); + state.applied_revision.resources.insert( + graph_address.clone(), + StateResource { + digest: composite, + applies_to: None, + embedding_provider, + embedding_profile: None, + }, + ); set_resource_status_applied(state, &graph_address); set_resource_status_applied(state, &schema_addr); state.recovery_records.insert( @@ -274,7 +321,11 @@ pub(crate) async fn sweep_graph_delete_sidecar( return; } - if !state.applied_revision.resources.contains_key(&graph_address) { + if !state + .applied_revision + .resources + .contains_key(&graph_address) + { // Row 7: already tombstoned (or never recorded); crash fell between // the state CAS and sidecar delete. outcome.completed_sidecars.push(path); @@ -283,7 +334,12 @@ pub(crate) async fn sweep_graph_delete_sidecar( // Row 7b: the root is gone, the ledger is stale — roll forward the // tombstone, consume the approval the sidecar carries, audit. - tombstone_graph_subtree(state, &sidecar.graph_id, sidecar.approval_id.as_deref(), sidecar.actor.as_deref()); + tombstone_graph_subtree( + state, + &sidecar.graph_id, + sidecar.approval_id.as_deref(), + sidecar.actor.as_deref(), + ); state.recovery_records.insert( sidecar.operation_id.clone(), json!({ @@ -342,7 +398,11 @@ pub(crate) fn tombstone_graph_subtree( /// Record approval consumption in the state ledger. The artifact FILE is /// rewritten with consumed_at only after the state write lands, so a failed /// CAS leaves the approval valid for the retry. -pub(crate) fn record_approval_consumed(state: &mut ClusterState, approval_id: &str, operation_id: &str) { +pub(crate) fn record_approval_consumed( + state: &mut ClusterState, + approval_id: &str, + operation_id: &str, +) { state.approval_records.insert( approval_id.to_string(), json!({ diff --git a/crates/omnigraph-cluster/src/tests.rs b/crates/omnigraph-cluster/src/tests.rs index 805ecda..ac448cf 100644 --- a/crates/omnigraph-cluster/src/tests.rs +++ b/crates/omnigraph-cluster/src/tests.rs @@ -56,6 +56,39 @@ policies: dir } + fn write_mock_embedding_cluster(config_dir: &Path, model: &str) { + fs::write( + config_dir.join(CLUSTER_CONFIG_FILE), + format!( + r#" +version: 1 +metadata: + name: test +state: + backend: cluster + lock: true +providers: + embedding: + default: + kind: mock + model: {model} +graphs: + knowledge: + schema: ./people.pg + embedding_provider: default + queries: + find_person: + file: ./people.gq +policies: + base: + file: ./base.policy.yaml + applies_to: [knowledge] +"# + ), + ) + .unwrap(); + } + async fn init_derived_graph(root: &Path) { let graph_dir = root.join(CLUSTER_GRAPHS_DIR); fs::create_dir_all(&graph_dir).unwrap(); @@ -194,6 +227,95 @@ policies: assert!(codes.contains("dangling_graph_reference")); } + #[test] + fn embedding_provider_config_accepts_provider_resources_and_graph_refs() { + let dir = fixture(); + write_mock_embedding_cluster(dir.path(), "recorded-x"); + + let out = validate_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + let provider_digest = out + .resource_digests + .get("provider.embedding.default") + .expect("provider resource digest"); + assert!( + out.resources + .iter() + .any(|resource| resource.address == "provider.embedding.default" + && resource.kind == "embedding_provider" + && resource.path.is_none()) + ); + assert!( + out.dependencies + .iter() + .any(|dep| dep.from == "graph.knowledge" && dep.to == "provider.embedding.default"), + "{:?}", + out.dependencies + ); + let schema_digest = out.resource_digests.get("schema.knowledge").unwrap(); + let query_digest = out + .resource_digests + .get("query.knowledge.find_person") + .unwrap(); + let expected_graph_digest = graph_digest( + "knowledge", + Some(schema_digest), + Some( + &[("find_person".to_string(), query_digest.clone())] + .into_iter() + .collect(), + ), + Some("provider.embedding.default"), + Some(provider_digest), + ); + assert_eq!( + out.resource_digests["graph.knowledge"], + expected_graph_digest + ); + } + + #[test] + fn embedding_provider_config_rejects_bad_refs_and_inline_secrets() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +providers: + embedding: + default: + kind: openai-compatible + api_key: sk-inline +graphs: + knowledge: + schema: ./people.pg + embedding_provider: provider.policy.default + missing_provider: + schema: ./people.pg + embedding_provider: absent +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); + assert!( + codes.contains("embedding_api_key_inline"), + "{:?}", + out.diagnostics + ); + assert!( + codes.contains("wrong_kind_reference"), + "{:?}", + out.diagnostics + ); + assert!( + codes.contains("dangling_embedding_provider_reference"), + "{:?}", + out.diagnostics + ); + } + #[test] fn query_key_mismatch_fails() { let dir = fixture(); @@ -1012,8 +1134,13 @@ graphs: let out = validate_config_dir(config_dir); assert!(out.ok, "{:?}", out.diagnostics); let schema_digest = out.resource_digests.get("schema.knowledge").unwrap().clone(); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + let graph_composite = graph_digest( + "knowledge", + Some(&schema_digest), + Some(&BTreeMap::new()), + None, + None, + ); write_state_resources( config_dir, &[ @@ -1122,6 +1249,8 @@ graphs: .into_iter() .collect(), ), + None, + None, ); assert_eq!(resources["graph.knowledge"]["digest"], expected_composite); assert_eq!( @@ -1136,6 +1265,117 @@ graphs: assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); } + #[tokio::test] + async fn apply_records_embedding_provider_profile_and_graph_binding() { + let dir = fixture(); + write_mock_embedding_cluster(dir.path(), "recorded-x"); + write_applyable_state(dir.path()); + let desired = validate_config_dir(dir.path()); + let query_digest = desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap() + .clone(); + let schema_digest = desired + .resource_digests + .get("schema.knowledge") + .unwrap() + .clone(); + let provider_digest = desired + .resource_digests + .get("provider.embedding.default") + .unwrap() + .clone(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.converged, "{out:?}"); + + let state = read_state_json(dir.path()); + let resources = &state["applied_revision"]["resources"]; + let provider = resources["provider.embedding.default"] + .as_object() + .expect("provider resource"); + assert_eq!(provider["digest"], provider_digest); + assert_eq!(provider["embedding_profile"]["kind"], "mock"); + assert_eq!(provider["embedding_profile"]["model"], "recorded-x"); + assert!(provider["embedding_profile"].get("api_key").is_none()); + assert_eq!( + resources["graph.knowledge"]["embedding_provider"], + "provider.embedding.default" + ); + let expected_graph_digest = graph_digest( + "knowledge", + Some(&schema_digest), + Some( + &[("find_person".to_string(), query_digest)] + .into_iter() + .collect(), + ), + Some("provider.embedding.default"), + Some(&provider_digest), + ); + assert_eq!(resources["graph.knowledge"]["digest"], expected_graph_digest); + } + + #[tokio::test] + async fn embedding_provider_changes_update_provider_and_graph_plan() { + let dir = fixture(); + write_mock_embedding_cluster(dir.path(), "recorded-x"); + write_applyable_state(dir.path()); + let first = apply_config_dir(dir.path()).await; + assert!(first.ok && first.converged, "{first:?}"); + + write_mock_embedding_cluster(dir.path(), "recorded-y"); + let plan = plan_config_dir(dir.path()).await; + assert!(plan.ok, "{:?}", plan.diagnostics); + let by_resource: BTreeMap<&str, &PlanChange> = plan + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + assert_eq!( + by_resource["provider.embedding.default"].operation, + PlanOperation::Update + ); + assert_eq!( + by_resource["provider.embedding.default"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["graph.knowledge"].operation, + PlanOperation::Update + ); + assert_eq!( + by_resource["graph.knowledge"].disposition, + Some(ApplyDisposition::Derived) + ); + } + + #[tokio::test] + async fn embedding_binding_survives_refresh() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_mock_embedding_cluster(dir.path(), "recorded-x"); + write_applyable_state(dir.path()); + let apply = apply_config_dir(dir.path()).await; + assert!(apply.ok && apply.converged, "{apply:?}"); + + let refresh = refresh_config_dir(dir.path()).await; + assert!(refresh.ok, "{:?}", refresh.diagnostics); + + let state = read_state_json(dir.path()); + let resources = &state["applied_revision"]["resources"]; + assert_eq!( + resources["graph.knowledge"]["embedding_provider"], + "provider.embedding.default" + ); + assert_eq!( + resources["provider.embedding.default"]["embedding_profile"]["model"], + "recorded-x" + ); + } + fn desired_revision_digest(out: &ApplyOutput) -> String { out.desired_revision.config_digest.clone().unwrap() } @@ -1150,8 +1390,13 @@ graphs: .unwrap() .clone(); let old_digest = "0".repeat(64); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + let graph_composite = graph_digest( + "knowledge", + Some(&schema_digest), + Some(&BTreeMap::new()), + None, + None, + ); write_state_resources( dir.path(), &[ @@ -1190,8 +1435,13 @@ graphs: .clone(); let stale_query_digest = "1".repeat(64); let stale_policy_digest = "2".repeat(64); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + let graph_composite = graph_digest( + "knowledge", + Some(&schema_digest), + Some(&BTreeMap::new()), + None, + None, + ); write_state_resources( dir.path(), &[ @@ -1234,6 +1484,8 @@ graphs: "knowledge", Some(&schema_digest), Some(&[("find_person".to_string(), query_digest)].into_iter().collect()), + None, + None, ); assert_eq!(resources["graph.knowledge"]["digest"], expected_composite); } @@ -1494,8 +1746,13 @@ graphs: .get("schema.knowledge") .unwrap() .clone(); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + let graph_composite = graph_digest( + "knowledge", + Some(&schema_digest), + Some(&BTreeMap::new()), + None, + None, + ); write_state_resources( dir.path(), &[ @@ -2864,6 +3121,54 @@ policies: assert!(snapshot.policies[0].source.contains("rules:")); } + #[tokio::test] + async fn serving_snapshot_uses_applied_embedding_provider_profile() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_mock_embedding_cluster(dir.path(), "recorded-x"); + write_applyable_state(dir.path()); + let converge = apply_config_dir(dir.path()).await; + assert!(converge.converged, "{converge:?}"); + + let snapshot = read_serving_snapshot(dir.path()).await.unwrap(); + let profile = snapshot.graphs[0].embedding.as_ref().unwrap(); + assert_eq!(profile.kind.as_deref(), Some("mock")); + assert_eq!(profile.model.as_deref(), Some("recorded-x")); + } + + #[tokio::test] + async fn serving_snapshot_refuses_missing_embedding_provider_metadata() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_mock_embedding_cluster(dir.path(), "recorded-x"); + write_applyable_state(dir.path()); + let converge = apply_config_dir(dir.path()).await; + assert!(converge.converged, "{converge:?}"); + + let mut state = read_state_json(dir.path()); + state["applied_revision"]["resources"]["provider.embedding.default"] + .as_object_mut() + .unwrap() + .remove("embedding_profile"); + fs::write( + dir.path().join(CLUSTER_STATE_FILE), + serde_json::to_string_pretty(&state).unwrap(), + ) + .unwrap(); + + let err = read_serving_snapshot(dir.path()).await.unwrap_err(); + assert!( + err.iter() + .any(|diagnostic| diagnostic.code == "embedding_provider_profile_missing"), + "{err:?}" + ); + assert!( + err.iter() + .any(|diagnostic| diagnostic.code == "embedding_provider_missing"), + "{err:?}" + ); + } + #[tokio::test] async fn serving_snapshot_refuses_missing_state() { let dir = fixture(); diff --git a/crates/omnigraph-cluster/src/types.rs b/crates/omnigraph-cluster/src/types.rs index e44e2f4..97ad406 100644 --- a/crates/omnigraph-cluster/src/types.rs +++ b/crates/omnigraph-cluster/src/types.rs @@ -325,6 +325,7 @@ pub(crate) struct DesiredCluster { /// The declared `storage:` root, if any (None ⇒ the config dir itself). pub(crate) storage_root: Option, pub(crate) state_lock: bool, + pub(crate) embedding_providers: BTreeMap, pub(crate) graphs: Vec, pub(crate) resource_digests: BTreeMap, pub(crate) resources: Vec, @@ -337,6 +338,7 @@ pub(crate) struct DesiredCluster { pub(crate) struct DesiredGraph { pub(crate) id: String, pub(crate) schema_digest: String, + pub(crate) embedding_provider: Option, } #[derive(Debug)] @@ -376,6 +378,8 @@ pub(crate) struct RawClusterConfig { #[serde(default)] pub(crate) state: StateConfig, #[serde(default)] + pub(crate) providers: ProvidersConfig, + #[serde(default)] pub(crate) graphs: BTreeMap, #[serde(default)] pub(crate) policies: BTreeMap, @@ -394,12 +398,123 @@ pub(crate) struct StateConfig { pub(crate) lock: Option, } +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProvidersConfig { + #[serde(default)] + pub(crate) embedding: BTreeMap, +} + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct GraphConfig { pub(crate) schema: PathBuf, #[serde(default)] pub(crate) queries: QueriesDecl, + /// Optional reference to a top-level `providers.embedding.` profile. + #[serde(default)] + pub(crate) embedding_provider: Option, +} + +/// A named cluster embedding provider profile (RFC-012 Phase 5). `kind`/`base_url`/ +/// `model` default exactly as the engine's `EmbeddingConfig::from_env` does. +/// `api_key`, when required, must be a `${NAME}` env reference resolved at +/// serving boot, never an inline secret. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct EmbeddingProviderConfig { + #[serde(default, alias = "provider", skip_serializing_if = "Option::is_none")] + pub kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub api_key: Option, +} + +impl EmbeddingProviderConfig { + pub(crate) fn validate(&self, path: String, diagnostics: &mut Vec) { + if let Err(error) = omnigraph::embedding::EmbeddingConfig::from_parts( + self.kind.as_deref(), + self.base_url.clone(), + self.model.clone(), + "validation-placeholder".to_string(), + ) { + diagnostics.push(Diagnostic::error( + "invalid_embedding_provider", + path.clone(), + error.to_string(), + )); + } + + if self.kind.as_deref() == Some("mock") { + if let Some(api_key) = self.api_key.as_deref() { + if secret_ref_name(api_key).is_err() { + diagnostics.push(Diagnostic::error( + "embedding_api_key_inline", + format!("{path}.api_key"), + "embedding api_key must be a ${NAME} env reference, not an inline secret", + )); + } + } + return; + } + + match self.api_key.as_deref() { + Some(api_key) if secret_ref_name(api_key).is_err() => diagnostics.push( + Diagnostic::error( + "embedding_api_key_inline", + format!("{path}.api_key"), + "embedding api_key must be a ${NAME} env reference, not an inline secret", + ), + ), + Some(_) => {} + None => diagnostics.push(Diagnostic::error( + "embedding_api_key_required", + format!("{path}.api_key"), + "non-mock embedding providers must set api_key to a ${NAME} env reference", + )), + } + } + + /// Resolve into an engine `EmbeddingConfig`, reading the `${NAME}` api-key + /// reference from process env. Mock profiles do not read env and may omit + /// `api_key`; real providers error if the reference is missing or unset. + pub fn resolve(&self) -> Result { + let api_key = if self.kind.as_deref() == Some("mock") { + String::new() + } else { + resolve_secret_ref(self.api_key.as_deref().ok_or_else(|| { + "embedding api_key is required for non-mock providers".to_string() + })?)? + }; + omnigraph::embedding::EmbeddingConfig::from_parts( + self.kind.as_deref(), + self.base_url.clone(), + self.model.clone(), + api_key, + ) + .map_err(|e| e.to_string()) + } +} + +fn secret_ref_name(value: &str) -> Result<&str, String> { + value + .trim() + .strip_prefix("${") + .and_then(|s| s.strip_suffix('}')) + .filter(|name| !name.trim().is_empty()) + .ok_or_else(|| { + format!("embedding api_key must be a ${{NAME}} env reference, got '{}'", value.trim()) + }) +} + +/// Resolve a `${NAME}` secret reference from process env. Rejects an inline value +/// (anything not wrapped in `${…}`) so secrets never sit in the cluster config. +fn resolve_secret_ref(value: &str) -> Result { + let name = secret_ref_name(value)?; + std::env::var(name).map_err(|_| format!("embedding api_key env var '{name}' is not set")) } /// How a graph declares its stored queries. Terraform-style: the `.gq` @@ -457,6 +572,16 @@ pub(crate) struct StateResource { /// non-policy resources. #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) applies_to: Option>, + /// Graph resources only: the applied `provider.embedding.` binding. + /// The provider profile itself is stored on the provider resource so + /// serving can boot without re-reading mutable desired config. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) embedding_provider: Option, + /// Embedding provider resources only: the applied profile with unresolved + /// `${ENV}` references. The server resolves the referenced env var exactly + /// once at boot and injects the resulting engine config into the graph. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) embedding_profile: Option, } #[derive(Debug, Serialize, Deserialize)] @@ -518,3 +643,74 @@ pub(crate) struct SweepOutcome { /// files are rewritten with consumed_at only after the state write lands. pub(crate) consumed_approvals: Vec, } + +#[cfg(test)] +mod embedding_provider_config_tests { + use super::EmbeddingProviderConfig; + + #[test] + fn resolves_secret_from_env_and_applies_defaults() { + // SAFETY: a unique var name, no concurrent reader. + unsafe { std::env::set_var("OG_TEST_EMBED_KEY_A", "secret-x") }; + let profile = EmbeddingProviderConfig { + kind: Some("openai-compatible".to_string()), + base_url: None, + model: Some("m".to_string()), + api_key: Some("${OG_TEST_EMBED_KEY_A}".to_string()), + }; + let config = profile.resolve().unwrap(); + assert_eq!(config.api_key, "secret-x"); + assert_eq!(config.model, "m"); + unsafe { std::env::remove_var("OG_TEST_EMBED_KEY_A") }; + } + + #[test] + fn rejects_inline_api_key() { + let profile = EmbeddingProviderConfig { + kind: None, + base_url: None, + model: None, + api_key: Some("sk-inline".to_string()), + }; + let err = profile.resolve().unwrap_err(); + assert!(err.contains("${NAME}"), "got: {err}"); + } + + #[test] + fn errors_on_unset_secret() { + let profile = EmbeddingProviderConfig { + kind: None, + base_url: None, + model: None, + api_key: Some("${OG_TEST_DEFINITELY_UNSET_VAR}".to_string()), + }; + let err = profile.resolve().unwrap_err(); + assert!(err.contains("not set"), "got: {err}"); + } + + #[test] + fn rejects_unknown_provider() { + unsafe { std::env::set_var("OG_TEST_EMBED_KEY_B", "x") }; + let profile = EmbeddingProviderConfig { + kind: Some("cohere".to_string()), + base_url: None, + model: None, + api_key: Some("${OG_TEST_EMBED_KEY_B}".to_string()), + }; + let err = profile.resolve().unwrap_err(); + assert!(err.contains("unknown embedding provider"), "got: {err}"); + unsafe { std::env::remove_var("OG_TEST_EMBED_KEY_B") }; + } + + #[test] + fn mock_does_not_require_secret_env() { + let profile = EmbeddingProviderConfig { + kind: Some("mock".to_string()), + base_url: None, + model: Some("cluster-mock".to_string()), + api_key: None, + }; + let config = profile.resolve().unwrap(); + assert_eq!(config.model, "cluster-mock"); + } +} diff --git a/crates/omnigraph-compiler/Cargo.toml b/crates/omnigraph-compiler/Cargo.toml index bbf03f1..4645b81 100644 --- a/crates/omnigraph-compiler/Cargo.toml +++ b/crates/omnigraph-compiler/Cargo.toml @@ -20,10 +20,5 @@ pest_derive = { workspace = true } thiserror = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -reqwest = { workspace = true } ahash = { workspace = true } -tokio = { workspace = true } sha2 = { workspace = true } - -[dev-dependencies] -tokio = { workspace = true } diff --git a/crates/omnigraph-compiler/src/catalog/mod.rs b/crates/omnigraph-compiler/src/catalog/mod.rs index 0bb536d..93f8d89 100644 --- a/crates/omnigraph-compiler/src/catalog/mod.rs +++ b/crates/omnigraph-compiler/src/catalog/mod.rs @@ -26,6 +26,15 @@ pub struct InterfaceType { pub properties: HashMap, } +/// The `@embed` binding for a vector property: its source text property and, +/// optionally, the embedding model recorded by `@embed("source", model="…")`. +/// The model is what the query-time same-space check validates against. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmbedSource { + pub source: String, + pub model: Option, +} + #[derive(Debug, Clone)] pub struct NodeType { pub name: String, @@ -42,8 +51,8 @@ pub struct NodeType { pub range_constraints: Vec, /// Regex check constraints pub check_constraints: Vec, - /// Maps @embed target property -> source text property - pub embed_sources: HashMap, + /// Maps @embed target property -> its source text property + recorded model. + pub embed_sources: HashMap, pub blob_properties: HashSet, pub arrow_schema: SchemaRef, } @@ -156,14 +165,18 @@ pub fn build_catalog(schema: &SchemaFile) -> Result { if matches!(prop.prop_type.scalar, ScalarType::Blob) { blob_properties.insert(prop.name.clone()); } - // Extract @embed from property annotations (stays as annotation) - if let Some(source_prop) = prop - .annotations - .iter() - .find(|ann| ann.name == "embed") - .and_then(|ann| ann.value.clone()) - { - embed_sources.insert(prop.name.clone(), source_prop); + // Extract @embed: the source text property (positional) and the + // optional recorded model (the `model` kwarg). + if let Some(ann) = prop.annotations.iter().find(|ann| ann.name == "embed") { + if let Some(source) = ann.value.clone() { + embed_sources.insert( + prop.name.clone(), + EmbedSource { + source, + model: ann.kwargs.get("model").cloned(), + }, + ); + } } } diff --git a/crates/omnigraph-compiler/src/catalog/schema_plan.rs b/crates/omnigraph-compiler/src/catalog/schema_plan.rs index a9e26b2..dc9d466 100644 --- a/crates/omnigraph-compiler/src/catalog/schema_plan.rs +++ b/crates/omnigraph-compiler/src/catalog/schema_plan.rs @@ -1137,6 +1137,7 @@ node Person @description("new") { annotations: vec![Annotation { name: "description".to_string(), value: Some("new".to_string()), + kwargs: Default::default(), }], })); } diff --git a/crates/omnigraph-compiler/src/catalog/tests.rs b/crates/omnigraph-compiler/src/catalog/tests.rs index 883b4a9..4ab3956 100644 --- a/crates/omnigraph-compiler/src/catalog/tests.rs +++ b/crates/omnigraph-compiler/src/catalog/tests.rs @@ -31,6 +31,33 @@ fn test_build_catalog() { assert!(catalog.node_types.contains_key("Company")); } +#[test] +fn test_embed_source_records_model_kwarg() { + let schema = parse_schema( + r#" +node Doc { +title: String +embedding: Vector(3) @embed("title", model="openai/text-embedding-3-large") +plain: Vector(3) @embed("title") +} +"#, + ) + .unwrap(); + let catalog = build_catalog(&schema).unwrap(); + let doc = catalog.node_types.get("Doc").unwrap(); + + let embedding = doc.embed_sources.get("embedding").unwrap(); + assert_eq!(embedding.source, "title"); + assert_eq!( + embedding.model.as_deref(), + Some("openai/text-embedding-3-large") + ); + + let plain = doc.embed_sources.get("plain").unwrap(); + assert_eq!(plain.source, "title"); + assert_eq!(plain.model, None); +} + #[test] fn test_edge_lookup() { let schema = parse_schema(test_schema()).unwrap(); diff --git a/crates/omnigraph-compiler/src/embedding.rs b/crates/omnigraph-compiler/src/embedding.rs deleted file mode 100644 index 6c9e6f3..0000000 --- a/crates/omnigraph-compiler/src/embedding.rs +++ /dev/null @@ -1,379 +0,0 @@ -#![allow(dead_code)] - -use std::time::Duration; - -use reqwest::Client; -use serde::Deserialize; -use tokio::time::sleep; - -use crate::error::{NanoError, Result}; - -const DEFAULT_EMBED_MODEL: &str = "text-embedding-3-small"; -const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; -const DEFAULT_TIMEOUT_MS: u64 = 30_000; -const DEFAULT_RETRY_ATTEMPTS: usize = 4; -const DEFAULT_RETRY_BACKOFF_MS: u64 = 200; - -#[derive(Clone)] -enum EmbeddingTransport { - Mock, - OpenAi { - api_key: String, - base_url: String, - http: Client, - }, -} - -#[derive(Clone)] -pub(crate) struct EmbeddingClient { - model: String, - retry_attempts: usize, - retry_backoff_ms: u64, - transport: EmbeddingTransport, -} - -struct EmbedCallError { - message: String, - retryable: bool, -} - -#[derive(Debug, Deserialize)] -struct OpenAiEmbeddingResponse { - data: Vec, -} - -#[derive(Debug, Deserialize)] -struct OpenAiEmbeddingDatum { - index: usize, - embedding: Vec, -} - -#[derive(Debug, Deserialize)] -struct OpenAiErrorEnvelope { - error: OpenAiErrorBody, -} - -#[derive(Debug, Deserialize)] -struct OpenAiErrorBody { - message: String, -} - -impl EmbeddingClient { - pub(crate) fn from_env() -> Result { - let model = std::env::var("NANOGRAPH_EMBED_MODEL") - .ok() - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) - .unwrap_or_else(|| DEFAULT_EMBED_MODEL.to_string()); - let retry_attempts = - parse_env_usize("NANOGRAPH_EMBED_RETRY_ATTEMPTS", DEFAULT_RETRY_ATTEMPTS); - let retry_backoff_ms = - parse_env_u64("NANOGRAPH_EMBED_RETRY_BACKOFF_MS", DEFAULT_RETRY_BACKOFF_MS); - - if env_flag("NANOGRAPH_EMBEDDINGS_MOCK") { - return Ok(Self { - model, - retry_attempts, - retry_backoff_ms, - transport: EmbeddingTransport::Mock, - }); - } - - let api_key = std::env::var("OPENAI_API_KEY") - .ok() - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) - .ok_or_else(|| { - NanoError::Execution( - "OPENAI_API_KEY is required when an embedding call is needed".to_string(), - ) - })?; - let base_url = std::env::var("OPENAI_BASE_URL") - .ok() - .map(|v| v.trim_end_matches('/').to_string()) - .filter(|v| !v.is_empty()) - .unwrap_or_else(|| DEFAULT_OPENAI_BASE_URL.to_string()); - let timeout_ms = parse_env_u64("NANOGRAPH_EMBED_TIMEOUT_MS", DEFAULT_TIMEOUT_MS); - let http = Client::builder() - .timeout(Duration::from_millis(timeout_ms)) - .build() - .map_err(|e| { - NanoError::Execution(format!("failed to initialize HTTP client: {}", e)) - })?; - - Ok(Self { - model, - retry_attempts, - retry_backoff_ms, - transport: EmbeddingTransport::OpenAi { - api_key, - base_url, - http, - }, - }) - } - - #[cfg(test)] - pub(crate) fn mock_for_tests() -> Self { - Self { - model: DEFAULT_EMBED_MODEL.to_string(), - retry_attempts: DEFAULT_RETRY_ATTEMPTS, - retry_backoff_ms: DEFAULT_RETRY_BACKOFF_MS, - transport: EmbeddingTransport::Mock, - } - } - - pub(crate) fn model(&self) -> &str { - &self.model - } - - pub(crate) async fn embed_text(&self, input: &str, expected_dim: usize) -> Result> { - let mut vectors = self.embed_texts(&[input.to_string()], expected_dim).await?; - vectors.pop().ok_or_else(|| { - NanoError::Execution("embedding provider returned no vector".to_string()) - }) - } - - pub(crate) async fn embed_texts( - &self, - inputs: &[String], - expected_dim: usize, - ) -> Result>> { - if expected_dim == 0 { - return Err(NanoError::Execution( - "embedding dimension must be greater than zero".to_string(), - )); - } - if inputs.is_empty() { - return Ok(Vec::new()); - } - - match &self.transport { - EmbeddingTransport::Mock => Ok(inputs - .iter() - .map(|input| mock_embedding(input, expected_dim)) - .collect()), - EmbeddingTransport::OpenAi { .. } => { - self.embed_texts_openai_with_retry(inputs, expected_dim) - .await - } - } - } - - async fn embed_texts_openai_with_retry( - &self, - inputs: &[String], - expected_dim: usize, - ) -> Result>> { - let max_attempt = self.retry_attempts.max(1); - let mut attempt = 0usize; - loop { - attempt += 1; - match self.embed_texts_openai_once(inputs, expected_dim).await { - Ok(vectors) => return Ok(vectors), - Err(err) => { - if !err.retryable || attempt >= max_attempt { - return Err(NanoError::Execution(err.message)); - } - let shift = (attempt - 1).min(10) as u32; - let delay = self.retry_backoff_ms.saturating_mul(1u64 << shift); - sleep(Duration::from_millis(delay)).await; - } - } - } - } - - async fn embed_texts_openai_once( - &self, - inputs: &[String], - expected_dim: usize, - ) -> std::result::Result>, EmbedCallError> { - let (api_key, base_url, http) = match &self.transport { - EmbeddingTransport::OpenAi { - api_key, - base_url, - http, - } => (api_key, base_url, http), - EmbeddingTransport::Mock => unreachable!("mock transport should not call OpenAI"), - }; - - let request = serde_json::json!({ - "model": self.model, - "input": inputs, - "dimensions": expected_dim, - }); - let url = format!("{}/embeddings", base_url); - let response = http - .post(&url) - .bearer_auth(api_key) - .json(&request) - .send() - .await; - - let response = match response { - Ok(resp) => resp, - Err(err) => { - let retryable = err.is_timeout() || err.is_connect() || err.is_request(); - return Err(EmbedCallError { - message: format!("embedding request failed: {}", err), - retryable, - }); - } - }; - - let status = response.status(); - let body = match response.text().await { - Ok(body) => body, - Err(err) => { - return Err(EmbedCallError { - message: format!( - "embedding response read failed (status {}): {}", - status, err - ), - retryable: status.is_server_error() || status.as_u16() == 429, - }); - } - }; - - if !status.is_success() { - let message = parse_openai_error_message(&body).unwrap_or_else(|| body.clone()); - return Err(EmbedCallError { - message: format!( - "embedding request failed with status {}: {}", - status, message - ), - retryable: status.is_server_error() || status.as_u16() == 429, - }); - } - - let mut parsed: OpenAiEmbeddingResponse = - serde_json::from_str(&body).map_err(|err| EmbedCallError { - message: format!("embedding response decode failed: {}", err), - retryable: false, - })?; - - if parsed.data.len() != inputs.len() { - return Err(EmbedCallError { - message: format!( - "embedding response size mismatch: expected {}, got {}", - inputs.len(), - parsed.data.len() - ), - retryable: false, - }); - } - - parsed.data.sort_by_key(|item| item.index); - let mut vectors = Vec::with_capacity(parsed.data.len()); - for (idx, item) in parsed.data.into_iter().enumerate() { - if item.index != idx { - return Err(EmbedCallError { - message: format!( - "embedding response index mismatch at position {}: got {}", - idx, item.index - ), - retryable: false, - }); - } - if item.embedding.len() != expected_dim { - return Err(EmbedCallError { - message: format!( - "embedding dimension mismatch: expected {}, got {}", - expected_dim, - item.embedding.len() - ), - retryable: false, - }); - } - vectors.push(item.embedding); - } - Ok(vectors) - } -} - -fn parse_openai_error_message(body: &str) -> Option { - serde_json::from_str::(body) - .ok() - .map(|e| e.error.message) - .filter(|msg| !msg.trim().is_empty()) -} - -fn parse_env_usize(name: &str, default: usize) -> usize { - std::env::var(name) - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0) - .unwrap_or(default) -} - -fn parse_env_u64(name: &str, default: u64) -> u64 { - std::env::var(name) - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0) - .unwrap_or(default) -} - -fn env_flag(name: &str) -> bool { - std::env::var(name) - .ok() - .map(|v| { - let s = v.trim().to_ascii_lowercase(); - s == "1" || s == "true" || s == "yes" || s == "on" - }) - .unwrap_or(false) -} - -fn mock_embedding(input: &str, dim: usize) -> Vec { - let mut seed = fnv1a64(input.as_bytes()); - let mut out = Vec::with_capacity(dim); - for _ in 0..dim { - seed = xorshift64(seed); - let ratio = (seed as f64 / u64::MAX as f64) as f32; - out.push((ratio * 2.0) - 1.0); - } - - let norm = out - .iter() - .map(|v| (*v as f64) * (*v as f64)) - .sum::() - .sqrt() as f32; - if norm > f32::EPSILON { - for value in &mut out { - *value /= norm; - } - } - out -} - -fn fnv1a64(bytes: &[u8]) -> u64 { - let mut hash = 14695981039346656037u64; - for byte in bytes { - hash ^= *byte as u64; - hash = hash.wrapping_mul(1099511628211u64); - } - hash -} - -fn xorshift64(mut x: u64) -> u64 { - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - x -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn mock_embeddings_are_deterministic() { - let client = EmbeddingClient::mock_for_tests(); - let a = client.embed_text("alpha", 8).await.unwrap(); - let b = client.embed_text("alpha", 8).await.unwrap(); - let c = client.embed_text("beta", 8).await.unwrap(); - assert_eq!(a, b); - assert_ne!(a, c); - assert_eq!(a.len(), 8); - } -} diff --git a/crates/omnigraph-compiler/src/lib.rs b/crates/omnigraph-compiler/src/lib.rs index ba1aba2..4f85c08 100644 --- a/crates/omnigraph-compiler/src/lib.rs +++ b/crates/omnigraph-compiler/src/lib.rs @@ -1,5 +1,4 @@ pub mod catalog; -pub mod embedding; pub mod error; pub mod ir; pub mod json_output; diff --git a/crates/omnigraph-compiler/src/query/typecheck.rs b/crates/omnigraph-compiler/src/query/typecheck.rs index 658f083..b2c235a 100644 --- a/crates/omnigraph-compiler/src/query/typecheck.rs +++ b/crates/omnigraph-compiler/src/query/typecheck.rs @@ -261,13 +261,13 @@ fn typecheck_mutation(catalog: &Catalog, mutation: &Mutation, params: &[Param]) continue; } - if let Some(source_prop) = node_type.embed_sources.get(prop_name) { - if assigned_props.contains(source_prop.as_str()) { + if let Some(embed) = node_type.embed_sources.get(prop_name) { + if assigned_props.contains(embed.source.as_str()) { continue; } return Err(NanoError::Type(format!( "T12: insert for `{}` must provide non-nullable property `{}` or @embed source `{}`", - insert.type_name, prop_name, source_prop + insert.type_name, prop_name, embed.source ))); } diff --git a/crates/omnigraph-compiler/src/schema/ast.rs b/crates/omnigraph-compiler/src/schema/ast.rs index f8ed18a..9be0e56 100644 --- a/crates/omnigraph-compiler/src/schema/ast.rs +++ b/crates/omnigraph-compiler/src/schema/ast.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use crate::types::PropType; use serde::{Deserialize, Serialize}; @@ -50,6 +52,11 @@ pub struct PropDecl { pub struct Annotation { pub name: String, pub value: Option, + /// Keyword arguments, e.g. `model="…"` on `@embed("source", model="…")`. + /// Empty is skipped in serialization so existing schemas' IR JSON (and + /// hash) stay byte-identical; `BTreeMap` keeps the order deterministic. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub kwargs: BTreeMap, } /// A typed constraint declared in a node or edge body. diff --git a/crates/omnigraph-compiler/src/schema/parser.rs b/crates/omnigraph-compiler/src/schema/parser.rs index 43e11ed..c5f4355 100644 --- a/crates/omnigraph-compiler/src/schema/parser.rs +++ b/crates/omnigraph-compiler/src/schema/parser.rs @@ -556,12 +556,32 @@ fn parse_type_ref(pair: pest::iterators::Pair) -> Result { fn parse_annotation(pair: pest::iterators::Pair) -> Result { let mut inner = pair.into_inner(); let name = inner.next().unwrap().as_str().to_string(); - let value = inner - .next() - .map(|p| decode_string_literal(p.as_str())) - .transpose()?; + let mut value = None; + let mut kwargs = std::collections::BTreeMap::new(); + if let Some(args) = inner.next() { + // `annotation_args`: one positional arg followed by zero or more + // `key = literal` kwargs (e.g. `@embed("source", model="…")`). + for arg in args.into_inner() { + match arg.as_rule() { + Rule::annotation_arg => { + value = Some(decode_string_literal(arg.as_str())?); + } + Rule::annotation_kwarg => { + let mut kw = arg.into_inner(); + let key = kw.next().unwrap().as_str().to_string(); + let raw = kw.next().unwrap().as_str(); + kwargs.insert(key, decode_string_literal(raw)?); + } + _ => {} + } + } + } - Ok(Annotation { name, value }) + Ok(Annotation { + name, + value, + kwargs, + }) } fn validate_string_annotation( @@ -823,6 +843,17 @@ fn validate_property_annotations( type_name, source_prop ))); } + + // `model` is the only supported kwarg; reject the rest loudly so + // a typo can't be silently ignored (it would never validate). + for key in ann.kwargs.keys() { + if key != "model" { + return Err(NanoError::Parse(format!( + "@embed on {}.{} has unknown argument '{}=' (only 'model' is supported)", + type_name, prop.name, key + ))); + } + } } _ => {} } diff --git a/crates/omnigraph-compiler/src/schema/parser_tests.rs b/crates/omnigraph-compiler/src/schema/parser_tests.rs index 2302cfb..9a2e1ba 100644 --- a/crates/omnigraph-compiler/src/schema/parser_tests.rs +++ b/crates/omnigraph-compiler/src/schema/parser_tests.rs @@ -508,6 +508,66 @@ embedding: Vector(3) @embed(title) } } +#[test] +fn test_parse_embed_annotation_with_model_kwarg() { + let input = r#" +node Doc { +title: String +embedding: Vector(3) @embed("title", model="openai/text-embedding-3-large") +} +"#; + let schema = parse_schema(input).unwrap(); + match &schema.declarations[0] { + SchemaDecl::Node(n) => { + let ann = &n.properties[1].annotations[0]; + assert_eq!(ann.name, "embed"); + assert_eq!(ann.value.as_deref(), Some("title")); + assert_eq!( + ann.kwargs.get("model").map(String::as_str), + Some("openai/text-embedding-3-large") + ); + } + _ => panic!("expected Node"), + } +} + +#[test] +fn test_parse_embed_annotation_without_model_has_empty_kwargs() { + let input = r#" +node Doc { +title: String +embedding: Vector(3) @embed("title") +} +"#; + let schema = parse_schema(input).unwrap(); + match &schema.declarations[0] { + SchemaDecl::Node(n) => { + let ann = &n.properties[1].annotations[0]; + assert!(ann.kwargs.is_empty()); + // Empty kwargs must NOT serialize, so existing schemas' IR JSON (and + // thus the schema hash) stay byte-identical after this field is added. + let json = serde_json::to_string(ann).unwrap(); + assert!(!json.contains("kwargs"), "unexpected kwargs in {json}"); + } + _ => panic!("expected Node"), + } +} + +#[test] +fn test_parse_embed_annotation_rejects_unknown_kwarg() { + let input = r#" +node Doc { +title: String +embedding: Vector(3) @embed("title", provider="openai") +} +"#; + let err = parse_schema(input).unwrap_err(); + assert!( + err.to_string().contains("only 'model' is supported"), + "got: {err}" + ); +} + #[test] fn test_parse_edge_no_body() { let input = "edge WorksAt: Person -> Company\n"; diff --git a/crates/omnigraph-compiler/src/schema/schema.pest b/crates/omnigraph-compiler/src/schema/schema.pest index 395c516..b02724e 100644 --- a/crates/omnigraph-compiler/src/schema/schema.pest +++ b/crates/omnigraph-compiler/src/schema/schema.pest @@ -42,8 +42,10 @@ enum_value = @{ (ASCII_ALPHANUMERIC | "_" | "-")+ } base_type = { "String" | "Blob" | "Bool" | "I32" | "I64" | "U32" | "U64" | "F32" | "F64" | "DateTime" | "Date" } // Annotation rule excludes constraint keywords followed by "(" — those are body_constraints -annotation = { "@" ~ !(constraint_name ~ "(") ~ ident ~ ("(" ~ annotation_arg ~ ")")? } +annotation = { "@" ~ !(constraint_name ~ "(") ~ ident ~ ("(" ~ annotation_args ~ ")")? } +annotation_args = { annotation_arg ~ ("," ~ annotation_kwarg)* } annotation_arg = { literal | ident } +annotation_kwarg = { ident ~ "=" ~ literal } literal = { string_lit | float_lit | integer | bool_lit } diff --git a/crates/omnigraph-server/Cargo.toml b/crates/omnigraph-server/Cargo.toml index 614711e..a6a0717 100644 --- a/crates/omnigraph-server/Cargo.toml +++ b/crates/omnigraph-server/Cargo.toml @@ -22,6 +22,7 @@ aws = ["dep:aws-config", "dep:aws-sdk-secretsmanager"] omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.7.0" } omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.7.0" } omnigraph-policy = { path = "../omnigraph-policy", version = "0.7.0" } +omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.7.0" } omnigraph-cluster = { path = "../omnigraph-cluster", version = "0.7.0" } axum = { workspace = true } clap = { workspace = true } diff --git a/crates/omnigraph-server/examples/bench_concurrent_http.rs b/crates/omnigraph-server/examples/bench_concurrent_http.rs index 6a8411a..044b2ce 100644 --- a/crates/omnigraph-server/examples/bench_concurrent_http.rs +++ b/crates/omnigraph-server/examples/bench_concurrent_http.rs @@ -1,14 +1,15 @@ //! Server-level concurrent HTTP benchmark for MR-686 (PR 0 baseline). //! //! Drives concurrent `/change` requests against an in-process Omnigraph HTTP -//! server. Measures the global `Arc>` lock penalty on -//! current `main` so PR 1 + PR 2 can be evaluated against a real baseline. +//! server. Originally written to measure the global `Arc>` +//! lock penalty as an MR-686 baseline; that lock has since been removed +//! (engine write APIs are `&self`, the server holds a lockless +//! `Arc`), so this now measures the concurrent write path itself +//! (per-`(table, branch)` queue contention + Lance I/O). //! -//! Per the MR-686 plan: this is the load-bearing bench. `Omnigraph::mutate_as` -//! is `&mut self`, so an engine-level concurrent bench either serializes on the -//! borrow checker (measures nothing) or drives multiple handles (measures Lance -//! contention, not the server bottleneck). Driving the HTTP server is the only -//! way to measure the actual `RwLock` contention this work removes. +//! Driving the HTTP server is still the right level: an engine-level bench on +//! a single handle measures Lance contention, not the server's request-path +//! concurrency. //! //! Usage: //! ```sh diff --git a/crates/omnigraph-server/src/api.rs b/crates/omnigraph-server/src/api.rs index ff3cf67..cf0d604 100644 --- a/crates/omnigraph-server/src/api.rs +++ b/crates/omnigraph-server/src/api.rs @@ -1,452 +1,14 @@ -use omnigraph::db::{GraphCommit, MergeOutcome, ReadTarget, SchemaApplyResult, Snapshot}; -use omnigraph::error::{MergeConflict, MergeConflictKind}; -use omnigraph::loader::{LoadMode, LoadResult}; +//! HTTP wire DTOs. The types and their engine-result -> DTO mappings live +//! in the shared `omnigraph-api-types` crate (RFC-009 Phase 2) so the CLI +//! and server share one definition; re-exported here so every +//! `omnigraph_server::api::*` path (handlers, the OpenApi schema list, +//! CLI imports) keeps resolving unchanged. Only `query_catalog_entry` +//! stays — it maps the server's runtime `StoredQuery` (not a wire type) +//! into the shared `QueryCatalogEntry` DTO. + +pub use omnigraph_api_types::*; + use crate::queries::StoredQuery; -use omnigraph_compiler::SchemaMigrationStep; -use omnigraph_compiler::query::ast::Param; -use omnigraph_compiler::result::QueryResult; -use omnigraph_compiler::types::{PropType, ScalarType}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use utoipa::{IntoParams, ToSchema}; - -/// Shadow enum for documenting [`LoadMode`] in the OpenAPI schema. -#[derive(ToSchema)] -#[schema(as = LoadMode)] -#[allow(dead_code)] -enum LoadModeSchema { - /// Overwrite existing data. - #[schema(rename = "overwrite")] - Overwrite, - /// Append to existing data. - #[schema(rename = "append")] - Append, - /// Merge by id key (upsert). - #[schema(rename = "merge")] - Merge, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct SnapshotTableOutput { - pub table_key: String, - pub table_path: String, - pub table_version: u64, - pub table_branch: Option, - pub row_count: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct SnapshotOutput { - pub branch: String, - pub manifest_version: u64, - pub tables: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct BranchCreateRequest { - /// Parent branch to fork from. Defaults to `main`. - pub from: Option, - /// Name of the new branch. Must not already exist. - pub name: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct BranchCreateOutput { - pub uri: String, - pub from: String, - pub name: String, - pub actor_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct BranchListOutput { - pub branches: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct BranchDeleteOutput { - pub uri: String, - pub name: String, - pub actor_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct BranchMergeRequest { - /// Source branch whose commits will be merged. - pub source: String, - /// Target branch that will receive the merge. Defaults to `main`. - pub target: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum BranchMergeOutcome { - AlreadyUpToDate, - FastForward, - Merged, -} - -impl From for BranchMergeOutcome { - fn from(value: MergeOutcome) -> Self { - match value { - MergeOutcome::AlreadyUpToDate => Self::AlreadyUpToDate, - MergeOutcome::FastForward => Self::FastForward, - MergeOutcome::Merged => Self::Merged, - } - } -} - -impl BranchMergeOutcome { - pub fn as_str(self) -> &'static str { - match self { - Self::AlreadyUpToDate => "already_up_to_date", - Self::FastForward => "fast_forward", - Self::Merged => "merged", - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct BranchMergeOutput { - pub source: String, - pub target: String, - pub outcome: BranchMergeOutcome, - pub actor_id: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum MergeConflictKindOutput { - DivergentInsert, - DivergentUpdate, - DeleteVsUpdate, - OrphanEdge, - UniqueViolation, - CardinalityViolation, - ValueConstraintViolation, -} - -impl MergeConflictKindOutput { - pub fn as_str(self) -> &'static str { - match self { - Self::DivergentInsert => "divergent_insert", - Self::DivergentUpdate => "divergent_update", - Self::DeleteVsUpdate => "delete_vs_update", - Self::OrphanEdge => "orphan_edge", - Self::UniqueViolation => "unique_violation", - Self::CardinalityViolation => "cardinality_violation", - Self::ValueConstraintViolation => "value_constraint_violation", - } - } -} - -impl From for MergeConflictKindOutput { - fn from(value: MergeConflictKind) -> Self { - match value { - MergeConflictKind::DivergentInsert => Self::DivergentInsert, - MergeConflictKind::DivergentUpdate => Self::DivergentUpdate, - MergeConflictKind::DeleteVsUpdate => Self::DeleteVsUpdate, - MergeConflictKind::OrphanEdge => Self::OrphanEdge, - MergeConflictKind::UniqueViolation => Self::UniqueViolation, - MergeConflictKind::CardinalityViolation => Self::CardinalityViolation, - MergeConflictKind::ValueConstraintViolation => Self::ValueConstraintViolation, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct MergeConflictOutput { - pub table_key: String, - pub row_id: Option, - pub kind: MergeConflictKindOutput, - pub message: String, -} - -impl From<&MergeConflict> for MergeConflictOutput { - fn from(value: &MergeConflict) -> Self { - Self { - table_key: value.table_key.clone(), - row_id: value.row_id.clone(), - kind: value.kind.into(), - message: value.message.clone(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ReadTargetOutput { - pub branch: Option, - pub snapshot: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ReadOutput { - pub query_name: String, - pub target: ReadTargetOutput, - pub row_count: usize, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub columns: Vec, - pub rows: Value, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ChangeOutput { - pub branch: String, - pub query_name: String, - pub affected_nodes: usize, - pub affected_edges: usize, - pub actor_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct IngestTableOutput { - pub table_key: String, - pub rows_loaded: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct IngestOutput { - pub uri: String, - pub branch: String, - /// Base branch a fork was requested from (the request's `from`), echoed - /// even when the branch already existed. `null` when `from` was absent. - pub base_branch: Option, - pub branch_created: bool, - #[schema(value_type = LoadModeSchema)] - pub mode: LoadMode, - pub tables: Vec, - pub actor_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct CommitOutput { - pub graph_commit_id: String, - pub manifest_branch: Option, - pub manifest_version: u64, - pub parent_commit_id: Option, - pub merged_parent_commit_id: Option, - pub actor_id: Option, - /// Commit creation time as Unix epoch microseconds. - #[schema(example = 1714000000000000i64)] - pub created_at: i64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct CommitListOutput { - pub commits: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ReadRequest { - /// GQ query source. May declare one or more named queries; pick one with - /// `query_name` if there is more than one. - #[schema( - example = "query get_person($name: String) {\n match {\n $p: Person { name: $name }\n }\n return { $p.name, $p.age }\n}" - )] - pub query_source: String, - /// Name of the query to run when `query_source` declares multiple. Optional - /// when only one query is declared. - pub query_name: Option, - /// JSON object whose keys match the query's declared parameters. - pub params: Option, - /// Branch to read from. Mutually exclusive with `snapshot`. Defaults to `main`. - pub branch: Option, - /// Snapshot id to read from. Mutually exclusive with `branch`. - pub snapshot: Option, -} - -/// Inline read-query request for `POST /query`. -/// -/// Friendlier-named alternative to [`ReadRequest`] for ad-hoc reads and -/// AI-agent integration. Mutations are rejected with 400 — use `POST -/// /mutate` (or its deprecated alias `POST /change`) for write queries. -/// Field names are deliberately short (`query`, `name`) to match the GQ -/// keyword and the CLI `-e` flag. -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct QueryRequest { - /// GQ read-query source. May declare one or more named queries; pick one - /// with `name` when more than one is declared. Mutations - /// (`insert`/`update`/`delete`) get 400 — use `POST /mutate` (or its - /// deprecated alias `POST /change`) instead. - #[schema(example = "query get_person($name: String) {\n match {\n $p: Person { name: $name }\n }\n return { $p.name, $p.age }\n}")] - pub query: String, - /// Name of the query to run when `query` declares multiple. Optional when - /// only one query is declared. - pub name: Option, - /// JSON object whose keys match the query's declared parameters. - pub params: Option, - /// Branch to read from. Mutually exclusive with `snapshot`. Defaults to `main`. - pub branch: Option, - /// Snapshot id to read from. Mutually exclusive with `branch`. - pub snapshot: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ChangeRequest { - /// GQ mutation source containing `insert`, `update`, or `delete` statements. - /// May declare multiple named mutations; pick one with `name`. - /// - /// Accepts the legacy field name `query_source` as a deserialization alias. - #[schema( - example = "query insert_person($name: String, $age: I32) {\n insert Person { name: $name, age: $age }\n}" - )] - #[serde(alias = "query_source")] - pub query: String, - /// Name of the mutation to run when `query` declares multiple. - /// - /// Accepts the legacy field name `query_name` as a deserialization alias. - #[serde(default, alias = "query_name")] - pub name: Option, - /// JSON object whose keys match the mutation's declared parameters. - #[serde(default)] - pub params: Option, - /// Target branch. Defaults to `main`. - #[serde(default)] - pub branch: Option, -} - -/// Body for `POST /queries/{name}` — invokes the server-side stored query -/// named in the path. The query source and name come from the registry, -/// never the body; only the runtime inputs are supplied here. -#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)] -pub struct InvokeStoredQueryRequest { - /// JSON object whose keys match the stored query's declared parameters. - #[serde(default)] - pub params: Option, - /// Branch to run against. Defaults to `main`; for a stored mutation the - /// write targets this branch. - #[serde(default)] - pub branch: Option, - /// Snapshot id to read from (read queries only — rejected for a stored - /// mutation). Mutually exclusive with `branch`. - #[serde(default)] - pub snapshot: Option, -} - -/// Response for `POST /queries/{name}`: the read envelope for a stored -/// read, or the mutation envelope for a stored mutation. Serialized -/// **untagged**, so the wire shape is exactly [`ReadOutput`] or -/// [`ChangeOutput`] — classification follows the stored query, not a -/// wrapper field. -#[derive(Debug, Serialize, ToSchema)] -#[serde(untagged)] -pub enum InvokeStoredQueryResponse { - Read(ReadOutput), - Change(ChangeOutput), -} - -/// The kind of a stored-query parameter, decomposed so a client (e.g. an -/// MCP server) can build a typed input schema with a closed `match` and -/// never re-parse omnigraph's type spelling. `bigint`/`date`/`datetime`/ -/// `blob` are carried as JSON strings on the wire: a 64-bit integer past -/// 2^53 loses precision as a JSON number, and Date/DateTime are ISO -/// strings, Blob a blob-URI string. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum ParamKind { - String, - Bool, - Int, - #[serde(rename = "bigint")] - BigInt, - Float, - Date, - #[serde(rename = "datetime")] - DateTime, - Blob, - Vector, - List, -} - -/// One declared parameter of a stored query, projected for the catalog. -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ParamDescriptor { - pub name: String, - pub kind: ParamKind, - /// Element kind when `kind == list` (always a scalar — the grammar - /// forbids lists of vectors or nested lists). - #[serde(skip_serializing_if = "Option::is_none")] - pub item_kind: Option, - /// Dimension when `kind == vector`. - #[serde(skip_serializing_if = "Option::is_none")] - pub vector_dim: Option, - /// `false` → the caller must supply it; `true` → optional. - pub nullable: bool, -} - -/// One entry in the stored-query catalog (`GET /queries`). -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct QueryCatalogEntry { - /// Registry key / invoke path segment (`POST /queries/{name}`). - pub name: String, - /// MCP tool id (the `tool_name` override, else `name`). - pub tool_name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub instruction: Option, - /// `true` for a stored mutation → an MCP read-only hint of `false`. - pub mutation: bool, - pub params: Vec, -} - -/// Response for `GET /queries`: the `mcp.expose` subset of a graph's -/// stored-query registry, each with typed parameters. -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct QueriesCatalogOutput { - pub queries: Vec, -} - -/// Total map from a resolved scalar to its catalog kind. Exhaustive on -/// purpose: a new `ScalarType` is a compile error here until catalogued. -fn scalar_kind(scalar: ScalarType) -> ParamKind { - match scalar { - ScalarType::String => ParamKind::String, - ScalarType::Bool => ParamKind::Bool, - ScalarType::I32 | ScalarType::U32 => ParamKind::Int, - ScalarType::I64 | ScalarType::U64 => ParamKind::BigInt, - ScalarType::F32 | ScalarType::F64 => ParamKind::Float, - ScalarType::Date => ParamKind::Date, - ScalarType::DateTime => ParamKind::DateTime, - ScalarType::Blob => ParamKind::Blob, - ScalarType::Vector(_) => ParamKind::Vector, - } -} - -fn param_descriptor(param: &Param) -> ParamDescriptor { - match PropType::from_param_type_name(¶m.type_name, param.nullable) { - Some(pt) if pt.list => ParamDescriptor { - name: param.name.clone(), - kind: ParamKind::List, - item_kind: Some(scalar_kind(pt.scalar)), - vector_dim: None, - nullable: param.nullable, - }, - Some(pt) => { - let (kind, vector_dim) = match pt.scalar { - ScalarType::Vector(dim) => (ParamKind::Vector, Some(dim)), - other => (scalar_kind(other), None), - }; - ParamDescriptor { - name: param.name.clone(), - kind, - item_kind: None, - vector_dim, - nullable: param.nullable, - } - } - // Unreachable for a parsed query (every declared param type is - // grammatical); fall back to an opaque string so the field is still - // usable rather than dropped. - None => ParamDescriptor { - name: param.name.clone(), - kind: ParamKind::String, - item_kind: None, - vector_dim: None, - nullable: param.nullable, - }, - } -} /// Project a loaded stored query into its catalog entry (typed params, /// MCP tool name, read/mutate flag, description/instruction). @@ -460,246 +22,3 @@ pub fn query_catalog_entry(query: &StoredQuery) -> QueryCatalogEntry { params: query.decl.params.iter().map(param_descriptor).collect(), } } - -#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)] -pub struct SchemaApplyRequest { - /// Project schema in `.pg` source form. The diff against the current - /// schema produces the migration steps that will be applied. - #[schema( - example = "node Person {\n name: String @key\n age: I32?\n}\n\nedge Knows: Person -> Person" - )] - pub schema_source: String, - /// When true, promote every `DropMode::Soft` step in the plan to - /// `DropMode::Hard`, making the prior column data unreachable - /// after the apply. Matches the CLI's `--allow-data-loss` flag. - /// Defaults to `false` (drops remain reversible via time travel). - #[serde(default)] - pub allow_data_loss: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct SchemaApplyOutput { - pub uri: String, - pub supported: bool, - pub applied: bool, - pub step_count: usize, - pub manifest_version: u64, - #[schema(value_type = Vec)] - pub steps: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct SchemaOutput { - pub schema_source: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct IngestRequest { - /// Target branch. Defaults to `main`. Without `from`, the branch must - /// already exist — a missing branch is a 404, never an implicit fork. - pub branch: Option, - /// Parent branch used to create `branch` if it does not exist. Branch - /// creation is opt-in by presence of this field; omit it to require an - /// existing branch. - pub from: Option, - /// How existing rows are handled. Defaults to `merge`. - #[schema(value_type = Option)] - pub mode: Option, - /// NDJSON payload: one record per line, each shaped - /// `{"type": "", "data": {...}}`. - #[schema( - example = "{\"type\": \"Person\", \"data\": {\"name\": \"Alice\", \"age\": 30}}\n{\"type\": \"Person\", \"data\": {\"name\": \"Bob\", \"age\": 25}}" - )] - pub data: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ExportRequest { - /// Branch to export. Defaults to `main`. - pub branch: Option, - /// Restrict the export to these node/edge type names. Empty exports all types. - #[serde(default)] - pub type_names: Vec, - /// Restrict the export to these table keys. Empty exports all tables. - #[serde(default)] - pub table_keys: Vec, -} - -#[derive(Debug, Clone, Deserialize, IntoParams)] -pub struct SnapshotQuery { - pub branch: Option, -} - -#[derive(Debug, Clone, Deserialize, IntoParams)] -pub struct CommitListQuery { - pub branch: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct HealthOutput { - pub status: String, - pub version: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub source_version: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum ErrorCode { - Unauthorized, - Forbidden, - BadRequest, - NotFound, - /// 405 Method Not Allowed — the route exists but the active server - /// mode doesn't serve this method (e.g. `GET /graphs` in single-graph - /// mode). Distinct from 404 so clients can tell "wrong context" from - /// "no such resource." - MethodNotAllowed, - Conflict, - /// 429 Too Many Requests — per-actor admission cap exceeded. - /// Clients should respect the `Retry-After` header. - TooManyRequests, - Internal, -} - -/// Structured details for a publisher-level OCC failure. Surfaces alongside -/// HTTP 409 when a write was rejected because the caller's pre-write view of -/// one table's manifest version was stale relative to the current head. The -/// expected/actual fields tell the client which table to refresh. -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ManifestConflictOutput { - pub table_key: String, - pub expected: u64, - pub actual: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct ErrorOutput { - pub error: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub code: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub merge_conflicts: Vec, - /// Set when the conflict is a publisher CAS rejection - /// (`ManifestConflictDetails::ExpectedVersionMismatch`). The caller's - /// pre-write view of `table_key` was at version `expected` but the - /// manifest is now at `actual`. Refresh and retry. - #[serde(skip_serializing_if = "Option::is_none")] - pub manifest_conflict: Option, -} - -pub fn snapshot_payload(branch: &str, snapshot: &Snapshot) -> SnapshotOutput { - let mut entries: Vec<_> = snapshot.entries().cloned().collect(); - entries.sort_by(|a, b| a.table_key.cmp(&b.table_key)); - let tables = entries - .iter() - .map(|entry| SnapshotTableOutput { - table_key: entry.table_key.clone(), - table_path: entry.table_path.clone(), - table_version: entry.table_version, - table_branch: entry.table_branch.clone(), - row_count: entry.row_count, - }) - .collect::>(); - SnapshotOutput { - branch: branch.to_string(), - manifest_version: snapshot.version(), - tables, - } -} - -pub fn schema_apply_output(uri: &str, result: SchemaApplyResult) -> SchemaApplyOutput { - SchemaApplyOutput { - uri: uri.to_string(), - supported: result.supported, - applied: result.applied, - step_count: result.steps.len(), - manifest_version: result.manifest_version, - steps: result.steps, - } -} - -pub fn commit_output(commit: &GraphCommit) -> CommitOutput { - CommitOutput { - graph_commit_id: commit.graph_commit_id.clone(), - manifest_branch: commit.manifest_branch.clone(), - manifest_version: commit.manifest_version, - parent_commit_id: commit.parent_commit_id.clone(), - merged_parent_commit_id: commit.merged_parent_commit_id.clone(), - actor_id: commit.actor_id.clone(), - created_at: commit.created_at, - } -} - -pub fn read_output(query_name: String, target: &ReadTarget, result: QueryResult) -> ReadOutput { - let columns = result - .schema() - .fields() - .iter() - .map(|field| field.name().clone()) - .collect(); - ReadOutput { - query_name, - target: read_target_output(target), - row_count: result.num_rows(), - columns, - rows: result.to_rust_json(), - } -} - -pub fn ingest_output( - uri: &str, - result: &LoadResult, - mode: LoadMode, - actor_id: Option, -) -> IngestOutput { - IngestOutput { - uri: uri.to_string(), - branch: result.branch.clone(), - base_branch: result.base_branch.clone(), - branch_created: result.branch_created, - mode, - tables: result - .to_ingest_tables() - .into_iter() - .map(|table| IngestTableOutput { - table_key: table.table_key, - rows_loaded: table.rows_loaded, - }) - .collect(), - actor_id, - } -} - -pub fn read_target_output(target: &ReadTarget) -> ReadTargetOutput { - match target { - ReadTarget::Branch(branch) => ReadTargetOutput { - branch: Some(branch.clone()), - snapshot: None, - }, - ReadTarget::Snapshot(snapshot) => ReadTargetOutput { - branch: None, - snapshot: Some(snapshot.as_str().to_string()), - }, - } -} - -// ─── MR-668 — management endpoint shapes ────────────────────────────────── - -/// One entry in the response from `GET /graphs`. Cluster operators -/// consume this list to discover which graphs the server is currently -/// serving. The shape is intentionally minimal — `graph_id` and `uri` -/// are the only fields a routing client needs. -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct GraphInfo { - pub graph_id: String, - pub uri: String, -} - -/// Response from `GET /graphs`. Lists every graph registered with the -/// server in alphabetical order by `graph_id` (sorted server-side so -/// clients get deterministic output across requests). -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct GraphListResponse { - pub graphs: Vec, -} diff --git a/crates/omnigraph-server/src/config.rs b/crates/omnigraph-server/src/config.rs deleted file mode 100644 index 15b957d..0000000 --- a/crates/omnigraph-server/src/config.rs +++ /dev/null @@ -1,1103 +0,0 @@ -use std::collections::BTreeMap; -use std::env; -use std::fs; -use std::path::{Path, PathBuf}; - -use clap::ValueEnum; -use color_eyre::eyre::{Result, bail}; -use serde::{Deserialize, Serialize}; - -pub const DEFAULT_CONFIG_FILE: &str = "omnigraph.yaml"; - -pub fn graph_resource_id_for_selection( - selected_graph: Option<&str>, - normalized_uri: &str, -) -> String { - selected_graph.unwrap_or(normalized_uri).to_string() -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProjectConfig { - pub name: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TargetConfig { - pub uri: String, - pub bearer_token_env: Option, - /// Per-graph Cedar policy file (MR-668). In single-graph mode this - /// field is unused — the top-level `policy.file` applies. In - /// multi-graph mode, each `graphs..policy.file` governs that - /// graph's HTTP-layer Cedar enforcement. - #[serde(default)] - pub policy: PolicySettings, - /// Per-graph stored-query registry: an inline `name -> entry` - /// map. Mirrors the per-graph `policy` shape — each - /// `graphs..queries` declares that graph's stored queries. Absent - /// (or empty) = no stored queries for the graph. v1 is inline-only; - /// an external `queries.yaml` manifest indirection is a deferred - /// convenience. - #[serde(default)] - pub queries: BTreeMap, -} - -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, ValueEnum)] -#[serde(rename_all = "snake_case")] -pub enum ReadOutputFormat { - #[default] - Table, - Kv, - Csv, - Jsonl, - Json, -} - -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, ValueEnum)] -#[serde(rename_all = "snake_case")] -pub enum TableCellLayout { - #[default] - Truncate, - Wrap, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct CliDefaults { - #[serde(rename = "graph")] - pub graph: Option, - pub branch: Option, - pub output_format: Option, - pub table_max_column_width: Option, - pub table_cell_layout: Option, - /// Default actor identity for CLI direct-engine writes (MR-722). - /// Used when `policy.file` is configured and the operator hasn't - /// passed `--as ` on the command line. With policy configured - /// and neither this nor `--as` set, the engine-layer footgun guard - /// fires (no silent bypass). - pub actor: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ServerDefaults { - #[serde(rename = "graph")] - pub graph: Option, - pub bind: Option, - /// Server-level Cedar policy (MR-668). Governs management endpoints - /// — currently `GET /graphs`; future runtime add/remove endpoints - /// will plug in here too. In single-graph mode this is unused — the - /// top-level `policy.file` covers the single graph. - #[serde(default)] - pub policy: PolicySettings, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct AuthDefaults { - pub env_file: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct QueryDefaults { - #[serde(default)] - pub roots: Vec, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct PolicySettings { - pub file: Option, -} - -/// One stored-query registry entry. The map **key** is the query's -/// identity — it must equal the `query ` symbol declared inside -/// the referenced `.gq` file (asserted when the registry loads). -/// Renaming the key (or the symbol) is a breaking change to callers, by -/// design. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct QueryEntry { - /// Path to the `.gq` file (relative to the config's `base_dir`). The - /// file may declare several queries; the registry selects the one - /// whose symbol matches the map key. - pub file: String, - #[serde(default)] - pub mcp: McpSettings, -} - -/// MCP exposure for a stored query. A *deployment* concern (the same -/// `.gq` may be exposed in one graph and hidden in another), so it lives -/// in YAML rather than in the `.gq` source. **Default `expose: true`** — -/// declaring a query in the manifest *is* the opt-in, so it appears in the -/// MCP tool catalog (`GET /queries`) by default; set `expose: false` to -/// keep a query HTTP/service-callable but hidden from the agent tool list. -/// `expose` governs catalog membership only — it is **not** an -/// authorization gate (invocation is gated by `invoke_query`), so a hidden -/// query is still invocable by name with the right permission. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpSettings { - #[serde(default = "mcp_expose_default")] - pub expose: bool, - pub tool_name: Option, -} - -fn mcp_expose_default() -> bool { - true -} - -impl Default for McpSettings { - fn default() -> Self { - Self { - expose: mcp_expose_default(), - tool_name: None, - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AliasCommand { - /// Read alias (canonical: `query`). The legacy spelling `read` is - /// kept as the variant name for back-compat with serialized configs - /// and external SDK callers; `query` is accepted on the wire via the - /// serde alias. - #[serde(alias = "query")] - Read, - /// Mutation alias (canonical: `mutate`). The legacy spelling `change` - /// is kept as the variant name for back-compat; `mutate` is accepted - /// on the wire via the serde alias. - #[serde(alias = "mutate")] - Change, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AliasConfig { - pub command: AliasCommand, - pub query: String, - pub name: Option, - #[serde(default)] - pub args: Vec, - #[serde(rename = "graph")] - pub graph: Option, - pub branch: Option, - pub format: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OmnigraphConfig { - #[serde(default)] - pub project: ProjectConfig, - #[serde(default, rename = "graphs")] - pub graphs: BTreeMap, - #[serde(default)] - pub server: ServerDefaults, - #[serde(default)] - pub auth: AuthDefaults, - #[serde(default)] - pub cli: CliDefaults, - #[serde(default)] - pub query: QueryDefaults, - #[serde(default)] - pub aliases: BTreeMap, - #[serde(default)] - pub policy: PolicySettings, - /// Top-level stored-query registry, used in single-graph - /// mode — mirrors how the top-level `policy` applies to the single - /// graph. In multi-graph mode this is unused; each graph's - /// `graphs..queries` applies instead. - #[serde(default)] - pub queries: BTreeMap, - #[serde(skip)] - base_dir: PathBuf, -} - -impl Default for OmnigraphConfig { - fn default() -> Self { - Self { - project: ProjectConfig::default(), - graphs: BTreeMap::new(), - server: ServerDefaults::default(), - auth: AuthDefaults::default(), - cli: CliDefaults::default(), - query: QueryDefaults::default(), - aliases: BTreeMap::new(), - policy: PolicySettings::default(), - queries: BTreeMap::new(), - base_dir: PathBuf::new(), - } - } -} - -impl OmnigraphConfig { - pub fn base_dir(&self) -> &Path { - &self.base_dir - } - - pub fn cli_branch(&self) -> &str { - self.cli.branch.as_deref().unwrap_or("main") - } - - pub fn cli_output_format(&self) -> ReadOutputFormat { - self.cli.output_format.unwrap_or_default() - } - - pub fn table_max_column_width(&self) -> usize { - self.cli.table_max_column_width.unwrap_or(80) - } - - pub fn table_cell_layout(&self) -> TableCellLayout { - self.cli.table_cell_layout.unwrap_or_default() - } - - pub fn cli_graph_name(&self) -> Option<&str> { - self.cli.graph.as_deref() - } - - pub fn server_graph_name(&self) -> Option<&str> { - self.server.graph.as_deref() - } - - pub fn server_bind(&self) -> &str { - self.server.bind.as_deref().unwrap_or("127.0.0.1:8080") - } - - pub fn resolve_target_name<'a>( - &self, - explicit_uri: Option<&str>, - explicit_target: Option<&'a str>, - default_target: Option<&'a str>, - ) -> Option<&'a str> { - explicit_target.or_else(|| { - if explicit_uri.is_some() { - None - } else { - default_target - } - }) - } - - pub fn graph_bearer_token_env( - &self, - explicit_uri: Option<&str>, - explicit_target: Option<&str>, - default_target: Option<&str>, - ) -> Option<&str> { - let target_name = - self.resolve_target_name(explicit_uri, explicit_target, default_target)?; - self.graphs - .get(target_name) - .and_then(|target| target.bearer_token_env.as_deref()) - } - - pub fn resolve_auth_env_file(&self) -> Option { - self.auth - .env_file - .as_deref() - .map(|path| self.resolve_config_path(path)) - } - - pub fn resolve_policy_file(&self) -> Option { - self.policy - .file - .as_deref() - .map(|path| self.resolve_config_path(path)) - } - - /// Resolve the per-graph policy file path for the named target, - /// relative to the config file's `base_dir`. Returns `None` if the - /// target is unknown or no per-graph `policy.file` is set. - pub fn resolve_target_policy_file(&self, target_name: &str) -> Option { - let target = self.graphs.get(target_name)?; - target - .policy - .file - .as_deref() - .map(|path| self.resolve_config_path(path)) - } - - /// The top-level stored-query registry entries (single-graph mode). - pub fn query_entries(&self) -> &BTreeMap { - &self.queries - } - - /// The per-graph stored-query registry entries for a named target - /// (multi-graph mode). Returns `None` if the target is unknown. - pub fn target_query_entries( - &self, - target_name: &str, - ) -> Option<&BTreeMap> { - self.graphs.get(target_name).map(|target| &target.queries) - } - - /// The stored-query registry entries that apply for a graph - /// selection — the single definition of "which `queries:` block - /// governs graph X", shared by server boot and the CLI so the two - /// can't drift. A named graph present in `graphs:` uses its - /// per-graph block; everything else (no selection, or a name that is - /// not a known graph, e.g. a bare URI) falls back to the top-level - /// block (single-graph mode). - pub fn query_entries_for(&self, graph: Option<&str>) -> &BTreeMap { - match graph { - Some(name) if self.graphs.contains_key(name) => &self.graphs[name].queries, - _ => &self.queries, - } - } - - /// The single CLI gate that turns a raw graph selection into a *validated* - /// one — the fallible counterpart to the infallible - /// [`OmnigraphConfig::query_entries_for`]. Both `queries` subcommands route - /// their selection through here so neither can skip a check the other (or - /// server boot) applies: - /// * a known name passes through, but only after the same coherence check - /// server boot enforces - /// ([`OmnigraphConfig::ensure_top_level_blocks_honored`]) — a named graph - /// with a populated top-level block is rejected; - /// * an unknown name errors with the **same** message - /// [`OmnigraphConfig::resolve_target_uri`] produces, so a command that - /// opens no URI rejects an unknown `--target` exactly like the - /// URI-resolving commands do; - /// * an anonymous selection (`None`, e.g. a bare URI) stays anonymous, - /// resolving to the top-level registry downstream (top-level honored). - pub fn resolve_graph_selection<'a>(&self, graph: Option<&'a str>) -> Result> { - match graph { - Some(name) if self.graphs.contains_key(name) => { - self.ensure_top_level_blocks_honored(Some(name))?; - Ok(Some(name)) - } - Some(name) => bail!("graph '{}' not found in {}", name, DEFAULT_CONFIG_FILE), - None => Ok(None), - } - } - - pub fn resolve_policy_tooling_graph_selection(&self) -> Result> { - self.resolve_graph_selection(self.cli_graph_name().or_else(|| self.server_graph_name())) - } - - /// The policy file that applies for a graph selection — the policy - /// sibling of [`OmnigraphConfig::query_entries_for`], so policy and - /// queries resolve by the same identity rule. A named graph in - /// `graphs:` uses its per-graph `policy.file` with **no** top-level - /// fallback (a named graph with no per-graph policy has no policy — - /// that keeps the boot-time coherence check meaningful); anything else - /// (no selection, or a bare URI) uses the top-level `policy.file`. - pub fn resolve_policy_file_for(&self, graph: Option<&str>) -> Option { - match graph { - Some(name) if self.graphs.contains_key(name) => self.resolve_target_policy_file(name), - _ => self.resolve_policy_file(), - } - } - - /// Names of any top-level config blocks (`policy.file`, `queries:`) - /// that are populated. Used by the boot-time coherence check: when a - /// **named** graph is served (single-mode by name, or multi-mode), - /// the top-level blocks are not honored, so a populated one is a - /// configuration error rather than a silent no-op. - pub fn populated_top_level_blocks(&self) -> Vec<&'static str> { - let mut blocks = Vec::new(); - if self.policy.file.is_some() { - blocks.push("policy.file"); - } - if !self.queries.is_empty() { - blocks.push("queries"); - } - blocks - } - - /// A named graph uses its own `graphs.` block, so a populated - /// top-level block would be silently ignored — a config error. The single - /// definition of that rule, shared by server boot and the CLI selection - /// gate ([`OmnigraphConfig::resolve_graph_selection`]) so the two can't - /// drift. An anonymous selection (`None`, e.g. a bare URI) legitimately - /// honors the top-level blocks, so it is never rejected here. - pub fn ensure_top_level_blocks_honored(&self, selected: Option<&str>) -> Result<()> { - if let Some(name) = selected { - let unhonored = self.populated_top_level_blocks(); - if !unhonored.is_empty() { - bail!( - "named graph '{name}' uses its own `graphs.{name}.…` block, but top-level {} \ - {} set and would be ignored. Move it to `graphs.{name}` (e.g. \ - `graphs.{name}.policy.file`, `graphs.{name}.queries`).", - unhonored.join(" and "), - if unhonored.len() == 1 { "is" } else { "are" }, - ); - } - } - Ok(()) - } - - /// Resolve a stored-query `.gq` file path (from a registry entry), - /// relative to the config's `base_dir`. Mirrors policy-file - /// resolution; the registry loader calls this to turn each entry's - /// `file:` value into an absolute path. - pub fn resolve_query_file(&self, value: &str) -> PathBuf { - self.resolve_config_path(value) - } - - /// Resolve the server-level policy file path (used by management - /// endpoints). Returns `None` if `server.policy.file` is not set. - pub fn resolve_server_policy_file(&self) -> Option { - self.server - .policy - .file - .as_deref() - .map(|path| self.resolve_config_path(path)) - } - - /// Resolve a raw config-supplied URI (which may be relative) to its - /// absolute form. URIs containing `://` are passed through as-is; - /// relative paths are joined with the config file's `base_dir`. - pub fn resolve_uri_value(&self, value: &str) -> String { - self.resolve_config_uri(value) - } - - pub fn resolve_policy_tests_file(&self) -> Option { - let policy_file = self.resolve_policy_file()?; - Some(policy_file.with_file_name("policy.tests.yaml")) - } - - pub fn alias(&self, name: &str) -> Result<&AliasConfig> { - self.aliases - .get(name) - .ok_or_else(|| color_eyre::eyre::eyre!("alias '{}' not found", name)) - } - - pub fn resolve_target_uri( - &self, - explicit_uri: Option, - explicit_target: Option<&str>, - default_target: Option<&str>, - ) -> Result { - if let Some(uri) = explicit_uri { - return Ok(uri); - } - - let target_name = explicit_target.or(default_target).ok_or_else(|| { - color_eyre::eyre::eyre!("URI must be provided via , --target, or config") - })?; - let target = self.graphs.get(target_name).ok_or_else(|| { - color_eyre::eyre::eyre!( - "graph '{}' not found in {}", - target_name, - DEFAULT_CONFIG_FILE - ) - })?; - Ok(self.resolve_config_uri(&target.uri)) - } - - pub fn resolve_query_path(&self, query: &Path) -> Result { - if query.is_absolute() { - return Ok(query.to_path_buf()); - } - - let direct = self.base_dir.join(query); - if direct.exists() { - return Ok(direct); - } - - for root in &self.query.roots { - let candidate = self.base_dir.join(root).join(query); - if candidate.exists() { - return Ok(candidate); - } - } - - bail!("query file '{}' not found", query.display()); - } - - fn resolve_config_uri(&self, value: &str) -> String { - if value.contains("://") { - return value.to_string(); - } - - let path = Path::new(value); - if path.is_absolute() { - value.to_string() - } else { - self.base_dir.join(path).to_string_lossy().to_string() - } - } - - fn resolve_config_path(&self, value: &str) -> PathBuf { - let path = Path::new(value); - if path.is_absolute() { - path.to_path_buf() - } else { - self.base_dir.join(path) - } - } -} - -pub fn default_config_path() -> PathBuf { - PathBuf::from(DEFAULT_CONFIG_FILE) -} - -/// `OMNIGRAPH_CONFIG` env var: a first-class stand-in for `--config`, one -/// name with one meaning in both binaries (the container entrypoint already -/// uses it for the server; RFC-007 §D1 extends it to the CLI). -pub const CONFIG_PATH_ENV: &str = "OMNIGRAPH_CONFIG"; - -/// RFC-008 stage 4 — opt-in strict mode: when set, loading a legacy -/// `omnigraph.yaml` is a hard error instead of a warning. For teams that -/// finished migrating and want regressions caught (a stray legacy file -/// would otherwise silently outrank operator config during the window). -/// The rehearsal for stage 5's removal. -pub const NO_LEGACY_CONFIG_ENV: &str = "OMNIGRAPH_NO_LEGACY_CONFIG"; - -pub fn load_config(config_path: Option<&PathBuf>) -> Result { - let env_path = env::var_os(CONFIG_PATH_ENV).map(PathBuf::from); - let strict = env::var_os(NO_LEGACY_CONFIG_ENV).is_some(); - load_config_in(&env::current_dir()?, config_path, env_path.as_ref(), strict) -} - -fn load_config_in( - cwd: &Path, - config_path: Option<&PathBuf>, - env_path: Option<&PathBuf>, - strict_no_legacy: bool, -) -> Result { - // Precedence: explicit --config flag > $OMNIGRAPH_CONFIG > ./omnigraph.yaml. - let explicit_path = config_path.or(env_path).cloned(); - let config_path = explicit_path.or_else(|| { - let default_path = cwd.join(DEFAULT_CONFIG_FILE); - default_path.exists().then_some(default_path) - }); - - let mut config = if let Some(path) = &config_path { - if strict_no_legacy { - // Strict refuses the FILE, not its absence — flag-less - // invocations on migrated setups keep working. - bail!( - "legacy config '{}' refused: {NO_LEGACY_CONFIG_ENV} is set (RFC-008 strict mode); run `omnigraph config migrate`, then remove the file — or unset the variable", - path.display() - ); - } - let text = fs::read_to_string(path)?; - warn_yaml_deprecation_once(path, &text); - serde_yaml::from_str::(&text)? - } else { - OmnigraphConfig::default() - }; - - config.base_dir = if let Some(path) = config_path { - absolute_base_dir(cwd, &path)? - } else { - cwd.to_path_buf() - }; - - Ok(config) -} - -/// RFC-008 stage 1: suppress the legacy-config deprecation warning -/// (one process), for CI logs during the deprecation window. -pub const SUPPRESS_YAML_DEPRECATION_ENV: &str = "OMNIGRAPH_SUPPRESS_YAML_DEPRECATION"; - -/// RFC-008's migration map (the "Where every key goes" table), applied to -/// the keys actually present in a loaded file — never a generic banner. -/// Keys are `(yaml pointer, destination)`; the pointer is matched against -/// the file's real top-level/nested keys. -const YAML_DEPRECATION_MAP: &[(&str, &str)] = &[ - ("graphs", "cluster.yaml `graphs:` (team surface) — or flags/env for the zero-config tier"), - ("queries", "the cluster catalog (`.gq` discovery in cluster.yaml)"), - ("policy", "cluster.yaml `policies:` + `applies_to` bindings"), - ("server", "flags/env (`--bind`); meaningless under cluster boot"), - ("auth", "the operator credentials chain (`omnigraph login `)"), - ("aliases", "operator `aliases:` (bindings) + catalog stored queries (content)"), - ("query", "obsolete — cluster query discovery replaced `query.roots`"), - ("project", "cluster.yaml `metadata.name`"), - ("cli.actor", "`operator.actor` in ~/.omnigraph/config.yaml"), - ("cli.output_format", "`defaults.output` in ~/.omnigraph/config.yaml"), - ("cli.table_max_column_width", "`defaults.table_max_column_width` in ~/.omnigraph/config.yaml"), - ("cli.table_cell_layout", "`defaults.table_cell_layout` in ~/.omnigraph/config.yaml"), - ("cli.graph", "explicit `--target`/`--server` (no operator default-target yet)"), - ("cli.branch", "explicit `--branch`"), -]; - -/// Emit the per-key deprecation block once per process when a legacy -/// `omnigraph.yaml` is actually loaded. `omnigraph config migrate` -/// produces the split these lines describe. -fn warn_yaml_deprecation_once(path: &Path, text: &str) { - static WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); - if env::var_os(SUPPRESS_YAML_DEPRECATION_ENV).is_some() { - return; - } - let lines = yaml_deprecation_lines(text); - if lines.is_empty() { - return; - } - WARNED.get_or_init(|| { - eprintln!( - "warning: '{}' is deprecated (RFC-008) — its keys have new homes; run `omnigraph config migrate` for the split, set {SUPPRESS_YAML_DEPRECATION_ENV}=1 to silence:", - path.display() - ); - for line in &lines { - eprintln!(" {line}"); - } - }); -} - -fn yaml_deprecation_lines(text: &str) -> Vec { - let Ok(mapping) = serde_yaml::from_str::(text) else { - return Vec::new(); - }; - let present = |pointer: &str| -> bool { - match pointer.split_once('.') { - None => mapping.contains_key(pointer), - Some((outer, inner)) => mapping - .get(outer) - .and_then(|value| value.as_mapping()) - .is_some_and(|nested| nested.contains_key(inner)), - } - }; - YAML_DEPRECATION_MAP - .iter() - .filter(|(pointer, _)| present(pointer)) - .map(|(pointer, destination)| format!("`{pointer}` -> {destination}")) - .collect() -} - -fn absolute_base_dir(cwd: &Path, path: &Path) -> Result { - let path = if path.is_absolute() { - path.to_path_buf() - } else { - cwd.join(path) - }; - Ok(path - .parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| cwd.to_path_buf())) -} - -#[cfg(test)] -mod tests { - use std::fs; - use std::path::{Path, PathBuf}; - - use tempfile::tempdir; - - use super::{ - ReadOutputFormat, TableCellLayout, graph_resource_id_for_selection, load_config_in, - }; - - #[test] - fn env_config_path_stands_in_for_the_flag_but_loses_to_it() { - let temp = tempdir().unwrap(); - let flag_path = temp.path().join("flag.yaml"); - let env_path = temp.path().join("env.yaml"); - fs::write(&flag_path, "cli:\n actor: act-flag\n").unwrap(); - fs::write(&env_path, "cli:\n actor: act-env\n").unwrap(); - - // $OMNIGRAPH_CONFIG used when no flag… - let config = load_config_in(temp.path(), None, Some(&env_path), false).unwrap(); - assert_eq!(config.cli.actor.as_deref(), Some("act-env")); - - // …loses to an explicit --config… - let config = load_config_in(temp.path(), Some(&flag_path), Some(&env_path), false).unwrap(); - assert_eq!(config.cli.actor.as_deref(), Some("act-flag")); - - // …and beats the cwd default file. - fs::write(temp.path().join("omnigraph.yaml"), "cli:\n actor: act-cwd\n").unwrap(); - let config = load_config_in(temp.path(), None, Some(&env_path), false).unwrap(); - assert_eq!(config.cli.actor.as_deref(), Some("act-env")); - } - - #[test] - fn strict_mode_refuses_the_file_not_its_absence() { - let temp = tempdir().unwrap(); - // No file: strict mode changes nothing (defaults load). - let config = load_config_in(temp.path(), None, None, true).unwrap(); - assert!(config.cli.actor.is_none()); - - // File present: strict refuses with the migrate pointer. - fs::write(temp.path().join("omnigraph.yaml"), "cli:\n actor: a\n").unwrap(); - let err = load_config_in(temp.path(), None, None, true).unwrap_err(); - let message = err.to_string(); - assert!( - message.contains("OMNIGRAPH_NO_LEGACY_CONFIG") && message.contains("config migrate"), - "{message}" - ); - // Without strict, the same file loads. - assert!(load_config_in(temp.path(), None, None, false).is_ok()); - } - - #[test] - fn yaml_deprecation_lines_name_present_keys_only() { - let lines = super::yaml_deprecation_lines( - "graphs:\n g:\n uri: /tmp/x\ncli:\n actor: a\n branch: main\n", - ); - let joined = lines.join("\n"); - assert!(joined.contains("`graphs` ->"), "{joined}"); - assert!(joined.contains("`cli.actor` -> `operator.actor`"), "{joined}"); - assert!(joined.contains("`cli.branch` ->"), "{joined}"); - assert!(!joined.contains("`aliases`"), "{joined}"); - assert!(!joined.contains("`cli.output_format`"), "{joined}"); - - assert!(super::yaml_deprecation_lines("").is_empty()); - assert!(super::yaml_deprecation_lines("not: [valid").is_empty()); - } - - #[test] - fn load_config_reads_yaml_defaults_from_current_dir() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - r#" -graphs: - local: - uri: ./demo.omni - bearer_token_env: DEMO_TOKEN -auth: - env_file: .env.omni -cli: - graph: local - branch: main - output_format: kv - table_max_column_width: 40 - table_cell_layout: wrap -policy: {} -"#, - ) - .unwrap(); - - let config = load_config_in(temp.path(), None, None, false).unwrap(); - assert_eq!(config.cli_graph_name(), Some("local")); - assert_eq!(config.cli_branch(), "main"); - assert_eq!(config.cli_output_format(), ReadOutputFormat::Kv); - assert_eq!(config.table_max_column_width(), 40); - assert_eq!(config.table_cell_layout(), TableCellLayout::Wrap); - assert_eq!( - config.graph_bearer_token_env(None, None, config.cli_graph_name()), - Some("DEMO_TOKEN") - ); - assert_eq!( - config.resolve_auth_env_file().unwrap(), - temp.path().join(".env.omni") - ); - assert_eq!( - PathBuf::from( - config - .resolve_target_uri(None, None, config.cli_graph_name()) - .unwrap() - ), - temp.path().join("./demo.omni") - ); - } - - #[test] - fn load_config_does_not_walk_parent_directories() { - let temp = tempdir().unwrap(); - let child = temp.path().join("child"); - fs::create_dir_all(&child).unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./demo.omni\n", - ) - .unwrap(); - - let config = load_config_in(&child, None, None, false).unwrap(); - assert!(config.graphs.is_empty()); - } - - #[test] - fn graph_resource_id_for_selection_uses_name_or_anonymous_uri() { - assert_eq!( - graph_resource_id_for_selection(Some("local"), "/tmp/graph.omni"), - "local" - ); - assert_eq!( - graph_resource_id_for_selection(None, "/tmp/graph.omni"), - "/tmp/graph.omni" - ); - } - - #[test] - fn resolve_graph_selection_validates_membership_and_coherence() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./demo.omni\n", - ) - .unwrap(); - let config = load_config_in(temp.path(), None, None, false).unwrap(); - - // A known graph passes through unchanged. - assert_eq!(config.resolve_graph_selection(Some("local")).unwrap(), Some("local")); - // An anonymous selection stays anonymous (→ top-level registry downstream). - assert_eq!(config.resolve_graph_selection(None).unwrap(), None); - // An unknown name errors, naming the graph (matching resolve_target_uri). - let err = config.resolve_graph_selection(Some("ghost")).unwrap_err().to_string(); - assert!( - err.contains("ghost") && err.contains("not found"), - "unknown graph must error naming it: {err}" - ); - - // Coherence: a named graph plus a populated top-level block is the - // config server boot refuses, so the gate rejects it too (shared rule - // via ensure_top_level_blocks_honored). An anonymous selection still - // passes — top-level is honored when no graph is named. - let temp2 = tempdir().unwrap(); - fs::write( - temp2.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./demo.omni\npolicy:\n file: ./top.yaml\n", - ) - .unwrap(); - let incoherent = load_config_in(temp2.path(), None, None, false).unwrap(); - let err = incoherent - .resolve_graph_selection(Some("local")) - .unwrap_err() - .to_string(); - assert!( - err.contains("local") && err.contains("policy.file"), - "named graph + populated top-level block must be rejected, naming both: {err}" - ); - assert_eq!( - incoherent.resolve_graph_selection(None).unwrap(), - None, - "anonymous selection still honors top-level" - ); - } - - #[test] - fn policy_tooling_graph_selection_prefers_cli_then_server_and_validates() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./local.omni\n prod:\n uri: ./prod.omni\n\ - server:\n graph: local\ncli:\n graph: prod\n", - ) - .unwrap(); - let config = load_config_in(temp.path(), None, None, false).unwrap(); - assert_eq!( - config.resolve_policy_tooling_graph_selection().unwrap(), - Some("prod") - ); - - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./local.omni\nserver:\n graph: local\n", - ) - .unwrap(); - let config = load_config_in(temp.path(), None, None, false).unwrap(); - assert_eq!( - config.resolve_policy_tooling_graph_selection().unwrap(), - Some("local") - ); - - let temp = tempdir().unwrap(); - fs::write(temp.path().join("omnigraph.yaml"), "policy: {}\n").unwrap(); - let config = load_config_in(temp.path(), None, None, false).unwrap(); - assert_eq!(config.resolve_policy_tooling_graph_selection().unwrap(), None); - - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./local.omni\nserver:\n graph: ghost\n", - ) - .unwrap(); - let config = load_config_in(temp.path(), None, None, false).unwrap(); - let err = config - .resolve_policy_tooling_graph_selection() - .unwrap_err() - .to_string(); - assert!( - err.contains("ghost") && err.contains("not found"), - "unknown server.graph must use graph-selection validation: {err}" - ); - } - - #[test] - fn resolve_query_path_searches_config_roots() { - let temp = tempdir().unwrap(); - fs::create_dir_all(temp.path().join("queries")).unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "query:\n roots:\n - queries\npolicy: {}\n", - ) - .unwrap(); - fs::write( - temp.path().join("queries").join("test.gq"), - "query q { return {} }", - ) - .unwrap(); - - let config = load_config_in(temp.path(), None, None, false).unwrap(); - let resolved = config.resolve_query_path(Path::new("test.gq")).unwrap(); - assert_eq!(resolved, temp.path().join("queries").join("test.gq")); - } - - #[test] - fn resolve_query_path_prefers_config_base_dir_over_ambient_cwd() { - let workspace = tempdir().unwrap(); - let config_dir = workspace.path().join("config"); - let ambient_dir = workspace.path().join("ambient"); - fs::create_dir_all(&config_dir).unwrap(); - fs::create_dir_all(&ambient_dir).unwrap(); - fs::write(config_dir.join("omnigraph.yaml"), "policy: {}\n").unwrap(); - fs::write(config_dir.join("local.gq"), "query local { return {} }").unwrap(); - fs::write(ambient_dir.join("local.gq"), "query ambient { return {} }").unwrap(); - - let config = - load_config_in(&ambient_dir, Some(&config_dir.join("omnigraph.yaml")), None, false).unwrap(); - let resolved = config.resolve_query_path(Path::new("local.gq")).unwrap(); - - assert_eq!(resolved, config_dir.join("local.gq")); - } - - #[test] - fn queries_block_round_trips_inline_and_per_graph() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - r#" -graphs: - prod: - uri: s3://bucket/prod - queries: - find_user: - file: ./queries/find_user.gq - mcp: - expose: true - tool_name: lookup_user - internal_audit: - file: ./queries/audit.gq -queries: - single_mode_q: - file: ./q.gq -"#, - ) - .unwrap(); - - let config = load_config_in(temp.path(), None, None, false).unwrap(); - - // Per-graph registry (multi-graph mode). - let prod = config.target_query_entries("prod").unwrap(); - assert_eq!(prod.len(), 2); - let find_user = &prod["find_user"]; - assert_eq!(find_user.file, "./queries/find_user.gq"); - assert!(find_user.mcp.expose); - assert_eq!(find_user.mcp.tool_name.as_deref(), Some("lookup_user")); - // Default exposure is true (the manifest entry is the opt-in); tool_name absent. - let audit = &prod["internal_audit"]; - assert!(audit.mcp.expose); - assert!(audit.mcp.tool_name.is_none()); - - // Top-level registry (single-graph mode). - assert_eq!(config.query_entries().len(), 1); - - // The shared selector resolves the same blocks the server boot - // and the CLI use: a known graph → its per-graph block; no - // selection or an unknown name → the top-level block (the latter - // pins the behavior of the CLI's now-deleted fallback arm). - assert_eq!(config.query_entries_for(Some("prod")).len(), 2); - assert_eq!(config.query_entries_for(None).len(), 1); - assert_eq!(config.query_entries_for(Some("nonexistent")).len(), 1); - - // Path resolution joins against base_dir, like policy files. - assert_eq!( - config.resolve_query_file(&find_user.file), - temp.path().join("./queries/find_user.gq") - ); - } - - #[test] - fn resolve_policy_file_for_follows_identity() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "policy:\n file: ./top.yaml\ngraphs:\n prod:\n uri: s3://b/prod\n \ - policy:\n file: ./prod.yaml\n bare:\n uri: s3://b/bare\n", - ) - .unwrap(); - let config = load_config_in(temp.path(), None, None, false).unwrap(); - - // Named graph with its own policy → per-graph (not top-level). - assert!( - config - .resolve_policy_file_for(Some("prod")) - .unwrap() - .ends_with("prod.yaml") - ); - // Named graph with NO per-graph policy → None (no top-level fallback; - // load-bearing for the boot coherence check). - assert!(config.resolve_policy_file_for(Some("bare")).is_none()); - // Anonymous (bare URI) or an unknown name → top-level. - assert!( - config - .resolve_policy_file_for(None) - .unwrap() - .ends_with("top.yaml") - ); - assert!( - config - .resolve_policy_file_for(Some("nope")) - .unwrap() - .ends_with("top.yaml") - ); - } - - #[test] - fn queries_block_absent_yields_empty_registry() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "graphs:\n local:\n uri: ./demo.omni\n", - ) - .unwrap(); - - let config = load_config_in(temp.path(), None, None, false).unwrap(); - // Additive: no `queries:` anywhere → empty registries everywhere. - assert!(config.query_entries().is_empty()); - assert!( - config - .target_query_entries("local") - .unwrap() - .is_empty() - ); - } - - #[test] - fn policy_block_accepts_non_empty_mapping() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - "policy:\n file: ./policy.yaml\n", - ) - .unwrap(); - - let config = load_config_in(temp.path(), None, None, false).unwrap(); - assert_eq!( - config.resolve_policy_file().unwrap(), - temp.path().join("policy.yaml") - ); - } - - #[test] - fn scoped_auth_env_ignores_default_target_when_uri_is_explicit() { - let temp = tempdir().unwrap(); - fs::write( - temp.path().join("omnigraph.yaml"), - r#" -graphs: - demo: - uri: https://example.com - bearer_token_env: DEMO_TOKEN -cli: - graph: demo -"#, - ) - .unwrap(); - - let config = load_config_in(temp.path(), None, None, false).unwrap(); - assert_eq!( - config.graph_bearer_token_env( - Some("https://override.example.com"), - None, - config.cli_graph_name() - ), - None - ); - assert_eq!( - config.graph_bearer_token_env( - Some("https://override.example.com"), - Some("demo"), - config.cli_graph_name() - ), - Some("DEMO_TOKEN") - ); - } -} diff --git a/crates/omnigraph-server/src/handlers.rs b/crates/omnigraph-server/src/handlers.rs index 2ead0e3..7de38d2 100644 --- a/crates/omnigraph-server/src/handlers.rs +++ b/crates/omnigraph-server/src/handlers.rs @@ -51,25 +51,15 @@ pub(crate) async fn server_graphs_list( State(state): State, actor: Option>, ) -> std::result::Result, ApiError> { - // 405 in single mode — there's no registry to enumerate, and the - // legacy URL surface didn't expose this endpoint. - let registry = match state.routing() { - GraphRouting::Single { .. } => { - return Err(ApiError::method_not_allowed( - "GET /graphs is only available in multi-graph mode", - )); - } - GraphRouting::Multi { registry, .. } => registry, - }; + let registry = &state.routing().registry; - // Server-level Cedar gate. `state.server_policy` is loaded from - // `server.policy.file` in `omnigraph.yaml` at startup. When no - // server policy is configured, `authorize_request_server` falls - // through to the MR-723 default-deny semantics (every non-Read - // action denied for an authenticated actor). `GraphList` is not - // `Read`, so without a server policy the request gets 403 — which - // is the right default (don't leak the registry until the operator - // explicitly authorizes it). + // Server-level Cedar gate. `state.server_policy` is loaded from the + // cluster-scoped policy bundle at startup. When no server policy is + // configured, `authorize_request_server` falls through to the MR-723 + // default-deny semantics (every non-Read action denied for an + // authenticated actor). `GraphList` is not `Read`, so without a server + // policy the request gets 403 — which is the right default (don't leak + // the registry until the operator explicitly authorizes it). authorize_request( actor.as_ref().map(|Extension(actor)| actor), state.server_policy.as_deref(), @@ -93,17 +83,15 @@ pub(crate) async fn server_graphs_list( } pub(crate) async fn server_openapi(State(state): State) -> Json { - let mut doc = ApiDoc::openapi(); + // `served_openapi` is the single nesting source — the protected + // routes always live under `/graphs/{graph_id}/...` (public/management + // paths `/healthz`, `/graphs` stay flat). Building from it here means + // the runtime spec and the committed `openapi.json` share one nesting + // pass and can't drift. + let mut doc = crate::served_openapi(); if !state.requires_bearer_auth() { strip_security(&mut doc); } - // MR-668: in multi mode, the protected routes live under - // `/graphs/{graph_id}/...`. Rewrite the doc so the spec matches - // the routes the router actually serves. Public paths (`/healthz`) - // stay flat in both modes. - if matches!(state.routing(), GraphRouting::Multi { .. }) { - nest_paths_under_cluster_prefix(&mut doc); - } Json(doc) } @@ -248,16 +236,11 @@ pub(crate) async fn require_bearer_auth( Ok(next.run(request).await) } -/// Routing middleware (MR-668). Resolves the active graph for the -/// request and injects `Arc` as an extension so handlers can -/// extract it via `Extension>`. +/// Routing middleware (RFC-011 cluster-only). Resolves the active graph +/// for the request and injects `Arc` as an extension so +/// handlers can extract it via `Extension>`. /// -/// **Single mode**: the routing field holds the single handle directly. -/// Routes are flat; every request resolves to that handle, regardless -/// of the URI path. No registry walk, no sentinel key, no -/// programmer-error guard. -/// -/// **Multi mode**: routes are nested under `/graphs/{graph_id}/...`. The +/// Routes are always nested under `/graphs/{graph_id}/...`. The /// middleware extracts `{graph_id}` from the URI path and looks it up in /// the registry. Returns 404 if the graph is not registered. /// @@ -268,39 +251,33 @@ pub(crate) async fn resolve_graph_handle( mut request: Request, next: Next, ) -> std::result::Result { - let handle = match &state.routing { - GraphRouting::Single { handle } => Arc::clone(handle), - GraphRouting::Multi { registry, .. } => { - // `Router::nest("/graphs/{graph_id}", inner)` rewrites - // `request.uri().path()` to the inner suffix (e.g. `/snapshot`). - // The pre-rewrite URI is preserved in the `OriginalUri` - // request extension by axum's router; we read from there to - // extract `{graph_id}`. Fall back to the current URI only if - // the extension is missing, which shouldn't happen for - // nested routes but is safe defensive code. - let original_path: String = request - .extensions() - .get::() - .map(|OriginalUri(uri)| uri.path().to_string()) - .unwrap_or_else(|| request.uri().path().to_string()); - let graph_id_str = original_path - .strip_prefix("/graphs/") - .and_then(|rest| rest.split('/').next()) - .filter(|s| !s.is_empty()) - .ok_or_else(|| { - ApiError::bad_request( - "cluster route missing /graphs/{graph_id} prefix".to_string(), - ) - })?; - let graph_id = GraphId::try_from(graph_id_str.to_string()) - .map_err(|err| ApiError::bad_request(err.to_string()))?; - let key = GraphKey::cluster(graph_id.clone()); - match registry.get(&key) { - RegistryLookup::Ready(handle) => handle, - RegistryLookup::Gone => { - return Err(ApiError::not_found(format!("graph '{graph_id}' not found"))); - } - } + let registry = &state.routing.registry; + // `Router::nest("/graphs/{graph_id}", inner)` rewrites + // `request.uri().path()` to the inner suffix (e.g. `/snapshot`). + // The pre-rewrite URI is preserved in the `OriginalUri` + // request extension by axum's router; we read from there to + // extract `{graph_id}`. Fall back to the current URI only if + // the extension is missing, which shouldn't happen for + // nested routes but is safe defensive code. + let original_path: String = request + .extensions() + .get::() + .map(|OriginalUri(uri)| uri.path().to_string()) + .unwrap_or_else(|| request.uri().path().to_string()); + let graph_id_str = original_path + .strip_prefix("/graphs/") + .and_then(|rest| rest.split('/').next()) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + ApiError::bad_request("cluster route missing /graphs/{graph_id} prefix".to_string()) + })?; + let graph_id = GraphId::try_from(graph_id_str.to_string()) + .map_err(|err| ApiError::bad_request(err.to_string()))?; + let key = GraphKey::cluster(graph_id.clone()); + let handle = match registry.get(&key) { + RegistryLookup::Ready(handle) => handle, + RegistryLookup::Gone => { + return Err(ApiError::not_found(format!("graph '{graph_id}' not found"))); } }; @@ -382,22 +359,25 @@ pub(crate) fn authorize( // runtime state means the docstring contract on // `server_graphs_list` ("don't leak the registry until the // operator explicitly authorizes it") holds uniformly; the - // operator's only path to enabling it is configuring an - // explicit `server.policy.file` in omnigraph.yaml. + // operator's only path to enabling it is configuring a + // cluster-scoped policy bundle, applying the cluster, and + // restarting the server. if request.action.resource_kind() == PolicyResourceKind::Server { return Ok(Authz::Denied( - "server-scoped actions require an explicit `server.policy.file` \ - configured in omnigraph.yaml — the management surface is closed \ - by default in every runtime state, including --unauthenticated, \ - so that server topology is never exposed without operator opt-in." + "server-scoped actions require an explicit cluster policy bundle \ + applied with `omnigraph cluster apply` and served after restart — \ + the management surface is closed by default in every runtime state, \ + including --unauthenticated, so that server topology is never exposed \ + without operator opt-in." .to_string(), )); } if actor.is_some() && request.action != PolicyAction::Read { return Ok(Authz::Denied( "server runs in default-deny mode (bearer tokens configured but no \ - policy file). Only `read` actions are permitted; configure \ - `policy.file` in omnigraph.yaml to enable other actions." + applied policy bundle). Only `read` actions are permitted; configure \ + a graph or cluster policy bundle in the cluster config, run \ + `omnigraph cluster apply`, and restart the server to enable other actions." .to_string(), )); } @@ -510,7 +490,7 @@ pub(crate) fn deprecation_headers(successor_link: &'static str) -> [(HeaderName, operation_id = "read", request_body = ReadRequest, responses( - (status = 200, description = "Query results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", body = ReadOutput), + (status = 200, description = "Query results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", body = ReadOutput), (status = 400, description = "Bad request", body = ErrorOutput), (status = 401, description = "Unauthorized", body = ErrorOutput), (status = 403, description = "Forbidden", body = ErrorOutput), @@ -524,7 +504,7 @@ pub(crate) fn deprecation_headers(successor_link: &'static str) -> [(HeaderName, /// route is kept indefinitely for byte-stable back-compat. New integrations /// should target `POST /query`, which has clean field names (`query` / /// `name`) and a 400-on-mutation guard. Responses from this route include -/// `Deprecation: true` and `Link: ; rel="successor-version"` +/// `Deprecation: true` and `Link: ; rel="successor-version"` /// headers per RFC 9745 / RFC 8288 so SDKs and proxies can surface the /// signal. pub(crate) async fn server_read( @@ -544,7 +524,7 @@ pub(crate) async fn server_read( ) .await?; Ok(( - deprecation_headers("; rel=\"successor-version\""), + deprecation_headers("; rel=\"successor-version\""), Json(api::read_output(selected_name, &target, result)), )) } @@ -793,7 +773,7 @@ pub(crate) async fn run_query( operation_id = "change", request_body = ChangeRequest, responses( - (status = 200, description = "Mutation results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", body = ChangeOutput), + (status = 200, description = "Mutation results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", body = ChangeOutput), (status = 400, description = "Bad request", body = ErrorOutput), (status = 401, description = "Unauthorized", body = ErrorOutput), (status = 403, description = "Forbidden", body = ErrorOutput), @@ -809,7 +789,7 @@ pub(crate) async fn run_query( /// kept indefinitely for back-compat. New integrations should target /// `POST /mutate`, which has identical semantics and a name that pairs /// cleanly with `POST /query`. Responses from this route include -/// `Deprecation: true` and `Link: ; rel="successor-version"` +/// `Deprecation: true` and `Link: ; rel="successor-version"` /// headers per RFC 9745 / RFC 8288 so SDKs and proxies can surface the /// signal. pub(crate) async fn server_change( @@ -830,7 +810,7 @@ pub(crate) async fn server_change( ) .await?; Ok(( - deprecation_headers("; rel=\"successor-version\""), + deprecation_headers("; rel=\"successor-version\""), Json(output), )) } @@ -980,6 +960,22 @@ pub(crate) async fn server_invoke_query( let query_name = stored.name.clone(); let is_mutation = stored.is_mutation(); + // RFC-011 D3: the CLI verb asserts the stored query's kind. `query ` + // sends `expect_mutation: false`, `mutate ` sends `true`; a mismatch + // is rejected here so the wrong verb errors instead of silently running. + if let Some(expected) = req.expect_mutation { + if expected != is_mutation { + let (actual, verb) = if is_mutation { + ("mutation", "mutate") + } else { + ("read", "query") + }; + return Err(ApiError::bad_request(format!( + "'{query_name}' is a {actual} — use omnigraph {verb} {query_name}" + ))); + } + } + info!( graph = %handle.uri, actor = ?actor_ref.map(|a| a.actor_id.as_ref()), @@ -1117,12 +1113,16 @@ pub(crate) async fn server_schema_get( (status = 400, description = "Bad request", body = ErrorOutput), (status = 401, description = "Unauthorized", body = ErrorOutput), (status = 403, description = "Forbidden", body = ErrorOutput), + (status = 409, description = "Schema apply is disabled for cluster-backed serving; use `omnigraph cluster apply` and restart", body = ErrorOutput), (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), ), security(("bearer_token" = [])), )] /// Apply a schema migration. /// +/// Cluster-backed servers reject this route with `409 Conflict`; operators +/// must apply schema changes through `omnigraph cluster apply` and restart. +/// /// Diffs `schema_source` against the current schema and applies the resulting /// migration steps (add/drop type, add/drop column, etc.). **Destructive**: /// some steps drop data. Returns the list of steps applied; if `applied` is @@ -1149,6 +1149,17 @@ pub(crate) async fn server_schema_apply( target_branch: Some("main".to_string()), }, )?; + // Disable HTTP schema apply on cluster-backed serving AFTER the Cedar gate, + // so an unauthorized actor gets a 403 (not a 409 that would disclose the + // server is cluster-backed): 401 → 403 → 409, never leak topology before + // authorization. An authorized actor gets the actionable 409 signpost. + if state.routing().config_path.is_some() { + return Err(ApiError::conflict( + "server-side schema apply is disabled for cluster-backed serving; \ + update the cluster config, run `omnigraph cluster apply`, and restart \ + the server.", + )); + } let est_bytes = request.schema_source.len() as u64; let _admission = state .workload @@ -1180,49 +1191,44 @@ pub(crate) async fn server_schema_apply( .await .map_err(ApiError::from_omni)? }; + // Prompt index convergence (iss-848): schema apply records `@index` intent + // but defers the physical build. On a long-lived server, materialize it + // promptly rather than waiting for the next `optimize` cron — spawned + // detached so it never blocks or fails the apply response. Best-effort: a + // failure is logged and the index still converges on the next optimize. + // The CLI is one-shot, so it has no equivalent; its convergence path is the + // operator's optimize cadence. + if result.applied { + let engine = Arc::clone(&handle.engine); + tokio::spawn(async move { + if let Err(err) = engine.ensure_indices().await { + tracing::warn!( + target: "omnigraph::server", + error = %err, + "post-apply ensure_indices failed; indexes will converge on the next optimize", + ); + } + }); + } Ok(Json(schema_apply_output(handle.uri.as_str(), result))) } -#[utoipa::path( - post, - path = "/ingest", - tag = "mutations", - operation_id = "ingest", - request_body = IngestRequest, - responses( - (status = 200, description = "Ingest results", body = IngestOutput), - (status = 400, description = "Bad request", body = ErrorOutput), - (status = 401, description = "Unauthorized", body = ErrorOutput), - (status = 403, description = "Forbidden", body = ErrorOutput), - (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), - ), - security(("bearer_token" = [])), -)] -/// Bulk-load NDJSON data into a branch. -/// -/// `data` is NDJSON with one record per line. `mode` controls behavior on -/// existing rows: `merge` upserts by id (default), `append` blindly inserts, -/// `overwrite` replaces table contents. Branch creation is opt-in by -/// presence of `from`: with `from` set, a missing `branch` is created from -/// it; without `from`, `branch` must already exist — a missing branch is a -/// 404, never an implicit fork. **Destructive** when `mode` is `overwrite` -/// or when the load produces conflicting writes. -pub(crate) async fn server_ingest( - State(state): State, - Extension(handle): Extension>, - actor: Option>, - Json(request): Json, -) -> std::result::Result, ApiError> { +/// Shared body for `POST /load` (canonical) and `POST /ingest` (deprecated): +/// branch-exists / fork-if-`from` check, Cedar authorization, admission, the +/// bulk `load_as`, and the `IngestOutput` mapping. +async fn run_ingest( + state: AppState, + handle: Arc, + actor: Option<&ResolvedActor>, + request: IngestRequest, +) -> std::result::Result { let branch = request.branch.unwrap_or_else(|| "main".to_string()); let from = request.from; let mode = request.mode.unwrap_or(omnigraph::loader::LoadMode::Merge); let actor_arc = actor - .as_ref() - .map(|Extension(actor)| Arc::clone(&actor.actor_id)) + .map(|actor| Arc::clone(&actor.actor_id)) .unwrap_or_else(|| Arc::::from("anonymous")); - let actor_id = actor - .as_ref() - .map(|Extension(actor)| actor.actor_id.as_ref()); + let actor_id = actor.map(|actor| actor.actor_id.as_ref()); let branch_exists = { let db = &handle.engine; @@ -1244,7 +1250,7 @@ pub(crate) async fn server_ingest( ))); } Some(from) => authorize_request( - actor.as_ref().map(|Extension(actor)| actor), + actor, handle.policy.as_deref(), PolicyRequest { action: PolicyAction::BranchCreate, @@ -1255,7 +1261,7 @@ pub(crate) async fn server_ingest( } } authorize_request( - actor.as_ref().map(|Extension(actor)| actor), + actor, handle.policy.as_deref(), PolicyRequest { action: PolicyAction::Change, @@ -1276,12 +1282,98 @@ pub(crate) async fn server_ingest( .map_err(ApiError::from_omni)? }; - Ok(Json(ingest_output( + Ok(ingest_output( handle.uri.as_str(), &result, mode, actor_id.map(str::to_string), - ))) + )) +} + +#[utoipa::path( + post, + path = "/load", + tag = "mutations", + operation_id = "load", + request_body = IngestRequest, + responses( + (status = 200, description = "Load results", body = IngestOutput), + (status = 400, description = "Bad request", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "Forbidden", body = ErrorOutput), + (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +/// Bulk-load NDJSON data into a branch (canonical load endpoint). +/// +/// `data` is NDJSON with one record per line. `mode` controls behavior on +/// existing rows: `merge` upserts by id (default), `append` blindly inserts, +/// `overwrite` replaces table contents. Branch creation is opt-in by +/// presence of `from`: with `from` set, a missing `branch` is created from +/// it; without `from`, `branch` must already exist — a missing branch is a +/// 404, never an implicit fork. **Destructive** when `mode` is `overwrite` +/// or when the load produces conflicting writes. +/// +/// The legacy `POST /ingest` route has identical semantics and is kept as a +/// deprecated alias. +pub(crate) async fn server_load( + State(state): State, + Extension(handle): Extension>, + actor: Option>, + Json(request): Json, +) -> std::result::Result, ApiError> { + Ok(Json( + run_ingest( + state, + handle, + actor.as_ref().map(|Extension(actor)| actor), + request, + ) + .await?, + )) +} + +#[utoipa::path( + post, + path = "/ingest", + tag = "mutations", + operation_id = "ingest", + request_body = IngestRequest, + responses( + (status = 200, description = "Load results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", body = IngestOutput), + (status = 400, description = "Bad request", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "Forbidden", body = ErrorOutput), + (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +#[deprecated(note = "use POST /load instead; /ingest is kept indefinitely for back-compat")] +/// **Deprecated** — use [`POST /load`](#tag/mutations/operation/load) instead. +/// +/// Bulk-load NDJSON data into a branch. Behavior is unchanged; the route is +/// kept indefinitely for back-compat. New integrations should target +/// `POST /load`, which has identical semantics. Responses from this route +/// include `Deprecation: true` and `Link: ; rel="successor-version"` +/// headers per RFC 9745 / RFC 8288 so SDKs and proxies can surface the signal. +pub(crate) async fn server_ingest( + State(state): State, + Extension(handle): Extension>, + actor: Option>, + Json(request): Json, +) -> std::result::Result<([(HeaderName, HeaderValue); 2], Json), ApiError> { + let output = run_ingest( + state, + handle, + actor.as_ref().map(|Extension(actor)| actor), + request, + ) + .await?; + Ok(( + deprecation_headers("; rel=\"successor-version\""), + Json(output), + )) } #[utoipa::path( @@ -1663,4 +1755,3 @@ pub(crate) fn query_params_from_json( json_params_to_param_map(params_json, query_params, JsonParamMode::Standard) .map_err(|err| color_eyre::eyre::eyre!(err.to_string())) } - diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index 3bde2a7..5451b05 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -1,11 +1,10 @@ pub mod api; mod handlers; mod settings; -pub use settings::{load_server_settings, classify_server_runtime_state, server_config_is_multi, ServerRuntimeState}; +pub use settings::{load_server_settings, classify_server_runtime_state, ServerRuntimeState}; use settings::*; use handlers::*; pub mod auth; -pub mod config; pub mod graph_id; pub mod identity; pub mod policy; @@ -46,11 +45,6 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post}; use axum::{Json, Router}; use color_eyre::eyre::{Result, WrapErr, bail, eyre}; -pub use config::{ - AliasCommand, AliasConfig, CliDefaults, DEFAULT_CONFIG_FILE, OmnigraphConfig, PolicySettings, - ProjectConfig, QueryDefaults, ReadOutputFormat, ServerDefaults, TableCellLayout, TargetConfig, - graph_resource_id_for_selection, load_config, -}; use futures::stream; use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::error::{ManifestConflictDetails, ManifestErrorKind, OmniError}; @@ -107,7 +101,10 @@ fn hash_bearer_token(token: &str) -> BearerTokenHash { handlers::server_invoke_query, handlers::server_schema_apply, handlers::server_schema_get, - handlers::server_ingest, + handlers::server_load, + // deprecated; the #[deprecated] attribute on the handler surfaces as + // `deprecated: true` on the OpenAPI operation. + #[allow(deprecated)] handlers::server_ingest, handlers::server_branch_list, handlers::server_branch_create, handlers::server_branch_delete, @@ -119,6 +116,20 @@ fn hash_bearer_token(token: &str) -> BearerTokenHash { )] pub struct ApiDoc; +/// The canonical served OpenAPI shape (RFC-011 cluster-only): the static +/// `ApiDoc` with every protected path nested under `/graphs/{graph_id}/…` +/// and `cluster_`-prefixed operation ids. `/healthz` and `/graphs` stay +/// flat. This is the single source of nesting — both the runtime +/// `server_openapi` handler and the committed `openapi.json` derive from +/// it, so the published spec can never describe routes the server does +/// not serve. The handler additionally strips security in open mode; the +/// committed spec retains it. +pub fn served_openapi() -> utoipa::openapi::OpenApi { + let mut doc = ApiDoc::openapi(); + handlers::nest_paths_under_cluster_prefix(&mut doc); + doc +} + struct SecurityAddon; impl utoipa::Modify for SecurityAddon { @@ -140,11 +151,10 @@ const SERVER_SOURCE_VERSION: Option<&str> = option_env!("OMNIGRAPH_SOURCE_VERSIO #[derive(Debug, Clone)] pub struct ServerConfig { - /// Server topology + the graphs to open at startup. Single-mode - /// invocations (`omnigraph-server ` or `--target `) - /// produce `ServerConfigMode::Single`; multi-mode invocations - /// (`--config omnigraph.yaml` with a non-empty `graphs:` map and - /// no single-mode selector) produce `ServerConfigMode::Multi`. + /// Server topology + the graphs to open at startup. RFC-011 + /// cluster-only: the server always boots from a cluster + /// (`--cluster `) and serves N graphs under cluster + /// routes. pub mode: ServerConfigMode, pub bind: String, /// Operator opt-in for fully-unauthenticated dev mode (MR-723). @@ -158,49 +168,33 @@ pub struct ServerConfig { pub allow_unauthenticated: bool, } -/// What `load_server_settings` produces after applying the four-rule -/// mode inference matrix (MR-668 decision 2). +/// What `load_server_settings` produces. RFC-011 cluster-only: the +/// server always boots from a cluster's applied revision into a +/// multi-graph deployment (N ≥ 1 graphs). #[derive(Debug, Clone)] pub enum ServerConfigMode { - /// Legacy invocation — one graph at the given URI. Either: - /// * `omnigraph-server ` (CLI positional), or - /// * `omnigraph-server --target --config omnigraph.yaml`, or - /// * `omnigraph-server --config omnigraph.yaml` with `server.graph` - /// set to a named target. - Single { - uri: String, - /// Cedar graph resource id for the single graph. A named selection - /// uses the graph name; an anonymous URI uses the normalized URI to - /// preserve legacy single-graph policy identity. - graph_id: String, - /// Top-level `policy.file` (single-graph Cedar policy). - policy_file: Option, - /// Top-level stored-query registry, loaded and identity-checked - /// at settings-build time; type-checked against the schema when - /// the engine opens. - queries: QueryRegistry, - }, - /// Multi-graph invocation — `--config omnigraph.yaml` with a - /// non-empty `graphs:` map and no single-mode selector. + /// Cluster boot — `--cluster ` resolves the applied + /// revision into per-graph startup configs plus an optional + /// server-level policy. Multi { /// Per-graph startup configs, sorted by graph id (BTreeMap /// iteration order). The parallel-open loop iterates this. graphs: Vec, - /// Path to the config file the server was started from. Kept on - /// the mode so future runtime mutation (deferred — see release - /// notes) can locate the source of truth without re-parsing CLI - /// args. + /// The cluster boot source (config directory or storage root). + /// Kept on the mode so future runtime mutation (deferred — see + /// release notes) can locate the source of truth without + /// re-parsing CLI args. config_path: PathBuf, - /// `server.policy.file` (server-level Cedar policy for the - /// management endpoints). Wired into `GET /graphs` authorization. + /// Server-level Cedar policy for the management endpoints + /// (`GET /graphs`). Wired into `GET /graphs` authorization. server_policy: Option, }, } -/// Where a Cedar policy bundle comes from at startup. File-based for -/// omnigraph.yaml deployments; inline (digest-verified catalog content) -/// for cluster-mode boots, where the catalog may live on object storage -/// and the server must not re-read mutable state after the snapshot. +/// Where a Cedar policy bundle comes from at startup. Cluster-local files are +/// used during config application; inline digest-verified catalog content is +/// used for serving, where the catalog may live on object storage and the +/// server must not re-read mutable state after the snapshot. #[derive(Debug, Clone)] pub enum PolicySource { File(PathBuf), @@ -215,42 +209,34 @@ pub struct GraphStartupConfig { pub graph_id: String, pub uri: String, pub policy: Option, + /// Pre-resolved embedding config from an applied cluster provider profile. + /// Legacy config paths leave this unset and continue to use env resolution. + pub embedding: Option, /// Per-graph stored-query registry, loaded and identity-checked at /// settings-build time; type-checked against the schema when this /// graph's engine opens. pub queries: QueryRegistry, } -/// Runtime routing for the server. Single mode = legacy -/// `omnigraph-server ` invocation, one graph, flat HTTP routes. -/// Multi mode = `--config omnigraph.yaml` with a non-empty `graphs:` -/// map, N graphs, cluster routes (`/graphs/{graph_id}/...`). Mode is -/// determined at startup by `load_server_settings`. +/// Runtime routing for the server (RFC-011 cluster-only). Every +/// deployment serves cluster routes (`/graphs/{graph_id}/...`) backed by +/// a registry of N graphs (N ≥ 1). The single-graph convenience +/// constructors build a one-graph registry keyed by `default`; the +/// cluster boot path builds an N-graph registry. There is no longer a +/// flat-route mode. /// -/// In single mode the handle lives here directly — there is no -/// registry, no sentinel key, no walk-and-assert. In multi mode the -/// registry carries N handles and the middleware dispatches on the -/// URL's `{graph_id}` segment. +/// `config_path` is the boot source (the cluster directory or storage +/// root); preserved here so future runtime mutation (deferred) can find +/// the source of truth without re-parsing CLI args. The server treats +/// the source as operator-owned and never writes it. /// -/// Both modes share the same handler bodies — the routing middleware +/// All handler bodies are mode-agnostic — the routing middleware /// (`resolve_graph_handle`) injects `Arc` as a request -/// extension so handlers never see the routing discriminator. +/// extension by looking up the `{graph_id}` URL segment in the registry. #[derive(Clone)] -pub enum GraphRouting { - /// Single-graph deployment: one handle, flat routes (`/snapshot`, - /// `/read`, …). The `handle.uri` field carries the URI the engine - /// was opened from. Backward compatible with v0.6.0 deployments. - Single { handle: Arc }, - /// Multi-graph deployment: many handles, cluster routes - /// (`/graphs/{graph_id}/...`). `config_path` is the `omnigraph.yaml` - /// the server reads at startup; preserved here so future runtime - /// mutation (deferred) can find the source of truth without - /// re-parsing CLI args. The server treats the file as - /// operator-owned and never writes it. - Multi { - registry: Arc, - config_path: Option, - }, +pub struct GraphRouting { + pub registry: Arc, + pub config_path: Option, } #[derive(Clone)] @@ -266,12 +252,10 @@ pub struct AppState { /// see MR-668 decision Q6. workload: Arc, bearer_tokens: Arc<[(BearerTokenHash, Arc)]>, - /// Server-level Cedar policy. Used by management endpoints (`POST - /// /graphs`, `GET /graphs`) which act on the registry resource, - /// not on a per-graph resource. Loaded from `server.policy.file` - /// in `omnigraph.yaml`. `None` outside multi mode and when no - /// server policy is configured. Per-graph policies live on each - /// `GraphHandle.policy`. + /// Server-level Cedar policy. Used by management endpoints (`GET + /// /graphs`) which act on the registry resource, not on a per-graph + /// resource. Loaded from the cluster-scoped policy binding when + /// configured. Per-graph policies live on each `GraphHandle.policy`. server_policy: Option>, } @@ -496,11 +480,13 @@ impl AppState { )) } - /// Single-mode shared construction: wraps the bare engine + per-graph - /// policy in a `GraphHandle` carried directly by `GraphRouting::Single`. - /// Per-graph policy enforcement on the engine (MR-722) is re-applied - /// via `Omnigraph::with_policy` so HTTP and engine layers can never - /// diverge. + /// Single-graph convenience construction (RFC-011 cluster-only): + /// wraps the bare engine + per-graph policy in a `GraphHandle` keyed + /// by `default`, then builds a one-graph registry so the deployment + /// serves the same `/graphs/{graph_id}/...` cluster routes as any + /// other. Per-graph policy enforcement on the engine (MR-722) is + /// re-applied via `Omnigraph::with_policy` so HTTP and engine layers + /// can never diverge. fn build_single_mode( uri: String, db: Omnigraph, @@ -519,18 +505,13 @@ impl AppState { } else { db }; - // `GraphHandle.key` is required by the struct, but in single - // mode it is never a registry key (there's no registry) and - // never compared against user input (routes are flat, no - // `{graph_id}` parameter). The label appears only in tracing - // output from `resolve_graph_handle`. The literal below is a - // log label, not a routing key — when the future cluster - // catalog ships, single mode may carry the catalog-assigned - // id here instead. + // The convenience constructors address the single graph by the + // reserved id `default` — both the registry key and the URL + // segment (`/graphs/default/...`). let uri = normalize_root_uri(&uri).unwrap_or(uri); - let key = GraphKey::cluster( - GraphId::try_from("default").expect("'default' is a valid GraphId log label"), - ); + let graph_id = + GraphId::try_from("default").expect("'default' is a valid GraphId"); + let key = GraphKey::cluster(graph_id); let handle = Arc::new(GraphHandle { key, uri, @@ -538,8 +519,15 @@ impl AppState { policy: policy_engine, queries, }); + let registry = Arc::new( + GraphRegistry::from_handles(vec![handle]) + .expect("a single handle never collides on graph id"), + ); Self { - routing: GraphRouting::Single { handle }, + routing: GraphRouting { + registry, + config_path: None, + }, workload, bearer_tokens, server_policy: None, @@ -547,12 +535,11 @@ impl AppState { } /// Multi-mode constructor — used by the startup loop. Operators - /// reach this by invoking `omnigraph-server --config omnigraph.yaml` - /// with a non-empty `graphs:` map. + /// reach this by invoking `omnigraph-server --cluster `. /// /// Caller supplies the already-opened `GraphHandle`s and (optionally) - /// the path to the source config file. `server_policy` is loaded - /// from `server.policy.file` if configured. + /// the path to the source cluster. `server_policy` is loaded from the + /// cluster-scoped policy binding if configured. pub fn new_multi( handles: Vec>, bearer_tokens: Vec<(String, String)>, @@ -563,7 +550,7 @@ impl AppState { let bearer_tokens = hash_bearer_tokens(bearer_tokens); let registry = Arc::new(GraphRegistry::from_handles(handles)?); Ok(Self { - routing: GraphRouting::Multi { + routing: GraphRouting { registry, config_path, }, @@ -575,9 +562,7 @@ impl AppState { /// Runtime routing accessor. Handlers don't typically inspect this — /// they extract `Arc` via the routing middleware — but - /// `build_app` matches on it to decide flat vs nested route - /// mounting, and a handful of management endpoints (`GET /graphs`, - /// the OpenAPI cluster rewrite) match on the discriminant. + /// `server_graphs_list` reads the registry through it. pub fn routing(&self) -> &GraphRouting { &self.routing } @@ -591,13 +576,9 @@ impl AppState { } // Any per-graph policy also requires auth — otherwise the // policy gate would receive unauthenticated requests. Reading - // from `routing` is O(1) in both arms: single mode is a direct - // `handle.policy.is_some()` check, multi mode reads the - // cached `any_per_graph_policy` flag on the registry snapshot. - match &self.routing { - GraphRouting::Single { handle } => handle.policy.is_some(), - GraphRouting::Multi { registry, .. } => registry.snapshot_ref().any_per_graph_policy, - } + // the cached `any_per_graph_policy` flag off the registry + // snapshot is O(1). + self.routing.registry.snapshot_ref().any_per_graph_policy } fn authenticate_bearer_token(&self, provided_token: &str) -> Option { @@ -892,18 +873,6 @@ fn validate_and_attach( }) } -/// Format every load error (parse / identity failure) into a multi-line -/// boot-abort message. -fn format_registry_load_errors(label: &str, errors: &[queries::LoadError]) -> String { - let joined = errors - .iter() - .map(|e| e.to_string()) - .collect::>() - .join("\n "); - format!("graph '{label}': stored-query registry failed to load:\n {joined}") -} - - pub fn build_app(state: AppState) -> Router { // The per-graph protected routes, identical in single + multi mode. // Two middleware layers wrap them (outer first, inner last): @@ -934,9 +903,20 @@ pub fn build_app(state: AppState) -> Router { .route("/queries/{name}", post(server_invoke_query)) .route("/schema", get(server_schema_get)) .route("/schema/apply", post(server_schema_apply)) + .route( + "/load", + post(server_load).layer(DefaultBodyLimit::max(INGEST_REQUEST_BODY_LIMIT_BYTES)), + ) + // /ingest is the deprecated alias of /load; its handler carries + // #[deprecated] (OpenAPI operation flagged) and emits RFC 9745 + // Deprecation + RFC 8288 Link headers. Suppress the call-site warning. .route( "/ingest", - post(server_ingest).layer(DefaultBodyLimit::max(INGEST_REQUEST_BODY_LIMIT_BYTES)), + post({ + #[allow(deprecated)] + server_ingest + }) + .layer(DefaultBodyLimit::max(INGEST_REQUEST_BODY_LIMIT_BYTES)), ) .route( "/branches", @@ -958,13 +938,9 @@ pub fn build_app(state: AppState) -> Router { // Management endpoints (`GET /graphs`) live alongside the per-graph // router. They go through bearer auth but NOT through // `resolve_graph_handle` — they operate on the registry directly. - // The endpoint is mounted in both modes; in single mode the handler - // returns 405 so clients see "resource exists, wrong context" - // rather than 404 "no such resource." // // Runtime add/remove (`POST /graphs`, `DELETE /graphs/{id}`) is not - // exposed in v0.6.0 — operators add graphs by editing - // `omnigraph.yaml` and restarting. + // exposed — operators run `cluster apply` and restart. let management = Router::new() .route("/graphs", get(server_graphs_list)) .route_layer(middleware::from_fn_with_state( @@ -972,15 +948,11 @@ pub fn build_app(state: AppState) -> Router { require_bearer_auth, )); - // Mount the protected routes differently per mode: - // * Single → flat routes (legacy: `/snapshot`, `/read`, etc.) - // * Multi → nested under `/graphs/{graph_id}/...` - let protected: Router = match state.routing() { - GraphRouting::Single { .. } => per_graph_protected.merge(management), - GraphRouting::Multi { .. } => Router::new() - .nest("/graphs/{graph_id}", per_graph_protected) - .merge(management), - }; + // RFC-011 cluster-only: per-graph routes always nest under + // `/graphs/{graph_id}/...`; there are no flat single-graph routes. + let protected: Router = Router::new() + .nest("/graphs/{graph_id}", per_graph_protected) + .merge(management); Router::new() .route("/healthz", get(server_health)) @@ -1001,7 +973,6 @@ pub async fn serve(config: ServerConfig) -> Result<()> { // policy OR any per-graph policy file. Mirrors the // `requires_bearer_auth` semantics on AppState. let has_policy_configured = match &config.mode { - ServerConfigMode::Single { policy_file, .. } => policy_file.is_some(), ServerConfigMode::Multi { graphs, server_policy, @@ -1022,36 +993,14 @@ pub async fn serve(config: ServerConfig) -> Result<()> { ServerRuntimeState::DefaultDeny => warn!( "bearer tokens are configured but no policy file is set — running in \ default-deny mode (only `read` actions are permitted for authenticated \ - actors). Configure `policy.file` in omnigraph.yaml to enable Cedar rules." + actors). Configure a graph or cluster policy bundle in the cluster config, \ + run `omnigraph cluster apply`, and restart to enable Cedar rules." ), ServerRuntimeState::PolicyEnabled => {} } let bind = config.bind.clone(); let state = match config.mode { - ServerConfigMode::Single { - uri, - graph_id, - policy_file, - queries, - } => { - let uri_for_log = uri.clone(); - info!( - uri = %uri_for_log, - graph_id = %graph_id, - bind = %bind, - mode = "single", - "serving omnigraph" - ); - AppState::open_single_with_queries_for_graph_id( - uri, - tokens, - policy_file.as_ref(), - queries, - Some(graph_id), - ) - .await? - } ServerConfigMode::Multi { graphs, config_path, @@ -1059,7 +1008,7 @@ pub async fn serve(config: ServerConfig) -> Result<()> { } => { info!( bind = %bind, - mode = "multi", + mode = "cluster", graph_count = graphs.len(), config = %config_path.display(), "serving omnigraph" @@ -1142,6 +1091,11 @@ async fn open_single_graph(cfg: GraphStartupConfig) -> Result> let db = Omnigraph::open(&uri) .await .map_err(|err| color_eyre::eyre::eyre!("open graph '{}' at {}: {err}", graph_id, uri))?; + let db = if let Some(embedding) = cfg.embedding { + db.with_embedding_config(Arc::new(embedding)) + } else { + db + }; // Validate this graph's stored queries against the live schema and // resolve them to an attachable handle (refuse boot on breakage). @@ -1175,5 +1129,3 @@ async fn shutdown_signal() { } info!("shutdown signal received"); } - - diff --git a/crates/omnigraph-server/src/main.rs b/crates/omnigraph-server/src/main.rs index a138d12..482c9af 100644 --- a/crates/omnigraph-server/src/main.rs +++ b/crates/omnigraph-server/src/main.rs @@ -8,16 +8,10 @@ use omnigraph_server::{ServerConfig, init_tracing, load_server_settings, serve}; #[command(name = "omnigraph-server")] #[command(about = "HTTP server for the Omnigraph graph database")] struct Cli { - /// Graph URI - uri: Option, - #[arg(long)] - target: Option, - #[arg(long)] - config: Option, /// Boot from a cluster: either a config directory (storage resolved /// through cluster.yaml) or a storage-root URI directly /// (s3://bucket/prefix — config-free serving from the bucket). - /// Exclusive: cannot combine with , --target, or --config. + /// The server's only boot source (RFC-011 cluster-only). #[arg(long)] cluster: Option, #[arg(long)] @@ -36,14 +30,7 @@ async fn main() -> Result<()> { init_tracing(); let cli = Cli::parse(); - let settings: ServerConfig = load_server_settings( - cli.config.as_ref(), - cli.cluster.as_ref(), - cli.uri, - cli.target, - cli.bind, - cli.unauthenticated, - ) - .await?; + let settings: ServerConfig = + load_server_settings(cli.cluster.as_ref(), cli.bind, cli.unauthenticated).await?; serve(settings).await } diff --git a/crates/omnigraph-server/src/queries.rs b/crates/omnigraph-server/src/queries.rs index bf131c8..09d2491 100644 --- a/crates/omnigraph-server/src/queries.rs +++ b/crates/omnigraph-server/src/queries.rs @@ -13,7 +13,6 @@ //! Renaming either is a breaking change to callers, by design. use std::collections::BTreeMap; -use std::fs; use std::sync::Arc; use omnigraph_compiler::catalog::Catalog; @@ -22,8 +21,6 @@ use omnigraph_compiler::query::parser::parse_query; use omnigraph_compiler::query::typecheck::typecheck_query_decl; use omnigraph_compiler::types::{PropType, ScalarType}; -use crate::config::{OmnigraphConfig, QueryEntry}; - /// One loaded stored query. `source` is the full `.gq` file text — the /// invocation handler hands it to `run_query` / `run_mutate` verbatim, /// which reuse the same parse/IR/exec path as the inline routes (no @@ -68,8 +65,9 @@ pub struct QueryRegistry { by_name: BTreeMap, } -/// In-memory registry entry before file I/O. Used by [`QueryRegistry::load`] -/// (after reading each `.gq` from disk) and directly by tests. +/// In-memory registry spec: a query's name + already-read `.gq` source. The +/// input to [`QueryRegistry::from_specs`] — built by the server's cluster boot +/// and by the CLI's `queries` tooling from a cluster serving snapshot. #[derive(Debug, Clone)] pub struct RegistrySpec { pub name: String, @@ -169,47 +167,6 @@ impl QueryRegistry { } } - /// Read each registry entry's `.gq` file from disk and build the - /// registry. `entries` is either the top-level `queries` map (single - /// mode) or a graph's `queries` map (multi mode); `config` resolves - /// each entry's relative `file:` path against `base_dir`. - pub fn load( - config: &OmnigraphConfig, - entries: &BTreeMap, - ) -> Result> { - let mut specs = Vec::with_capacity(entries.len()); - let mut errors = Vec::new(); - for (name, entry) in entries { - let path = config.resolve_query_file(&entry.file); - match fs::read_to_string(&path) { - Ok(source) => specs.push(RegistrySpec { - name: name.clone(), - source, - expose: entry.mcp.expose, - tool_name: entry.mcp.tool_name.clone(), - }), - Err(err) => errors.push(LoadError { - query: Some(name.clone()), - message: format!("cannot read '{}': {err}", path.display()), - }), - } - } - - // Parse/identity/uniqueness-check the readable specs even when some - // files failed to read, so every broken entry (I/O, parse, identity, - // tool-name collision) surfaces in one pass rather than one per - // restart. I/O errors come first (in `entries` key order), then the - // spec errors. A non-empty `errors` always fails the load. - match Self::from_specs(specs) { - Ok(registry) if errors.is_empty() => Ok(registry), - Ok(_) => Err(errors), - Err(spec_errors) => { - errors.extend(spec_errors); - Err(errors) - } - } - } - pub fn lookup(&self, name: &str) -> Option<&StoredQuery> { self.by_name.get(name) } @@ -653,36 +610,4 @@ embedding: Vector(4) assert!(entry2.params.is_empty(), "no declared params → empty list"); } - // --- load() error collection (file I/O + parse in one pass) --- - - #[test] - fn load_collects_io_and_parse_errors_in_one_pass() { - use crate::config::load_config; - let temp = tempfile::tempdir().unwrap(); - std::fs::write( - temp.path().join("good.gq"), - "query good() { match { $u: User } return { $u.name } }", - ) - .unwrap(); - std::fs::write(temp.path().join("broken.gq"), "query broken( {{ not valid").unwrap(); - // `missing.gq` is deliberately not written (an I/O failure). - std::fs::write( - temp.path().join("omnigraph.yaml"), - "queries:\n good:\n file: ./good.gq\n \ - missing:\n file: ./missing.gq\n broken:\n file: ./broken.gq\n", - ) - .unwrap(); - let config = load_config(Some(&temp.path().join("omnigraph.yaml"))).unwrap(); - - let errors = QueryRegistry::load(&config, config.query_entries()).unwrap_err(); - let joined = errors.iter().map(|e| e.to_string()).collect::>().join("\n"); - // Both the missing file AND the parse error surface in one pass — - // the I/O failure must not mask the parse failure. - assert!(joined.contains("missing"), "I/O error must surface: {joined}"); - assert!( - joined.contains("broken") && joined.contains("parse error"), - "the parse error in a readable file must surface in the same pass: {joined}" - ); - assert!(!joined.contains("'good'"), "the valid entry is not an error: {joined}"); - } } diff --git a/crates/omnigraph-server/src/settings.rs b/crates/omnigraph-server/src/settings.rs index 59c437b..bb6febd 100644 --- a/crates/omnigraph-server/src/settings.rs +++ b/crates/omnigraph-server/src/settings.rs @@ -1,14 +1,13 @@ -//! Server settings: omnigraph.yaml/CLI/env resolution, mode inference -//! (single vs multi vs cluster), bearer-token sources, and runtime-state -//! classification (moved verbatim from lib.rs in the modularization). +//! Server settings: cluster/CLI/env resolution, bearer-token sources, and +//! runtime-state classification (moved verbatim from lib.rs in the +//! modularization). use super::*; /// Build serving settings from a cluster directory's applied revision /// (RFC-005 §D2): graphs at derived roots, stored queries from verified /// catalog blob content, policy bundles from blob paths with their applied -/// bindings. Always multi-graph routing. The unauthenticated/env handling -/// matches the omnigraph.yaml path. +/// bindings. Always multi-graph routing. pub(crate) async fn load_cluster_settings( cluster_dir: &PathBuf, cli_bind: Option, @@ -99,6 +98,15 @@ pub(crate) async fn load_cluster_settings( graph_id: graph.graph_id.clone(), uri: graph.root.to_string_lossy().to_string(), policy: graph_policies.get(&graph.graph_id).cloned(), + embedding: graph + .embedding + .as_ref() + .map(|profile| { + profile.resolve().map_err(|err| { + eyre!("embedding provider for graph '{}': {err}", graph.graph_id) + }) + }) + .transpose()?, queries: registry, }); } @@ -122,162 +130,24 @@ pub(crate) async fn load_cluster_settings( }) } +/// RFC-011 cluster-only boot: the server serves exclusively from a +/// cluster's applied revision (`--cluster `). The legacy +/// omnigraph.yaml / `--target` / positional-URI single-graph boot paths +/// were removed — a deployment serves from exactly one source. pub async fn load_server_settings( - config_path: Option<&PathBuf>, cli_cluster: Option<&PathBuf>, - cli_uri: Option, - cli_target: Option, cli_bind: Option, cli_allow_unauthenticated: bool, ) -> Result { - // Rule 0 (RFC-005): --cluster is an exclusive boot source. It is checked - // before anything reads omnigraph.yaml — in cluster mode that file is - // never opened, not even the implicit current-directory search. - if let Some(cluster_dir) = cli_cluster { - if cli_uri.is_some() || cli_target.is_some() || config_path.is_some() { - bail!( - "--cluster is an exclusive boot source; it cannot combine with a graph URI, --target, or --config (axiom 15: a deployment serves from one source)" - ); - } - return load_cluster_settings(cluster_dir, cli_bind, cli_allow_unauthenticated).await; - } - let config = load_config(config_path)?; - let bind = cli_bind.unwrap_or_else(|| config.server_bind().to_string()); - // Either `--unauthenticated` or `OMNIGRAPH_UNAUTHENTICATED=1` flips - // this. Treat any non-empty, non-"0"/"false" string as truthy — - // standard 12-factor "any value is true" reading of the env var. - let env_unauth = std::env::var("OMNIGRAPH_UNAUTHENTICATED") - .ok() - .map(|v| { - let trimmed = v.trim(); - !trimmed.is_empty() && trimmed != "0" && !trimmed.eq_ignore_ascii_case("false") - }) - .unwrap_or(false); - let allow_unauthenticated = cli_allow_unauthenticated || env_unauth; - - // MR-668 decision 2 — four-rule mode inference matrix. - // - // 1. CLI `` positional → Single (URI = the value) - // 2. CLI `--target ` → Single (URI = graphs..uri) - // 3. `server.graph` in config → Single (URI = graphs..uri) - // 4. `--config` + non-empty `graphs:` + no single-mode selector - // → Multi (every entry in `graphs:`) - // 5. otherwise → error with migration hint - // - // Rules 1-3 are mutually compatible (CLI URI wins over `--target` - // wins over `server.graph`), reusing the existing - // `resolve_target_uri` precedence. - let has_cli_uri = cli_uri.is_some(); - let has_cli_target = cli_target.is_some(); - let has_server_graph = config.server_graph_name().is_some(); - let has_graphs_map = !config.graphs.is_empty(); - let has_explicit_config = config_path.is_some(); - - let mode = if has_cli_uri || has_cli_target || has_server_graph { - // Rules 1, 2, or 3 → Single mode. - let raw_uri = config.resolve_target_uri( - cli_uri, - cli_target.as_deref(), - config.server_graph_name(), - )?; - let uri = normalize_root_uri(&raw_uri).wrap_err_with(|| { - format!("normalize single-graph URI '{raw_uri}' from server settings") - })?; - // Config follows graph IDENTITY, not mode: a bare URI is anonymous - // (top-level config); a graph chosen by name uses its per-graph - // `graphs..{policy,queries}`. `resolve_target_uri` already - // errored on an unknown name, so a `Some(name)` here is a known graph. - let selected: Option<&str> = if has_cli_uri { - None - } else { - cli_target.as_deref().or_else(|| config.server_graph_name()) - }; - // A named selection must not leave a populated top-level block - // silently unused — refuse boot and point at the per-graph block. The - // same rule the CLI selection gate enforces, shared via one helper so - // the boot check and `omnigraph queries validate`/`list` can't drift. - config.ensure_top_level_blocks_honored(selected)?; - // Load + identity-check now (no engine needed); the schema - // type-check happens when the engine opens. - let policy_file = config.resolve_policy_file_for(selected); - let queries = QueryRegistry::load(&config, config.query_entries_for(selected)) - .map_err(|errs| color_eyre::eyre::eyre!(format_registry_load_errors(&uri, &errs)))?; - let graph_id = graph_resource_id_for_selection(selected, &uri); - ServerConfigMode::Single { - uri, - graph_id, - policy_file, - queries, - } - } else if has_explicit_config && has_graphs_map { - // Multi mode: every graph uses its per-graph block; top-level - // policy/queries are never honored, so a populated one is an error. - let unhonored = config.populated_top_level_blocks(); - if !unhonored.is_empty() { - bail!( - "multi-graph mode: top-level {} {} not honored — each graph uses its own \ - `graphs..…` block. Move per-graph rules there (and any \ - `graph_list` policy to `server.policy.file`).", - unhonored.join(" and "), - if unhonored.len() == 1 { "is" } else { "are" }, - ); - } - // Rule 4 → Multi mode. Build a startup config per graph. - let mut graphs = Vec::with_capacity(config.graphs.len()); - for (name, target) in &config.graphs { - // Validate the graph id can construct a `GraphId` newtype. - // Doing this here (not at registry insert) so a malformed - // omnigraph.yaml fails at startup with a clear error. - GraphId::try_from(name.clone()).map_err(|err| { - color_eyre::eyre::eyre!("invalid graph id '{name}' in omnigraph.yaml: {err}") - })?; - let raw_uri = config.resolve_uri_value(&target.uri); - let uri = normalize_root_uri(&raw_uri).wrap_err_with(|| { - format!("normalize URI '{raw_uri}' for graph '{name}' in omnigraph.yaml") - })?; - // Per-graph `queries:`, selected through the shared - // `query_entries_for` so server and CLI resolve identically. - // Load + identity-check now; the schema type-check happens - // when this graph's engine opens. - let queries = QueryRegistry::load(&config, config.query_entries_for(Some(name.as_str()))) - .map_err(|errs| color_eyre::eyre::eyre!(format_registry_load_errors(name, &errs)))?; - graphs.push(GraphStartupConfig { - graph_id: name.clone(), - uri, - policy: config.resolve_target_policy_file(name).map(PolicySource::File), - queries, - }); - } - let config_path = config_path - .cloned() - .expect("has_explicit_config implies config_path is Some"); - let server_policy = config.resolve_server_policy_file().map(PolicySource::File); - ServerConfigMode::Multi { - graphs, - config_path, - server_policy, - } - } else { - // Rule 5 → error with migration hint. + let Some(cluster_dir) = cli_cluster else { bail!( - "no graph to serve: pass a URI (`omnigraph-server `), select a target \ - (`--target --config omnigraph.yaml`), set `server.graph: ` in \ - omnigraph.yaml, or for multi-graph mode add a `graphs:` map to the config \ - file referenced by `--config`." + "omnigraph-server boots from a cluster: pass --cluster \ + (the cluster's applied revision is the deployment artifact). The legacy \ + single-graph boot (positional , --target, --config omnigraph.yaml) \ + was removed in RFC-011." ); }; - - Ok(ServerConfig { - mode, - bind, - allow_unauthenticated, - }) -} - -/// Whether the loaded config will run the server in multi-graph mode. -/// Useful for the test that constructs `ServerConfig` directly. -pub fn server_config_is_multi(config: &ServerConfig) -> bool { - matches!(config.mode, ServerConfigMode::Multi { .. }) + load_cluster_settings(cluster_dir, cli_bind, cli_allow_unauthenticated).await } /// MR-723 server runtime state, classified from the three-state matrix @@ -327,14 +197,15 @@ pub fn classify_server_runtime_state( "server has no bearer tokens and no policy file configured. This is a fully \ open server — pass `--unauthenticated` (or set OMNIGRAPH_UNAUTHENTICATED=1) \ if you actually want that, otherwise configure bearer tokens (see \ - docs/user/server.md) and/or `policy.file` in omnigraph.yaml." + docs/user/operations/server.md) and a graph or cluster policy bundle in \ + the cluster config, then run `omnigraph cluster apply` and restart." ), (false, false, true) => Ok(ServerRuntimeState::Open), (true, false, _) => Ok(ServerRuntimeState::DefaultDeny), (false, true, _) => bail!( "policy file is configured but no bearer tokens — every request would 401 \ because no token can ever match. Configure at least one bearer token (see \ - docs/user/server.md), or remove the policy file. To deny all unauthenticated \ + docs/user/operations/server.md), or remove the policy file. To deny all unauthenticated \ traffic deliberately, configure tokens plus a deny-all Cedar rule — that \ produces meaningful 403s with policy-decision logging instead of silent 401s." ), @@ -417,8 +288,8 @@ pub(crate) fn server_bearer_tokens_from_env() -> Result> { mod tests { use super::{ GraphStartupConfig, ServerConfig, ServerConfigMode, ServerRuntimeState, - classify_server_runtime_state, hash_bearer_token, load_server_settings, - normalize_bearer_token, parse_bearer_tokens_json, serve, server_bearer_tokens_from_env, + classify_server_runtime_state, hash_bearer_token, normalize_bearer_token, + parse_bearer_tokens_json, serve, server_bearer_tokens_from_env, }; use serial_test::serial; use std::env; @@ -577,108 +448,15 @@ mod tests { } #[tokio::test] - async fn server_settings_load_from_yaml_config() { - let temp = tempdir().unwrap(); - let config = temp.path().join("omnigraph.yaml"); - fs::write( - &config, - r#" -graphs: - local: - uri: /tmp/demo.omni -server: - graph: local - bind: 0.0.0.0:9090 -"#, - ) - .unwrap(); - - let settings = load_server_settings(Some(&config), None, None, None, None, false).await.unwrap(); - match &settings.mode { - ServerConfigMode::Single { uri, graph_id, .. } => { - assert_eq!(uri, "/tmp/demo.omni"); - assert_eq!(graph_id, "local"); - } - ServerConfigMode::Multi { .. } => panic!("expected Single mode, got Multi"), - } - assert_eq!(settings.bind, "0.0.0.0:9090"); - } - - #[tokio::test] - async fn server_settings_cli_flags_override_yaml_config() { - let temp = tempdir().unwrap(); - let config = temp.path().join("omnigraph.yaml"); - fs::write( - &config, - r#" -graphs: - local: - uri: /tmp/demo.omni -server: - graph: local - bind: 127.0.0.1:8080 -"#, - ) - .unwrap(); - - let settings = load_server_settings( - Some(&config), - None, - Some("/tmp/override.omni".to_string()), - None, - Some("0.0.0.0:9999".to_string()), - false, - ) - .await - .unwrap(); - match &settings.mode { - ServerConfigMode::Single { uri, graph_id, .. } => { - assert_eq!(uri, "/tmp/override.omni"); - assert_eq!(graph_id, "/tmp/override.omni"); - } - ServerConfigMode::Multi { .. } => panic!("expected Single mode, got Multi"), - } - assert_eq!(settings.bind, "0.0.0.0:9999"); - } - - #[tokio::test] - async fn server_settings_can_resolve_named_target() { - let temp = tempdir().unwrap(); - let config = temp.path().join("omnigraph.yaml"); - fs::write( - &config, - r#" -graphs: - local: - uri: ./demo.omni - dev: - uri: http://127.0.0.1:8080 -server: - graph: local - bind: 127.0.0.1:8080 -"#, - ) - .unwrap(); - - let settings = - load_server_settings(Some(&config), None, None, Some("dev".to_string()), None, false) - .await - .unwrap(); - match &settings.mode { - ServerConfigMode::Single { uri, graph_id, .. } => { - assert_eq!(uri, "http://127.0.0.1:8080"); - assert_eq!(graph_id, "dev"); - } - ServerConfigMode::Multi { .. } => panic!("expected Single mode, got Multi"), - } - } - - #[tokio::test] - async fn server_settings_require_uri_from_cli_or_config() { - let error = load_server_settings(None, None, None, None, None, false).await.unwrap_err(); + async fn server_settings_require_cluster_boot_source() { + // RFC-011 cluster-only: with no --cluster the server refuses to + // start and names the cluster-required remedy. + let error = super::load_server_settings(None, None, false) + .await + .unwrap_err(); assert!( - error.to_string().contains("no graph to serve"), - "expected mode-inference error, got: {error}", + error.to_string().contains("boots from a cluster"), + "expected cluster-required error, got: {error}", ); } @@ -748,6 +526,7 @@ server: .to_string_lossy() .into_owned(), policy: None, + embedding: None, queries: crate::queries::QueryRegistry::default(), }], config_path: temp.path().join("omnigraph.yaml"), @@ -788,17 +567,22 @@ server: ]); let temp = tempdir().unwrap(); // Graph path doesn't need to exist — classifier fires before - // `AppState::open_with_bearer_tokens_and_policy`. + // any engine open. let config = ServerConfig { - mode: ServerConfigMode::Single { - uri: temp - .path() - .join("graph.omni") - .to_string_lossy() - .into_owned(), - graph_id: "default".to_string(), - policy_file: None, - queries: crate::queries::QueryRegistry::default(), + mode: ServerConfigMode::Multi { + graphs: vec![GraphStartupConfig { + graph_id: "default".to_string(), + uri: temp + .path() + .join("graph.omni") + .to_string_lossy() + .into_owned(), + policy: None, + embedding: None, + queries: crate::queries::QueryRegistry::default(), + }], + config_path: temp.path().join("cluster"), + server_policy: None, }, bind: "127.0.0.1:0".to_string(), allow_unauthenticated: false, @@ -813,75 +597,6 @@ server: ); } - #[tokio::test] - #[serial] - async fn unauthenticated_env_var_classification() { - // MR-723 PR A: closes the gap where the env-var read path inside - // `load_server_settings` was structurally implemented but not - // exercised by any test. Three properties to pin, all in one - // sequential test because `cargo test` runs the mod test suite - // in parallel and `OMNIGRAPH_UNAUTHENTICATED` is process-global - // — interleaving with another test that sets the same env var - // (concurrent classifier tests, even the bearer-token suite - // sharing `EnvGuard`) corrupts the read. Sequential within one - // test fn is the simplest race-free shape. - let temp = tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - local: - uri: /tmp/demo-unauth.omni -server: - graph: local -"#, - ) - .unwrap(); - - // Truthy values flip Open mode on, even with CLI flag off. - for value in ["1", "true", "yes", "TRUE", "anything"] { - let _guard = EnvGuard::set(&[("OMNIGRAPH_UNAUTHENTICATED", Some(value))]); - let settings = load_server_settings(Some(&config_path), None, None, None, None, false).await - .expect("settings load should succeed"); - assert!( - settings.allow_unauthenticated, - "OMNIGRAPH_UNAUTHENTICATED={value:?} should enable Open mode", - ); - } - - // Falsy values keep refusal behavior, even with CLI flag off. - for value in ["0", "false", "FALSE", ""] { - let _guard = EnvGuard::set(&[("OMNIGRAPH_UNAUTHENTICATED", Some(value))]); - let settings = load_server_settings(Some(&config_path), None, None, None, None, false).await - .expect("settings load should succeed"); - assert!( - !settings.allow_unauthenticated, - "OMNIGRAPH_UNAUTHENTICATED={value:?} should NOT enable Open mode", - ); - } - - // Unset env var: also false. - let _guard = EnvGuard::set(&[("OMNIGRAPH_UNAUTHENTICATED", None)]); - let settings = load_server_settings(Some(&config_path), None, None, None, None, false).await - .expect("settings load should succeed"); - assert!( - !settings.allow_unauthenticated, - "OMNIGRAPH_UNAUTHENTICATED unset should NOT enable Open mode", - ); - drop(_guard); - - // CLI flag wins even when env is falsy — `serve()` honors the - // OR of both inputs. - let _guard = EnvGuard::set(&[("OMNIGRAPH_UNAUTHENTICATED", Some("0"))]); - let settings = load_server_settings(Some(&config_path), None, None, None, None, true).await - .expect("settings load should succeed"); - assert!( - settings.allow_unauthenticated, - "--unauthenticated CLI flag should win even when env is falsy", - ); - } - #[test] fn classify_policy_enabled_requires_tokens() { // State 3: tokens + policy → PolicyEnabled, regardless of the diff --git a/crates/omnigraph-server/tests/auth_policy.rs b/crates/omnigraph-server/tests/auth_policy.rs index 05c0c56..5cbbb97 100644 --- a/crates/omnigraph-server/tests/auth_policy.rs +++ b/crates/omnigraph-server/tests/auth_policy.rs @@ -50,7 +50,7 @@ async fn protected_routes_require_bearer_token() { let (status, body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -85,7 +85,7 @@ async fn protected_routes_accept_valid_bearer_token_while_healthz_stays_open() { let (status, body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .header("authorization", "Bearer demo-token") .body(Body::empty()) @@ -108,7 +108,7 @@ async fn protected_routes_accept_any_configured_team_bearer_token() { let (status, body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .header("authorization", "Bearer token-two") .body(Body::empty()) @@ -158,7 +158,7 @@ rules: let (ok_status, _) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-a") .body(Body::empty()) @@ -172,7 +172,7 @@ rules: let (denied_status, denied_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-b") .body(Body::empty()) @@ -190,7 +190,7 @@ rules: let (bad_status, _) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer wrong-token") .body(Body::empty()) @@ -245,7 +245,7 @@ rules: let (spoof_up_status, spoof_up_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-b") .header("x-actor-id", "act-a") @@ -270,7 +270,7 @@ rules: let (spoof_down_status, _) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-a") .header("x-actor-id", "act-b") @@ -290,7 +290,7 @@ rules: let (empty_spoof_status, _) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-b") .header("x-actor-id", "") @@ -316,7 +316,7 @@ async fn policy_allows_read_but_distinguishes_401_from_403() { let (missing_status, missing_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -332,7 +332,7 @@ async fn policy_allows_read_but_distinguishes_401_from_403() { let (snapshot_status, snapshot_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .header("authorization", "Bearer team-token") .body(Body::empty()) @@ -350,7 +350,7 @@ async fn policy_allows_read_but_distinguishes_401_from_403() { let (forbidden_status, forbidden_body) = json_response( &app, Request::builder() - .uri("/export") + .uri(g("/export")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -369,7 +369,7 @@ async fn policy_allows_read_but_distinguishes_401_from_403() { .clone() .oneshot( Request::builder() - .uri("/export") + .uri(g("/export")) .method(Method::POST) .header("authorization", "Bearer admin-token") .header("content-type", "application/json") @@ -410,7 +410,7 @@ async fn policy_uses_resolved_branch_for_snapshot_reads() { let (status, body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -458,7 +458,7 @@ async fn policy_blocks_change_on_protected_main_but_allows_unprotected_branch() let (main_status, main_body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -482,7 +482,7 @@ async fn policy_blocks_change_on_protected_main_but_allows_unprotected_branch() let (feature_status, feature_body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -533,7 +533,7 @@ async fn policy_blocks_non_admin_merge_to_main_and_allows_admin() { let (deny_status, deny_body) = json_response( &app, Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -551,7 +551,7 @@ async fn policy_blocks_non_admin_merge_to_main_and_allows_admin() { let (allow_status, allow_body) = json_response( &app, Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header("authorization", "Bearer admin-token") .header("content-type", "application/json") @@ -578,7 +578,7 @@ async fn authenticated_change_stamps_actor_on_commits() { let (change_status, change_body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("authorization", "Bearer token-one") .header("content-type", "application/json") @@ -592,7 +592,7 @@ async fn authenticated_change_stamps_actor_on_commits() { let (commits_status, commits_body) = json_response( &app, Request::builder() - .uri("/commits?branch=main") + .uri(g("/commits?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-one") .body(Body::empty()) @@ -623,7 +623,7 @@ async fn authenticated_branch_merge_stamps_merge_actor_on_head_commit() { let (create_status, _) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::POST) .header("authorization", "Bearer token-one") .header("content-type", "application/json") @@ -642,7 +642,7 @@ async fn authenticated_branch_merge_stamps_merge_actor_on_head_commit() { let (change_status, _) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("authorization", "Bearer token-one") .header("content-type", "application/json") @@ -659,7 +659,7 @@ async fn authenticated_branch_merge_stamps_merge_actor_on_head_commit() { let (merge_status, merge_body) = json_response( &app, Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header("authorization", "Bearer token-two") .header("content-type", "application/json") @@ -673,7 +673,7 @@ async fn authenticated_branch_merge_stamps_merge_actor_on_head_commit() { let (commit_status, commit_body) = json_response( &app, Request::builder() - .uri("/commits?branch=main") + .uri(g("/commits?branch=main")) .method(Method::GET) .header("authorization", "Bearer token-two") .body(Body::empty()) @@ -691,7 +691,6 @@ async fn authenticated_branch_merge_stamps_merge_actor_on_head_commit() { #[tokio::test(flavor = "multi_thread")] async fn engine_layer_policy_fires_via_direct_arc_omnigraph_from_new_single() { - use omnigraph_server::GraphRouting; let temp = init_loaded_graph().await; let graph = graph_path(temp.path()); let db = Omnigraph::open(graph.to_str().unwrap()).await.unwrap(); @@ -717,9 +716,14 @@ async fn engine_layer_policy_fires_via_direct_arc_omnigraph_from_new_single() { // embedded consumer holding `Arc` would. If `new_single` // failed to apply `with_policy` to the engine, this `mutate_as` // would succeed — the HTTP-layer is bypassed entirely. - let handle = match state.routing() { - GraphRouting::Single { handle } => Arc::clone(handle), - GraphRouting::Multi { .. } => panic!("expected single-mode routing"), + // RFC-011 cluster-only: the single-graph convenience constructor + // registers the graph under the reserved id `default`. + let key = omnigraph_server::GraphKey::cluster( + omnigraph_server::GraphId::try_from("default").unwrap(), + ); + let handle = match state.routing().registry.get(&key) { + omnigraph_server::RegistryLookup::Ready(handle) => handle, + omnigraph_server::RegistryLookup::Gone => panic!("default graph must be registered"), }; let engine = Arc::clone(&handle.engine); @@ -758,7 +762,7 @@ async fn oversized_request_body_returns_payload_too_large() { .clone() .oneshot( Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(oversized)) @@ -781,7 +785,7 @@ async fn default_deny_mode_allows_read_for_authenticated_actor() { let (status, _body) = json_response( &app, Request::builder() - .uri("/snapshot") + .uri(g("/snapshot")) .method(Method::GET) .header(AUTHORIZATION, "Bearer demo-token") .body(Body::empty()) @@ -808,7 +812,7 @@ async fn default_deny_mode_rejects_change_with_forbidden() { let (status, body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header(AUTHORIZATION, "Bearer demo-token") .header("content-type", "application/json") @@ -840,7 +844,7 @@ async fn default_deny_mode_rejects_schema_apply_with_forbidden() { let (status, body) = json_response( &app, Request::builder() - .uri("/schema/apply") + .uri(g("/schema/apply")) .method(Method::POST) .header(AUTHORIZATION, "Bearer demo-token") .header("content-type", "application/json") diff --git a/crates/omnigraph-server/tests/boot_settings.rs b/crates/omnigraph-server/tests/boot_settings.rs index 3869d27..4ccc8da 100644 --- a/crates/omnigraph-server/tests/boot_settings.rs +++ b/crates/omnigraph-server/tests/boot_settings.rs @@ -18,10 +18,7 @@ use support::*; mod multi_graph_startup { use super::*; use omnigraph::storage::normalize_root_uri; - use omnigraph_server::{ - GraphHandle, GraphId, GraphKey, GraphRegistry, InsertError, ServerConfig, ServerConfigMode, - load_server_settings, - }; + use omnigraph_server::{GraphHandle, GraphId, GraphKey, GraphRegistry, InsertError}; use std::sync::Arc; async fn build_multi_mode_app(graph_ids: &[&str]) -> (Vec, Router) { @@ -280,10 +277,11 @@ mod multi_graph_startup { ); } - /// Flat routes 404 in multi mode — the router only mounts under - /// `/graphs/{graph_id}/...` so `/snapshot` doesn't resolve. + /// RFC-011 cluster-only: flat per-graph routes never resolve — the + /// router only mounts under `/graphs/{graph_id}/...` so a root + /// `/snapshot` returns 404. #[tokio::test(flavor = "multi_thread")] - async fn flat_routes_404_in_multi_mode() { + async fn flat_routes_404_at_root() { let (_dirs, app) = build_multi_mode_app(&["alpha"]).await; let resp = app .oneshot( @@ -298,28 +296,6 @@ mod multi_graph_startup { assert_eq!(resp.status(), StatusCode::NOT_FOUND); } - /// `GraphId` validation runs at startup — a reserved name in - /// `omnigraph.yaml` produces a clear error rather than getting - /// rejected per-request. - #[tokio::test] - async fn load_server_settings_rejects_reserved_graph_id() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - policies: - uri: /tmp/g1.omni -"#, - ) - .unwrap(); - let err = load_server_settings(Some(&config_path), None, None, None, None, false).await.unwrap_err(); - assert!( - err.to_string().contains("invalid graph id 'policies'"), - "expected reserved-name rejection, got: {err}" - ); - } #[tokio::test(flavor = "multi_thread")] async fn registry_rejects_duplicate_normalized_graph_uris() { @@ -375,372 +351,6 @@ graphs: assert_eq!(listed[0].uri, graph_uri); } - // ── Four-rule mode inference matrix ─────────────────────────────── - - /// Rule 1: CLI positional URI → Single. - #[tokio::test] - async fn mode_inference_cli_uri_is_single() { - let settings = load_server_settings( - None, - None, - Some("/tmp/cli.omni".to_string()), - None, - None, - true, // allow unauth so we get past the runtime-state check - ) - .await - .unwrap(); - match settings.mode { - ServerConfigMode::Single { uri, .. } => assert_eq!(uri, "/tmp/cli.omni"), - ServerConfigMode::Multi { .. } => panic!("expected Single (rule 1), got Multi"), - } - } - - /// Rule 2: --target picks one graph from `graphs:` map → Single. - #[tokio::test] - async fn mode_inference_cli_target_is_single() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - alpha: - uri: /tmp/alpha.omni - beta: - uri: /tmp/beta.omni -"#, - ) - .unwrap(); - let settings = - load_server_settings(Some(&config_path), None, None, Some("alpha".into()), None, true) - .await - .unwrap(); - match settings.mode { - ServerConfigMode::Single { uri, .. } => assert_eq!(uri, "/tmp/alpha.omni"), - ServerConfigMode::Multi { .. } => panic!("expected Single (rule 2), got Multi"), - } - } - - /// Rule 3: `server.graph` set → Single (target picked from config). - #[tokio::test] - async fn mode_inference_server_graph_is_single() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - alpha: - uri: /tmp/alpha.omni - beta: - uri: /tmp/beta.omni -server: - graph: beta -"#, - ) - .unwrap(); - let settings = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap(); - match settings.mode { - ServerConfigMode::Single { uri, .. } => assert_eq!(uri, "/tmp/beta.omni"), - ServerConfigMode::Multi { .. } => panic!("expected Single (rule 3), got Multi"), - } - } - - /// Rule 4: `--config` + non-empty `graphs:` + no single-mode selector → Multi. - #[tokio::test] - async fn mode_inference_config_plus_graphs_is_multi() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - alpha: - uri: /tmp/alpha.omni - beta: - uri: /tmp/beta.omni -"#, - ) - .unwrap(); - let settings = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap(); - match settings.mode { - ServerConfigMode::Multi { graphs, .. } => { - let ids: Vec<&str> = graphs.iter().map(|g| g.graph_id.as_str()).collect(); - // BTreeMap iteration order is alphabetical. - assert_eq!(ids, vec!["alpha", "beta"]); - } - ServerConfigMode::Single { .. } => panic!("expected Multi (rule 4), got Single"), - } - } - - #[tokio::test] - async fn mode_inference_multi_rejects_top_level_policy_file() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -policy: - file: ./policy.yaml -graphs: - alpha: - uri: /tmp/alpha.omni -"#, - ) - .unwrap(); - let err = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap_err(); - let msg = err.to_string(); - assert!( - msg.contains("top-level") && msg.contains("policy.file") && msg.contains("not honored"), - "expected top-level-not-honored guidance, got: {msg}" - ); - assert!( - msg.contains("graphs."), - "expected per-graph migration guidance, got: {msg}" - ); - assert!( - msg.contains("server.policy.file"), - "expected server policy migration guidance, got: {msg}" - ); - } - - #[tokio::test] - async fn mode_inference_multi_rejects_top_level_queries() { - // Symmetric to the policy guard: a top-level `queries:` block in - // multi-graph mode is not honored (each graph uses its own), so it - // is a loud error rather than a silent no-op. - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - "queries:\n q:\n file: ./q.gq\ngraphs:\n alpha:\n uri: /tmp/alpha.omni\n", - ) - .unwrap(); - let err = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap_err(); - let msg = err.to_string(); - assert!( - msg.contains("queries") && msg.contains("not honored"), - "top-level queries must be rejected in multi-graph mode: {msg}" - ); - } - - #[tokio::test] - async fn single_mode_named_graph_rejects_top_level_blocks() { - // Serving a graph by name (`--target`/`server.graph`) uses its - // per-graph block; a populated top-level block would be silently - // shadowed, so boot refuses and names the per-graph location. - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - "policy:\n file: ./top.yaml\ngraphs:\n prod:\n uri: /tmp/prod.omni\n", - ) - .unwrap(); - let err = - load_server_settings(Some(&config_path), None, None, Some("prod".to_string()), None, true) - .await - .unwrap_err(); - let msg = err.to_string(); - assert!( - msg.contains("prod") && msg.contains("policy.file") && msg.contains("graphs.prod"), - "named single-mode + top-level policy must refuse, naming the graph: {msg}" - ); - } - - #[tokio::test] - async fn single_mode_named_graph_uses_per_graph_policy_and_queries() { - // The identity rule: `--target prod` attaches `graphs.prod`'s own - // policy + queries, not the top-level ones (which are absent here). - let temp = tempfile::tempdir().unwrap(); - fs::write( - temp.path().join("prod.gq"), - "query pq() { match { $u: User } return { $u.name } }", - ) - .unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - "graphs:\n prod:\n uri: /tmp/prod.omni\n policy:\n file: ./prod-policy.yaml\n \ - queries:\n pq:\n file: ./prod.gq\n", - ) - .unwrap(); - let settings = - load_server_settings(Some(&config_path), None, None, Some("prod".to_string()), None, true) - .await - .unwrap(); - match settings.mode { - ServerConfigMode::Single { - graph_id, - policy_file, - queries, - .. - } => { - assert_eq!(graph_id, "prod", "named single-mode keeps graph identity"); - assert!( - policy_file - .as_ref() - .is_some_and(|p| p.ends_with("prod-policy.yaml")), - "per-graph policy attached: {policy_file:?}" - ); - assert!(queries.lookup("pq").is_some(), "per-graph query attached"); - } - other => panic!("expected Single mode, got {other:?}"), - } - } - - #[tokio::test] - async fn mode_inference_normalizes_multi_graph_uris() { - let temp = tempfile::tempdir().unwrap(); - let graph = temp.path().join("alpha.omni"); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - format!( - r#" -graphs: - alpha: - uri: file://{}/ -"#, - graph.display() - ), - ) - .unwrap(); - let settings = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap(); - match settings.mode { - ServerConfigMode::Multi { graphs, .. } => { - assert_eq!(graphs[0].uri, graph.to_string_lossy()); - } - ServerConfigMode::Single { .. } => panic!("expected Multi"), - } - } - - /// Rule 5: nothing → error with migration hint. - #[tokio::test] - async fn mode_inference_no_inputs_errors_with_migration_hint() { - let err = load_server_settings(None, None, None, None, None, true).await.unwrap_err(); - let msg = err.to_string(); - assert!( - msg.contains("no graph to serve"), - "expected migration-hint error, got: {msg}" - ); - } - - /// Rule 4 sub-case: `--config` with empty `graphs:` map and no - /// single-mode selector → rule 5 fires (no graph to serve). - #[tokio::test] - async fn mode_inference_empty_graphs_map_errors() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write(&config_path, "server:\n bind: 127.0.0.1:8080\n").unwrap(); - let err = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap_err(); - assert!(err.to_string().contains("no graph to serve")); - } - - /// `--config` + `` together: URI wins → Single (the CLI URI - /// takes precedence over the config's graphs map). - #[tokio::test] - async fn mode_inference_cli_uri_overrides_graphs_map() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - alpha: - uri: /tmp/alpha.omni -"#, - ) - .unwrap(); - let settings = load_server_settings( - Some(&config_path), - None, - Some("/tmp/cli-override.omni".to_string()), - None, - None, - true, - ) - .await - .unwrap(); - match settings.mode { - ServerConfigMode::Single { uri, .. } => { - assert_eq!( - uri, "/tmp/cli-override.omni", - "CLI URI must win over graphs: map" - ); - } - ServerConfigMode::Multi { .. } => { - panic!("expected Single (CLI URI wins), got Multi") - } - } - } - - /// Per-graph `policy.file` is resolved relative to the config base_dir. - #[tokio::test] - async fn per_graph_policy_file_is_resolved_relative_to_base_dir() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -graphs: - alpha: - uri: /tmp/alpha.omni - policy: - file: ./policies/alpha.yaml - beta: - uri: /tmp/beta.omni -"#, - ) - .unwrap(); - let settings = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap(); - let graphs = match settings.mode { - ServerConfigMode::Multi { graphs, .. } => graphs, - _ => panic!("expected Multi"), - }; - // graphs is BTreeMap-iter order (alphabetical). - let alpha = &graphs[0]; - let beta = &graphs[1]; - assert_eq!(alpha.graph_id, "alpha"); - let omnigraph_server::PolicySource::File(alpha_policy) = - alpha.policy.as_ref().unwrap() - else { - panic!("yaml-configured policy must stay file-based"); - }; - assert_eq!(alpha_policy, &temp.path().join("policies/alpha.yaml")); - assert_eq!(beta.graph_id, "beta"); - assert!(beta.policy.is_none()); - } - - /// `server.policy.file` resolves alongside the graphs map. - #[tokio::test] - async fn server_policy_file_is_resolved_relative_to_base_dir() { - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("omnigraph.yaml"); - fs::write( - &config_path, - r#" -server: - policy: - file: ./server-policy.yaml -graphs: - alpha: - uri: /tmp/alpha.omni -"#, - ) - .unwrap(); - let settings = load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap(); - match settings.mode { - ServerConfigMode::Multi { server_policy, .. } => { - let omnigraph_server::PolicySource::File(path) = server_policy.unwrap() else { - panic!("yaml-configured server policy must stay file-based"); - }; - assert_eq!(path, temp.path().join("server-policy.yaml")); - } - _ => panic!("expected Multi"), - } - } - /// `GET /graphs` must NOT leak the registry in Open mode without /// an explicit server policy. Operators who pass `--unauthenticated` /// opted into trusting the network for graph DATA, not for leaking @@ -786,28 +396,6 @@ graphs: ); } - /// `GET /graphs` returns 405 in single mode (resource exists in the - /// API surface, just not operational without a `graphs:` map). - #[tokio::test(flavor = "multi_thread")] - async fn get_graphs_returns_405_in_single_mode() { - let temp = init_loaded_graph().await; - let graph = graph_path(temp.path()); - let state = AppState::open(graph.to_string_lossy().to_string()) - .await - .unwrap(); - let app = build_app(state); - let resp = app - .oneshot( - Request::builder() - .method(Method::GET) - .uri("/graphs") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); - } /// `GET /graphs` requires bearer auth when tokens are configured. #[tokio::test(flavor = "multi_thread")] @@ -971,52 +559,4 @@ rules: ); } - /// Loads an `omnigraph.yaml` with two graphs and verifies multi-mode - /// inference plus graph entry resolution. Cluster-route dispatch is - /// covered by the route tests above. - #[tokio::test(flavor = "multi_thread")] - async fn server_settings_load_multi_graph_config_entries() { - let cfg_dir = tempfile::tempdir().unwrap(); - // Real graph storage dirs (the URIs in the config must point to - // a graph init-able location). - let alpha_dir = cfg_dir.path().join("alpha.omni"); - let beta_dir = cfg_dir.path().join("beta.omni"); - let schema = fs::read_to_string(fixture("test.pg")).unwrap(); - Omnigraph::init(alpha_dir.to_str().unwrap(), &schema) - .await - .unwrap(); - Omnigraph::init(beta_dir.to_str().unwrap(), &schema) - .await - .unwrap(); - - let config_path = cfg_dir.path().join("omnigraph.yaml"); - fs::write( - &config_path, - format!( - r#" -graphs: - alpha: - uri: {alpha} - beta: - uri: {beta} -"#, - alpha = alpha_dir.display(), - beta = beta_dir.display(), - ), - ) - .unwrap(); - - let settings: ServerConfig = - load_server_settings(Some(&config_path), None, None, None, None, true).await.unwrap(); - assert!(matches!(settings.mode, ServerConfigMode::Multi { .. })); - - match settings.mode { - ServerConfigMode::Multi { graphs, .. } => { - assert_eq!(graphs.len(), 2); - let ids: Vec<&str> = graphs.iter().map(|g| g.graph_id.as_str()).collect(); - assert_eq!(ids, vec!["alpha", "beta"]); - } - _ => unreachable!(), - } - } } diff --git a/crates/omnigraph-server/tests/data_routes.rs b/crates/omnigraph-server/tests/data_routes.rs index cef2f9a..65af2c6 100644 --- a/crates/omnigraph-server/tests/data_routes.rs +++ b/crates/omnigraph-server/tests/data_routes.rs @@ -63,7 +63,7 @@ async fn export_route_returns_jsonl_for_branch_snapshot() { .clone() .oneshot( Request::builder() - .uri("/export") + .uri(g("/export")) .method(Method::POST) .header("content-type", "application/json") .header("authorization", format!("Bearer {}", token)) @@ -99,7 +99,7 @@ async fn snapshot_route_returns_manifest_dataset_version() { let (snapshot_status, snapshot_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -131,7 +131,7 @@ async fn ingest_creates_branch_returns_metadata_and_stamps_actor() { let (status, body) = json_response( &app, Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("authorization", "Bearer token-one") .header("content-type", "application/json") @@ -195,7 +195,7 @@ async fn ingest_existing_branch_skips_branch_create_policy_check() { let (status, body) = json_response( &app, Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -223,7 +223,7 @@ async fn ingest_without_from_returns_404_for_missing_branch_and_creates_nothing( let (status, body) = json_response( &app, Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&ingest).unwrap())) @@ -264,7 +264,7 @@ async fn ingest_without_from_loads_into_existing_branch() { let (status, body) = json_response( &app, Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&ingest).unwrap())) @@ -294,7 +294,7 @@ async fn ingest_denies_missing_branch_without_branch_create_permission() { let (status, body) = json_response( &app, Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -327,7 +327,7 @@ async fn ingest_denies_when_actor_lacks_change_permission() { let (status, body) = json_response( &app, Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("authorization", "Bearer team-token") .header("content-type", "application/json") @@ -357,7 +357,7 @@ async fn ingest_rejects_payloads_over_32_mib() { .clone() .oneshot( Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&oversize).unwrap())) @@ -419,7 +419,7 @@ async fn branch_merge_conflict_response_includes_structured_conflicts() { let (status, body) = json_response( &app, Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&merge).unwrap())) @@ -451,7 +451,7 @@ async fn repeated_read_after_change_sees_updated_state_from_same_app() { let (change_status, change_body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&change).unwrap())) @@ -471,7 +471,7 @@ async fn repeated_read_after_change_sees_updated_state_from_same_app() { let (read_status, read_body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&read).unwrap())) @@ -497,7 +497,7 @@ async fn query_endpoint_runs_inline_read() { let (status, body) = json_response( &app, Request::builder() - .uri("/query") + .uri(g("/query")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&query).unwrap())) @@ -524,7 +524,7 @@ async fn query_endpoint_rejects_mutation_with_400() { let (status, body) = json_response( &app, Request::builder() - .uri("/query") + .uri(g("/query")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&query).unwrap())) @@ -555,7 +555,7 @@ async fn mutate_endpoint_runs_inline_mutation() { .clone() .oneshot( Request::builder() - .uri("/mutate") + .uri(g("/mutate")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&request).unwrap())) @@ -580,7 +580,7 @@ async fn mutate_endpoint_runs_inline_mutation() { #[tokio::test(flavor = "multi_thread")] async fn change_endpoint_emits_deprecation_headers() { // `/change` is kept indefinitely for back-compat but flagged at runtime - // per RFC 9745 (`Deprecation: true`) + RFC 8288 (`Link: ; + // per RFC 9745 (`Deprecation: true`) + RFC 8288 (`Link: ; // rel="successor-version"`). The OpenAPI side is covered by // `openapi_change_is_deprecated` in tests/openapi.rs. let (_temp, app) = app_for_loaded_graph().await; @@ -595,7 +595,7 @@ async fn change_endpoint_emits_deprecation_headers() { .clone() .oneshot( Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&request).unwrap())) @@ -615,11 +615,88 @@ async fn change_endpoint_emits_deprecation_headers() { ); assert_eq!( response.headers().get("link").and_then(|v| v.to_str().ok()), - Some("; rel=\"successor-version\""), + Some("; rel=\"successor-version\""), "POST /change must point at /mutate via `Link` rel=successor-version (RFC 8288)" ); } +#[tokio::test(flavor = "multi_thread")] +async fn load_endpoint_loads_into_existing_branch() { + // Canonical bulk-load endpoint (RFC-009 Phase 5). Same wire shape as + // /ingest, no deprecation signal. + let (_temp, app) = app_for_loaded_graph().await; + let request = IngestRequest { + branch: Some("main".to_string()), + from: None, + mode: Some(LoadMode::Merge), + data: r#"{"type":"Person","data":{"name":"Loaded","age":7}}"#.to_string(), + }; + let response = app + .clone() + .oneshot( + Request::builder() + .uri(g("/load")) + .method(Method::POST) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&request).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert!( + response.headers().get("deprecation").is_none(), + "POST /load must not advertise itself as deprecated" + ); + let body_bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let body: Value = serde_json::from_slice(&body_bytes).unwrap(); + assert_eq!(body["branch"], "main"); + assert_eq!(body["tables"][0]["table_key"], "node:Person"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ingest_endpoint_emits_deprecation_headers() { + // `/ingest` is the deprecated alias of `/load` (RFC-009 Phase 5): flagged + // at runtime per RFC 9745 (`Deprecation: true`) + RFC 8288 (`Link: ; + // rel="successor-version"`). The OpenAPI side is covered by + // `openapi_ingest_is_deprecated` in tests/openapi.rs. + let (_temp, app) = app_for_loaded_graph().await; + let request = IngestRequest { + branch: Some("main".to_string()), + from: None, + mode: Some(LoadMode::Merge), + data: r#"{"type":"Person","data":{"name":"Legacyer","age":33}}"#.to_string(), + }; + let response = app + .clone() + .oneshot( + Request::builder() + .uri(g("/ingest")) + .method(Method::POST) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&request).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get("deprecation") + .and_then(|v| v.to_str().ok()), + Some("true"), + "POST /ingest must advertise `Deprecation: true` (RFC 9745)" + ); + assert_eq!( + response.headers().get("link").and_then(|v| v.to_str().ok()), + Some("; rel=\"successor-version\""), + "POST /ingest must point at /load via `Link` rel=successor-version (RFC 8288)" + ); +} + #[tokio::test(flavor = "multi_thread")] async fn read_endpoint_emits_deprecation_headers() { // `/read` is kept indefinitely for byte-stable back-compat but flagged @@ -637,7 +714,7 @@ async fn read_endpoint_emits_deprecation_headers() { .clone() .oneshot( Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&request).unwrap())) @@ -657,7 +734,7 @@ async fn read_endpoint_emits_deprecation_headers() { ); assert_eq!( response.headers().get("link").and_then(|v| v.to_str().ok()), - Some("; rel=\"successor-version\""), + Some("; rel=\"successor-version\""), "POST /read must point at /query via `Link` rel=successor-version (RFC 8288)" ); } @@ -680,7 +757,7 @@ async fn query_endpoint_does_not_emit_deprecation_headers() { .clone() .oneshot( Request::builder() - .uri("/query") + .uri(g("/query")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&request).unwrap())) @@ -712,7 +789,7 @@ async fn change_endpoint_accepts_legacy_field_names() { let (status, body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&legacy_body).unwrap())) @@ -731,7 +808,7 @@ async fn change_endpoint_accepts_legacy_field_names() { let (status, body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&canonical_body).unwrap())) @@ -749,7 +826,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (list_status, list_body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -765,7 +842,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (create_status, create_body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&create).unwrap())) @@ -779,7 +856,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (list_status, list_body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -797,7 +874,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (change_status, change_body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&change).unwrap())) @@ -818,7 +895,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (read_status, read_body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&read_main_before).unwrap())) @@ -835,7 +912,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (merge_status, merge_body) = json_response( &app, Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&merge).unwrap())) @@ -857,7 +934,7 @@ async fn remote_branch_list_create_merge_flow_works() { let (read_status, read_body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&read_main_after).unwrap())) @@ -880,7 +957,7 @@ async fn remote_branch_delete_flow_works() { let (create_status, _) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&create).unwrap())) @@ -892,7 +969,7 @@ async fn remote_branch_delete_flow_works() { let (delete_status, delete_body) = json_response( &app, Request::builder() - .uri("/branches/feature") + .uri(g("/branches/feature")) .method(Method::DELETE) .body(Body::empty()) .unwrap(), @@ -904,7 +981,7 @@ async fn remote_branch_delete_flow_works() { let (list_status, list_body) = json_response( &app, Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -932,7 +1009,7 @@ async fn branch_delete_denies_without_policy_permission() { let (status, body) = json_response( &app, Request::builder() - .uri("/branches/feature") + .uri(g("/branches/feature")) .method(Method::DELETE) .header("authorization", "Bearer token-team") .body(Body::empty()) @@ -1004,7 +1081,7 @@ query vector_search_string($q: String) { let (status, body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&read).unwrap())) @@ -1057,7 +1134,7 @@ async fn change_conflict_returns_manifest_conflict_409() { let (status, body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from( @@ -1129,7 +1206,7 @@ async fn change_concurrent_inserts_same_key_serialize_without_409() { }) .unwrap(); let req = Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -1161,7 +1238,7 @@ async fn change_concurrent_inserts_same_key_serialize_without_409() { let (snapshot_status, snapshot_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -1242,7 +1319,7 @@ async fn change_concurrent_updates_same_key_serialize_via_publisher_cas() { }) .unwrap(); let req = Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -1351,7 +1428,7 @@ query insert_c($name: String) { }) .unwrap(); let req = Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -1368,7 +1445,7 @@ query insert_c($name: String) { }) .unwrap(); let req = Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -1397,7 +1474,7 @@ query insert_c($name: String) { let (status, body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -1505,7 +1582,7 @@ async fn ingest_per_actor_admission_cap_returns_429() { }) .unwrap(); let req = Request::builder() - .uri("/ingest") + .uri(g("/ingest")) .method(Method::POST) .header("authorization", "Bearer flooder-token") .header("content-type", "application/json") diff --git a/crates/omnigraph-server/tests/multi_graph.rs b/crates/omnigraph-server/tests/multi_graph.rs index 5ad847f..617cc66 100644 --- a/crates/omnigraph-server/tests/multi_graph.rs +++ b/crates/omnigraph-server/tests/multi_graph.rs @@ -5,9 +5,12 @@ use std::fs; use axum::body::{Body, to_bytes}; use axum::http::{Method, Request, StatusCode}; -use omnigraph_server::api::ErrorOutput; +use omnigraph::db::Omnigraph; +use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph_server::api::{ErrorOutput, ReadRequest}; use omnigraph_server::{AppState, build_app}; use serde_json::Value; +use serial_test::serial; use tower::ServiceExt; @@ -245,7 +248,7 @@ async fn concurrent_branch_ops_morphological_matrix() { .clone() .oneshot( Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -366,7 +369,7 @@ async fn concurrent_branch_ops_morphological_matrix() { .clone() .oneshot( Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -457,6 +460,180 @@ async fn cluster_boot_serves_applied_state() { assert_eq!(status, StatusCode::OK, "{body}"); } +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn cluster_boot_injects_embedding_provider_config() { + const EMBED_SCHEMA: &str = r#" +node Doc { + slug: String @key + title: String @index + embedding: Vector(4) @embed("title", model="cluster-mock") @index +} +"#; + const EMBED_QUERY: &str = r#" +query vector_search_string($q: String) { + match { $d: Doc } + return { $d.slug, $d.title } + order { nearest($d.embedding, $q) } + limit 3 +} +"#; + + let alpha = mock_embedding("alpha", 4); + let beta = mock_embedding("beta", 4); + let gamma = mock_embedding("gamma", 4); + let data = format!( + concat!( + r#"{{"type":"Doc","data":{{"slug":"alpha-doc","title":"alpha guide","embedding":[{}]}}}}"#, + "\n", + r#"{{"type":"Doc","data":{{"slug":"beta-doc","title":"beta guide","embedding":[{}]}}}}"#, + "\n", + r#"{{"type":"Doc","data":{{"slug":"gamma-doc","title":"gamma handbook","embedding":[{}]}}}}"# + ), + format_vector(&alpha), + format_vector(&beta), + format_vector(&gamma), + ); + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("docs.pg"), EMBED_SCHEMA).unwrap(); + fs::write(temp.path().join("search.gq"), EMBED_QUERY).unwrap(); + fs::write( + temp.path().join("cluster.yaml"), + r#" +version: 1 +providers: + embedding: + default: + kind: mock + model: cluster-mock +graphs: + knowledge: + schema: ./docs.pg + embedding_provider: default + queries: + vector_search_string: + file: ./search.gq +"#, + ) + .unwrap(); + let import = omnigraph_cluster::import_config_dir(temp.path()).await; + assert!(import.ok, "{:?}", import.diagnostics); + let apply = omnigraph_cluster::apply_config_dir(temp.path()).await; + assert!(apply.ok && apply.converged, "{:?}", apply.diagnostics); + + let graph_uri = temp + .path() + .join("graphs/knowledge.omni") + .to_string_lossy() + .to_string(); + let mut db = Omnigraph::open(&graph_uri).await.unwrap(); + load_jsonl(&mut db, &data, LoadMode::Overwrite) + .await + .unwrap(); + + let _guard = EnvGuard::set(&[ + ("OMNIGRAPH_EMBEDDINGS_MOCK", None), + ("OMNIGRAPH_EMBED_PROVIDER", None), + ("OMNIGRAPH_EMBED_BASE_URL", None), + ("OMNIGRAPH_EMBED_MODEL", None), + ("OPENROUTER_API_KEY", None), + ("OPENAI_API_KEY", None), + ("GEMINI_API_KEY", None), + ]); + let settings = cluster_settings(temp.path()).await.unwrap(); + let omnigraph_server::ServerConfigMode::Multi { + graphs, + config_path, + server_policy, + } = settings.mode + else { + panic!("cluster boot must select multi-graph routing"); + }; + let state = omnigraph_server::open_multi_graph_state( + graphs, + Vec::new(), + server_policy.as_ref(), + config_path, + ) + .await + .unwrap(); + let app = build_app(state); + + let read = ReadRequest { + query_source: EMBED_QUERY.to_string(), + query_name: Some("vector_search_string".to_string()), + params: Some(serde_json::json!({ "q": "alpha" })), + branch: Some("main".to_string()), + snapshot: None, + }; + let (status, body) = json_response( + &app, + Request::builder() + .uri("/graphs/knowledge/read") + .method(Method::POST) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&read).unwrap())) + .unwrap(), + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["row_count"], 3); + assert_eq!(body["rows"][0]["d.slug"], "alpha-doc"); +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn cluster_boot_refuses_missing_embedding_secret_env() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("people.pg"), + "\nnode Person {\n name: String @key\n}\n", + ) + .unwrap(); + fs::write( + temp.path().join("people.gq"), + "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n", + ) + .unwrap(); + fs::write( + temp.path().join("cluster.yaml"), + r#" +version: 1 +providers: + embedding: + default: + kind: openai-compatible + api_key: ${OG_TEST_MISSING_EMBED_KEY} +graphs: + knowledge: + schema: ./people.pg + embedding_provider: default + queries: + find_person: + file: ./people.gq +"#, + ) + .unwrap(); + let import = omnigraph_cluster::import_config_dir(temp.path()).await; + assert!(import.ok, "{:?}", import.diagnostics); + let apply = omnigraph_cluster::apply_config_dir(temp.path()).await; + assert!(apply.ok && apply.converged, "{:?}", apply.diagnostics); + + let _guard = EnvGuard::set(&[ + ("OG_TEST_MISSING_EMBED_KEY", None), + ("OMNIGRAPH_EMBEDDINGS_MOCK", None), + ]); + let err = cluster_settings(temp.path()).await.unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("embedding provider for graph 'knowledge'"), + "{message}" + ); + assert!(message.contains("OG_TEST_MISSING_EMBED_KEY"), "{message}"); +} + #[tokio::test] async fn cluster_boot_wires_policy_bindings_into_cedar_slots() { let temp = tempfile::tempdir().unwrap(); @@ -540,31 +717,15 @@ graphs: #[tokio::test] async fn cluster_boot_refusals() { - // Mutual exclusion with --config / URI. + // RFC-011 cluster-only: with no --cluster, boot refuses with the + // cluster-required remedy. + let err = omnigraph_server::load_server_settings(None, None, true) + .await + .unwrap_err(); + assert!(err.to_string().contains("boots from a cluster"), "{err}"); + let temp = converged_cluster_dir("").await; let dir = temp.path().to_path_buf(); - let err = omnigraph_server::load_server_settings( - Some(&dir.join("omnigraph.yaml")), - Some(&dir), - None, - None, - None, - true, - ) - .await - .unwrap_err(); - assert!(err.to_string().contains("exclusive boot source"), "{err}"); - let err = omnigraph_server::load_server_settings( - None, - Some(&dir), - Some("file:///tmp/x.omni".to_string()), - None, - None, - true, - ) - .await - .unwrap_err(); - assert!(err.to_string().contains("exclusive boot source"), "{err}"); // Tampered catalog blob refuses boot with the remedy. let blob_dir = dir.join("__cluster/resources/query/knowledge/find_person"); diff --git a/crates/omnigraph-server/tests/openapi.rs b/crates/omnigraph-server/tests/openapi.rs index 3d13e74..9276482 100644 --- a/crates/omnigraph-server/tests/openapi.rs +++ b/crates/omnigraph-server/tests/openapi.rs @@ -8,10 +8,9 @@ use axum::body::{Body, to_bytes}; use axum::http::{Method, Request, StatusCode}; use omnigraph::db::Omnigraph; use omnigraph::loader::{LoadMode, load_jsonl}; -use omnigraph_server::{ApiDoc, AppState, build_app}; +use omnigraph_server::{AppState, build_app, served_openapi}; use serde_json::Value; use tower::ServiceExt; -use utoipa::OpenApi; fn fixture(name: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -71,7 +70,10 @@ async fn json_response(app: &Router, request: Request) -> (StatusCode, Val } fn openapi_doc() -> utoipa::openapi::OpenApi { - ApiDoc::openapi() + // RFC-011 cluster-only: the canonical committed spec is the SERVED + // shape — protected routes nested under `/graphs/{graph_id}/…`, + // `/healthz` and `/graphs` flat. This matches what the server serves. + served_openapi() } fn openapi_json() -> Value { @@ -159,25 +161,28 @@ fn openapi_info_contains_version() { // Path coverage tests // --------------------------------------------------------------------------- +// The canonical served spec keeps `/healthz` and `/graphs` flat; every +// protected route nests under `/graphs/{graph_id}/…`. const EXPECTED_PATHS: &[&str] = &[ "/healthz", "/graphs", - "/snapshot", - "/read", - "/query", - "/export", - "/change", - "/mutate", - "/queries", - "/queries/{name}", - "/schema", - "/schema/apply", - "/ingest", - "/branches", - "/branches/{branch}", - "/branches/merge", - "/commits", - "/commits/{commit_id}", + "/graphs/{graph_id}/snapshot", + "/graphs/{graph_id}/read", + "/graphs/{graph_id}/query", + "/graphs/{graph_id}/export", + "/graphs/{graph_id}/change", + "/graphs/{graph_id}/mutate", + "/graphs/{graph_id}/queries", + "/graphs/{graph_id}/queries/{name}", + "/graphs/{graph_id}/schema", + "/graphs/{graph_id}/schema/apply", + "/graphs/{graph_id}/load", + "/graphs/{graph_id}/ingest", + "/graphs/{graph_id}/branches", + "/graphs/{graph_id}/branches/{branch}", + "/graphs/{graph_id}/branches/merge", + "/graphs/{graph_id}/commits", + "/graphs/{graph_id}/commits/{commit_id}", ]; #[test] @@ -221,25 +226,25 @@ fn openapi_healthz_is_get() { #[test] fn openapi_read_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/read"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/read"]["post"].is_object()); } #[test] fn openapi_export_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/export"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/export"]["post"].is_object()); } #[test] fn openapi_change_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/change"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/change"]["post"].is_object()); } #[test] fn openapi_mutate_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/mutate"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/mutate"]["post"].is_object()); } // Deprecation flagging — `/read` and `/change` are kept indefinitely for @@ -252,7 +257,7 @@ fn openapi_mutate_is_post() { fn openapi_read_is_deprecated() { let doc = openapi_json(); assert_eq!( - doc["paths"]["/read"]["post"]["deprecated"], + doc["paths"]["/graphs/{graph_id}/read"]["post"]["deprecated"], serde_json::Value::Bool(true), "/read must be flagged deprecated in OpenAPI; use /query instead" ); @@ -262,7 +267,7 @@ fn openapi_read_is_deprecated() { fn openapi_change_is_deprecated() { let doc = openapi_json(); assert_eq!( - doc["paths"]["/change"]["post"]["deprecated"], + doc["paths"]["/graphs/{graph_id}/change"]["post"]["deprecated"], serde_json::Value::Bool(true), "/change must be flagged deprecated in OpenAPI; use /mutate instead" ); @@ -271,7 +276,7 @@ fn openapi_change_is_deprecated() { #[test] fn openapi_query_is_not_deprecated() { let doc = openapi_json(); - let deprecated = doc["paths"]["/query"]["post"] + let deprecated = doc["paths"]["/graphs/{graph_id}/query"]["post"] .get("deprecated") .and_then(serde_json::Value::as_bool) .unwrap_or(false); @@ -284,7 +289,7 @@ fn openapi_query_is_not_deprecated() { #[test] fn openapi_mutate_is_not_deprecated() { let doc = openapi_json(); - let deprecated = doc["paths"]["/mutate"]["post"] + let deprecated = doc["paths"]["/graphs/{graph_id}/mutate"]["post"] .get("deprecated") .and_then(serde_json::Value::as_bool) .unwrap_or(false); @@ -297,38 +302,64 @@ fn openapi_mutate_is_not_deprecated() { #[test] fn openapi_ingest_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/ingest"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/ingest"]["post"].is_object()); +} + +#[test] +fn openapi_load_is_not_deprecated() { + // RFC-009 Phase 5: /load is the canonical bulk-load endpoint. + let doc = openapi_json(); + assert!(doc["paths"]["/graphs/{graph_id}/load"]["post"].is_object()); + let deprecated = doc["paths"]["/graphs/{graph_id}/load"]["post"] + .get("deprecated") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + assert!( + !deprecated, + "/load is the canonical load endpoint and must not be deprecated" + ); +} + +#[test] +fn openapi_ingest_is_deprecated() { + // RFC-009 Phase 5: /ingest is now the deprecated alias of /load. + let doc = openapi_json(); + assert_eq!( + doc["paths"]["/graphs/{graph_id}/ingest"]["post"]["deprecated"], + serde_json::Value::Bool(true), + "/ingest must be flagged deprecated now that /load is canonical" + ); } #[test] fn openapi_branches_supports_get_and_post() { let doc = openapi_json(); - assert!(doc["paths"]["/branches"]["get"].is_object()); - assert!(doc["paths"]["/branches"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/branches"]["get"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/branches"]["post"].is_object()); } #[test] fn openapi_branch_delete_is_delete() { let doc = openapi_json(); - assert!(doc["paths"]["/branches/{branch}"]["delete"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/branches/{branch}"]["delete"].is_object()); } #[test] fn openapi_branch_merge_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/branches/merge"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/branches/merge"]["post"].is_object()); } #[test] fn openapi_commits_is_get() { let doc = openapi_json(); - assert!(doc["paths"]["/commits"]["get"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/commits"]["get"].is_object()); } #[test] fn openapi_commit_show_is_get() { let doc = openapi_json(); - assert!(doc["paths"]["/commits/{commit_id}"]["get"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/commits/{commit_id}"]["get"].is_object()); } // --------------------------------------------------------------------------- @@ -483,13 +514,13 @@ fn query_request_query_is_required() { #[test] fn openapi_query_is_post() { let doc = openapi_json(); - assert!(doc["paths"]["/query"]["post"].is_object()); + assert!(doc["paths"]["/graphs/{graph_id}/query"]["post"].is_object()); } #[test] fn query_endpoint_documents_mutation_400() { let doc = openapi_json(); - let four_hundred = &doc["paths"]["/query"]["post"]["responses"]["400"]; + let four_hundred = &doc["paths"]["/graphs/{graph_id}/query"]["post"]["responses"]["400"]; let description = four_hundred["description"].as_str().unwrap_or_default(); assert!( description.contains("mutations") || description.contains("POST /mutate"), @@ -700,20 +731,21 @@ fn openapi_defines_bearer_token_security_scheme() { fn protected_endpoints_reference_bearer_token_security() { let doc = openapi_json(); let protected_paths = [ - ("/read", "post"), - ("/change", "post"), - ("/schema/apply", "post"), - ("/queries", "get"), - ("/queries/{name}", "post"), - ("/ingest", "post"), - ("/export", "post"), - ("/snapshot", "get"), - ("/branches", "get"), - ("/branches", "post"), - ("/branches/{branch}", "delete"), - ("/branches/merge", "post"), - ("/commits", "get"), - ("/commits/{commit_id}", "get"), + ("/graphs/{graph_id}/read", "post"), + ("/graphs/{graph_id}/change", "post"), + ("/graphs/{graph_id}/schema/apply", "post"), + ("/graphs/{graph_id}/queries", "get"), + ("/graphs/{graph_id}/queries/{name}", "post"), + ("/graphs/{graph_id}/load", "post"), + ("/graphs/{graph_id}/ingest", "post"), + ("/graphs/{graph_id}/export", "post"), + ("/graphs/{graph_id}/snapshot", "get"), + ("/graphs/{graph_id}/branches", "get"), + ("/graphs/{graph_id}/branches", "post"), + ("/graphs/{graph_id}/branches/{branch}", "delete"), + ("/graphs/{graph_id}/branches/merge", "post"), + ("/graphs/{graph_id}/commits", "get"), + ("/graphs/{graph_id}/commits/{commit_id}", "get"), ]; for (path, method) in protected_paths { @@ -745,7 +777,7 @@ fn healthz_does_not_require_security() { #[test] fn branch_delete_has_branch_path_parameter() { let doc = openapi_json(); - let params = doc["paths"]["/branches/{branch}"]["delete"]["parameters"] + let params = doc["paths"]["/graphs/{graph_id}/branches/{branch}"]["delete"]["parameters"] .as_array() .unwrap(); let has_branch = params @@ -760,7 +792,7 @@ fn branch_delete_has_branch_path_parameter() { #[test] fn commit_show_has_commit_id_path_parameter() { let doc = openapi_json(); - let params = doc["paths"]["/commits/{commit_id}"]["get"]["parameters"] + let params = doc["paths"]["/graphs/{graph_id}/commits/{commit_id}"]["get"]["parameters"] .as_array() .unwrap(); let has_commit_id = params @@ -775,7 +807,7 @@ fn commit_show_has_commit_id_path_parameter() { #[test] fn snapshot_has_branch_query_parameter() { let doc = openapi_json(); - let params = doc["paths"]["/snapshot"]["get"]["parameters"] + let params = doc["paths"]["/graphs/{graph_id}/snapshot"]["get"]["parameters"] .as_array() .unwrap(); let has_branch = params @@ -790,7 +822,7 @@ fn snapshot_has_branch_query_parameter() { #[test] fn commits_has_branch_query_parameter() { let doc = openapi_json(); - let params = doc["paths"]["/commits"]["get"]["parameters"] + let params = doc["paths"]["/graphs/{graph_id}/commits"]["get"]["parameters"] .as_array() .unwrap(); let has_branch = params @@ -830,7 +862,7 @@ fn openapi_operations_have_tags() { #[test] fn read_endpoint_200_references_read_output_schema() { let doc = openapi_json(); - let content = &doc["paths"]["/read"]["post"]["responses"]["200"]["content"]; + let content = &doc["paths"]["/graphs/{graph_id}/read"]["post"]["responses"]["200"]["content"]; let schema = &content["application/json"]["schema"]; let ref_path = schema["$ref"].as_str().unwrap(); assert!( @@ -842,7 +874,7 @@ fn read_endpoint_200_references_read_output_schema() { #[test] fn change_endpoint_200_references_change_output_schema() { let doc = openapi_json(); - let content = &doc["paths"]["/change"]["post"]["responses"]["200"]["content"]; + let content = &doc["paths"]["/graphs/{graph_id}/change"]["post"]["responses"]["200"]["content"]; let schema = &content["application/json"]["schema"]; let ref_path = schema["$ref"].as_str().unwrap(); assert!( @@ -867,11 +899,11 @@ fn healthz_200_references_health_output_schema() { fn error_responses_reference_error_output_schema() { let doc = openapi_json(); let paths_with_errors = [ - ("/read", "post", "400"), - ("/read", "post", "401"), - ("/change", "post", "400"), - ("/change", "post", "409"), - ("/branches", "post", "409"), + ("/graphs/{graph_id}/read", "post", "400"), + ("/graphs/{graph_id}/read", "post", "401"), + ("/graphs/{graph_id}/change", "post", "400"), + ("/graphs/{graph_id}/change", "post", "409"), + ("/graphs/{graph_id}/branches", "post", "409"), ]; for (path, method, status) in paths_with_errors { @@ -893,13 +925,13 @@ fn error_responses_reference_error_output_schema() { fn post_endpoints_have_request_body() { let doc = openapi_json(); let post_paths = [ - ("/read", "ReadRequest"), - ("/change", "ChangeRequest"), - ("/schema/apply", "SchemaApplyRequest"), - ("/ingest", "IngestRequest"), - ("/export", "ExportRequest"), - ("/branches", "BranchCreateRequest"), - ("/branches/merge", "BranchMergeRequest"), + ("/graphs/{graph_id}/read", "ReadRequest"), + ("/graphs/{graph_id}/change", "ChangeRequest"), + ("/graphs/{graph_id}/schema/apply", "SchemaApplyRequest"), + ("/graphs/{graph_id}/ingest", "IngestRequest"), + ("/graphs/{graph_id}/export", "ExportRequest"), + ("/graphs/{graph_id}/branches", "BranchCreateRequest"), + ("/graphs/{graph_id}/branches/merge", "BranchMergeRequest"), ]; for (path, expected_schema) in post_paths { @@ -920,7 +952,7 @@ fn post_endpoints_have_request_body() { #[test] fn invoke_stored_query_request_body_is_optional() { let doc = openapi_json(); - let request_body = &doc["paths"]["/queries/{name}"]["post"]["requestBody"]; + let request_body = &doc["paths"]["/graphs/{graph_id}/queries/{name}"]["post"]["requestBody"]; assert!( request_body.is_object(), "POST /queries/{{name}} should document its optional request body" @@ -1023,12 +1055,14 @@ async fn auth_mode_spec_has_security_on_protected_operations() { .body(Body::empty()) .unwrap(); let (_, json) = json_response(&app, request).await; + // RFC-011 cluster-only: the served spec always nests protected + // routes under `/graphs/{graph_id}/...`. let protected_paths = [ - ("/read", "post"), - ("/change", "post"), - ("/snapshot", "get"), - ("/branches", "get"), - ("/commits", "get"), + ("/graphs/{graph_id}/read", "post"), + ("/graphs/{graph_id}/change", "post"), + ("/graphs/{graph_id}/snapshot", "get"), + ("/graphs/{graph_id}/branches", "get"), + ("/graphs/{graph_id}/commits", "get"), ]; for (path, method) in protected_paths { let security = &json["paths"][path][method]["security"]; @@ -1045,22 +1079,6 @@ async fn auth_mode_spec_has_security_on_protected_operations() { } } -#[tokio::test] -async fn auth_mode_spec_matches_static_generation() { - let (_temp, app) = app_for_loaded_graph_with_auth("secret").await; - let request = Request::builder() - .method(Method::GET) - .uri("/openapi.json") - .body(Body::empty()) - .unwrap(); - let (_, served) = json_response(&app, request).await; - let static_doc = openapi_json(); - assert_eq!( - served, static_doc, - "auth-mode served spec must match static generation" - ); -} - #[tokio::test] async fn auth_mode_healthz_still_has_no_security() { let (_temp, app) = app_for_loaded_graph_with_auth("secret").await; @@ -1366,8 +1384,9 @@ async fn multi_mode_operation_ids_are_unique() { } #[tokio::test] -async fn single_mode_openapi_unchanged_by_cluster_filter() { - // Regression: single mode still emits the legacy flat surface. +async fn served_spec_always_nests_under_cluster_prefix() { + // RFC-011 cluster-only: even a one-graph convenience app serves the + // nested cluster surface and never the flat protected routes. let (_temp, app) = app_for_loaded_graph().await; let request = Request::builder() .method(Method::GET) @@ -1377,16 +1396,37 @@ async fn single_mode_openapi_unchanged_by_cluster_filter() { let (_, json) = json_response(&app, request).await; let paths = json["paths"].as_object().unwrap(); let path_keys: HashSet<&str> = paths.keys().map(|k| k.as_str()).collect(); - for expected in EXPECTED_PATHS { - assert!( - path_keys.contains(expected), - "single mode must still emit flat path: {expected}" - ); - } for cluster in EXPECTED_CLUSTER_PATHS { assert!( - !path_keys.contains(cluster), - "single mode must NOT emit cluster path: {cluster}" + path_keys.contains(cluster), + "served spec must emit cluster path: {cluster}. Found: {path_keys:?}" + ); + } + // The flat protected routes must NOT appear — only the nested + // cluster surface plus the always-flat `/healthz` and `/graphs`. + let flat_protected = [ + "/snapshot", + "/read", + "/query", + "/export", + "/change", + "/mutate", + "/queries", + "/queries/{name}", + "/schema", + "/schema/apply", + "/load", + "/ingest", + "/branches", + "/branches/{branch}", + "/branches/merge", + "/commits", + "/commits/{commit_id}", + ]; + for flat in flat_protected { + assert!( + !path_keys.contains(flat), + "served spec must NOT emit flat protected path: {flat}" ); } } diff --git a/crates/omnigraph-server/tests/s3.rs b/crates/omnigraph-server/tests/s3.rs index 2c61125..99bf98d 100644 --- a/crates/omnigraph-server/tests/s3.rs +++ b/crates/omnigraph-server/tests/s3.rs @@ -43,7 +43,7 @@ async fn server_opens_s3_graph_directly_and_serves_snapshot_and_read() { let (snapshot_status, snapshot_body) = json_response( &app, Request::builder() - .uri("/snapshot") + .uri(g("/snapshot")) .method(Method::GET) .header("authorization", "Bearer s3-token") .body(Body::empty()) @@ -63,7 +63,7 @@ async fn server_opens_s3_graph_directly_and_serves_snapshot_and_read() { let (read_status, read_body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("authorization", "Bearer s3-token") .header("content-type", "application/json") @@ -134,11 +134,8 @@ async fn server_boots_cluster_from_bare_storage_uri_and_serves_query() { } let settings = omnigraph_server::load_server_settings( - None, Some(&std::path::PathBuf::from(&root)), None, - None, - None, true, ) .await diff --git a/crates/omnigraph-server/tests/schema_routes.rs b/crates/omnigraph-server/tests/schema_routes.rs index d250d8a..c73591c 100644 --- a/crates/omnigraph-server/tests/schema_routes.rs +++ b/crates/omnigraph-server/tests/schema_routes.rs @@ -2,6 +2,7 @@ //! Moved verbatim from tests/server.rs in the modularization. use std::fs; +use std::sync::Arc; use axum::body::Body; use axum::http::{Method, Request, StatusCode}; @@ -11,7 +12,9 @@ use omnigraph::loader::LoadMode; use omnigraph_server::api::{ ChangeRequest, ErrorOutput, ReadRequest, SchemaApplyRequest, SchemaOutput, }; -use omnigraph_server::{AppState, build_app}; +use omnigraph_server::{ + AppState, GraphHandle, GraphId, GraphKey, PolicyEngine, build_app, workload, +}; use serde_json::json; @@ -30,7 +33,7 @@ async fn schema_apply_route_updates_graph_for_authorized_admin() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -54,6 +57,111 @@ async fn schema_apply_route_updates_graph_for_authorized_admin() { ); } +#[tokio::test] +async fn schema_apply_route_refuses_cluster_backed_server_mode() { + let temp = init_graph_with_schema(&fs::read_to_string(fixture("test.pg")).unwrap()).await; + let graph = graph_path(temp.path()); + let graph_uri = graph.to_string_lossy().to_string(); + let engine = Omnigraph::open(&graph_uri).await.unwrap(); + let handle = Arc::new(GraphHandle { + key: GraphKey::cluster(GraphId::try_from("default").unwrap()), + uri: graph_uri.clone(), + engine: Arc::new(engine), + policy: None, + queries: None, + }); + let state = AppState::new_multi( + vec![handle], + Vec::new(), + None, + workload::WorkloadController::from_env(), + Some(temp.path().join("cluster.yaml")), + ) + .unwrap(); + let app = build_app(state); + + let request = Request::builder() + .method(Method::POST) + .uri(g("/schema/apply")) + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&SchemaApplyRequest { + schema_source: additive_schema_with_nickname(), + ..Default::default() + }) + .unwrap(), + )) + .unwrap(); + let (status, payload) = json_response(&app, request).await; + + assert_eq!(status, StatusCode::CONFLICT, "body: {payload}"); + assert!( + payload["error"] + .as_str() + .unwrap_or_default() + .contains("cluster apply"), + "body: {payload}" + ); + let reopened = Omnigraph::open(&graph_uri).await.unwrap(); + assert!( + !reopened.catalog().node_types["Person"] + .properties + .contains_key("nickname"), + "cluster-backed schema apply must not mutate the graph" + ); +} + +#[tokio::test] +async fn schema_apply_route_cluster_backed_denies_unauthorized_actor_before_409() { + // The cluster-backed 409 is reported AFTER the Cedar gate, so an actor + // without `schema_apply` permission gets a 403 — never a 409 that would + // disclose the server is cluster-backed (401 → 403 → 409, no topology leak + // before authorization). POLICY_YAML grants read/export but not schema_apply, + // so act-ragnor is denied. + let temp = init_graph_with_schema(&fs::read_to_string(fixture("test.pg")).unwrap()).await; + let graph = graph_path(temp.path()); + let graph_uri = graph.to_string_lossy().to_string(); + let engine = Omnigraph::open(&graph_uri).await.unwrap(); + let policy = PolicyEngine::load_graph_from_source(POLICY_YAML, "default").unwrap(); + let handle = Arc::new(GraphHandle { + key: GraphKey::cluster(GraphId::try_from("default").unwrap()), + uri: graph_uri, + engine: Arc::new(engine), + policy: Some(Arc::new(policy)), + queries: None, + }); + let state = AppState::new_multi( + vec![handle], + vec![("act-ragnor".to_string(), "admin-token".to_string())], + None, + workload::WorkloadController::from_env(), + Some(temp.path().join("cluster.yaml")), + ) + .unwrap(); + let app = build_app(state); + + let request = Request::builder() + .method(Method::POST) + .uri(g("/schema/apply")) + .header("content-type", "application/json") + .header("authorization", "Bearer admin-token") + .body(Body::from( + serde_json::to_vec(&SchemaApplyRequest { + schema_source: additive_schema_with_nickname(), + ..Default::default() + }) + .unwrap(), + )) + .unwrap(); + let (status, payload) = json_response(&app, request).await; + + assert_eq!( + status, + StatusCode::FORBIDDEN, + "an unauthorized actor must get 403 before the cluster-backed 409: {payload}" + ); +} + #[tokio::test(flavor = "multi_thread")] async fn schema_apply_route_rejects_stored_query_breakage_before_publish() { let (temp, app) = app_with_stored_queries( @@ -65,7 +173,7 @@ async fn schema_apply_route_rejects_stored_query_breakage_before_publish() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -115,7 +223,7 @@ async fn schema_apply_route_noop_keeps_valid_stored_query_registry() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -142,7 +250,7 @@ async fn schema_apply_route_requires_schema_apply_policy_permission() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -173,7 +281,7 @@ async fn schema_apply_route_requires_bearer_token_when_policy_enabled() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .body(Body::from( serde_json::to_vec(&SchemaApplyRequest { @@ -203,7 +311,7 @@ async fn schema_apply_route_can_rename_type() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -239,7 +347,7 @@ async fn schema_apply_route_can_rename_property() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -279,7 +387,7 @@ async fn schema_apply_route_can_add_index() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -294,6 +402,11 @@ async fn schema_apply_route_can_add_index() { assert_eq!(status, StatusCode::OK); assert_eq!(payload["applied"], true); + // iss-848: the /schema/apply route accepts the index-add and applies it as a + // metadata change — it records the `@index` intent in the catalog/IR but does + // NOT build the physical index inline (the build is deferred to + // ensure_indices/optimize; on this empty table nothing would build anyway). + // So the physical index count is unchanged by the apply. let reopened = Omnigraph::open(graph.to_str().unwrap()).await.unwrap(); let snapshot = reopened .snapshot_of(ReadTarget::branch("main")) @@ -301,7 +414,10 @@ async fn schema_apply_route_can_add_index() { .unwrap(); let dataset = snapshot.open("node:Person").await.unwrap(); let after_index_count = dataset.load_indices().await.unwrap().len(); - assert!(after_index_count > before_index_count); + assert_eq!( + after_index_count, before_index_count, + "schema apply records @index intent but defers the physical build (iss-848)" + ); } #[tokio::test] @@ -315,7 +431,7 @@ async fn schema_apply_route_rejects_unsupported_plan() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -356,7 +472,7 @@ async fn schema_apply_route_rejects_when_non_main_branch_exists() { let request = Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -385,7 +501,7 @@ async fn schema_drift_returns_conflict_for_snapshot_read_and_change() { let (snapshot_status, snapshot_body) = json_response( &app, Request::builder() - .uri("/snapshot?branch=main") + .uri(g("/snapshot?branch=main")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -413,7 +529,7 @@ async fn schema_drift_returns_conflict_for_snapshot_read_and_change() { let (read_status, read_body) = json_response( &app, Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&read).unwrap())) @@ -441,7 +557,7 @@ async fn schema_drift_returns_conflict_for_snapshot_read_and_change() { let (change_status, change_body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(serde_json::to_vec(&change).unwrap())) @@ -467,7 +583,7 @@ async fn schema_route_returns_current_source() { let (status, body) = json_response( &app, Request::builder() - .uri("/schema") + .uri(g("/schema")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -486,7 +602,7 @@ async fn schema_route_requires_bearer_token_when_auth_configured() { let (missing_status, missing_body) = json_response( &app, Request::builder() - .uri("/schema") + .uri(g("/schema")) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -502,7 +618,7 @@ async fn schema_route_requires_bearer_token_when_auth_configured() { let (ok_status, ok_body) = json_response( &app, Request::builder() - .uri("/schema") + .uri(g("/schema")) .method(Method::GET) .header("authorization", "Bearer demo-token") .body(Body::empty()) @@ -533,7 +649,7 @@ async fn schema_route_denied_when_actor_lacks_read_permission() { let (status, body) = json_response( &app, Request::builder() - .uri("/schema") + .uri(g("/schema")) .method(Method::GET) .header("authorization", "Bearer team-token") .body(Body::empty()) @@ -574,7 +690,7 @@ async fn schema_apply_route_soft_drops_property_via_http() { &app, Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -631,7 +747,7 @@ async fn schema_apply_route_soft_drops_node_type_via_http() { &app, Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -683,7 +799,7 @@ async fn schema_apply_route_hard_drops_property_with_allow_data_loss() { &app, Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -738,7 +854,7 @@ async fn schema_apply_route_keeps_drops_soft_without_flag() { &app, Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -770,29 +886,27 @@ async fn schema_apply_route_additive_property_preserves_existing_rows() { // AddProperty wasn't pinned with a row-count check anywhere. // Load N rows, apply schema adding nullable property, verify // every row is still readable and the new column is null. - let (temp, app) = app_for_graph_with_auth_tokens_and_policy( - &fs::read_to_string(fixture("test.pg")).unwrap(), + let (temp, app) = app_for_loaded_graph_with_auth_tokens_and_policy( &[("act-ragnor", "admin-token")], SCHEMA_APPLY_POLICY_YAML, ) .await; let graph = graph_path(temp.path()); - // Standard fixture data: 4 Persons + 1 Company. Load it. + // Standard fixture data is loaded before the app is built, so the server + // handle applies schema from the same manifest it is serving. let pre_count = { let db = Omnigraph::open(graph.to_str().unwrap()).await.unwrap(); - db.load( - "main", - &fs::read_to_string(fixture("test.jsonl")).unwrap(), - LoadMode::Append, - ) - .await - .unwrap(); let snap = db .snapshot_of(omnigraph::db::ReadTarget::branch("main")) .await .unwrap(); - snap.entry("node:Person").expect("Person").row_count + snap.open("node:Person") + .await + .expect("Person") + .count_rows(None) + .await + .unwrap() }; assert!(pre_count > 0, "fixture should have loaded Person rows"); @@ -800,7 +914,7 @@ async fn schema_apply_route_additive_property_preserves_existing_rows() { &app, Request::builder() .method(Method::POST) - .uri("/schema/apply") + .uri(g("/schema/apply")) .header("content-type", "application/json") .header("authorization", "Bearer admin-token") .body(Body::from( @@ -822,7 +936,13 @@ async fn schema_apply_route_additive_property_preserves_existing_rows() { .snapshot_of(omnigraph::db::ReadTarget::branch("main")) .await .unwrap(); - let post_count = snap.entry("node:Person").expect("Person").row_count; + let post_count = snap + .open("node:Person") + .await + .expect("Person") + .count_rows(None) + .await + .unwrap(); assert_eq!( post_count, pre_count, "AddProperty should preserve row count", diff --git a/crates/omnigraph-server/tests/stored_queries.rs b/crates/omnigraph-server/tests/stored_queries.rs index e4da1d3..02553a7 100644 --- a/crates/omnigraph-server/tests/stored_queries.rs +++ b/crates/omnigraph-server/tests/stored_queries.rs @@ -82,6 +82,58 @@ async fn invoke_stored_read_returns_rows() { assert!(body["rows"].is_array(), "read envelope shape; body: {body}"); } +#[tokio::test(flavor = "multi_thread")] +async fn invoke_with_mismatched_expected_kind_is_rejected() { + // RFC-011 D3: the CLI verb asserts the stored query's kind via + // `expect_mutation`. Invoking a read with `expect_mutation: true` + // (i.e. `omnigraph mutate `) is a 400 naming the right verb. + let (_temp, app) = app_with_stored_queries( + &[("find_person", FIND_PERSON_GQ, false)], + &[("act-invoke", "t-invoke")], + INVOKE_POLICY_YAML, + ) + .await; + let (status, body) = json_response( + &app, + invoke_request( + "find_person", + "t-invoke", + json!({ "expect_mutation": true, "params": { "name": "Alice" } }), + ), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "body: {body}"); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("'find_person' is a read — use omnigraph query find_person"), + "expected a kind-mismatch error; body: {body}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn invoke_with_matching_expected_kind_runs() { + // The matching assertion (`omnigraph query `) passes through. + let (_temp, app) = app_with_stored_queries( + &[("find_person", FIND_PERSON_GQ, false)], + &[("act-invoke", "t-invoke")], + INVOKE_POLICY_YAML, + ) + .await; + let (status, body) = json_response( + &app, + invoke_request( + "find_person", + "t-invoke", + json!({ "expect_mutation": false, "params": { "name": "Alice" } }), + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "matching kind should run; body: {body}"); + assert_eq!(body["query_name"], "find_person"); +} + #[tokio::test(flavor = "multi_thread")] async fn invoke_stored_read_accepts_absent_or_empty_body() { let no_param_query = "query list_people() { match { $p: Person } return { $p.name } }"; @@ -272,7 +324,7 @@ async fn list_queries_returns_only_exposed_with_typed_params() { INVOKE_POLICY_YAML, ) .await; - let (status, body) = json_response(&app, get_request("/queries", "t-invoke")).await; + let (status, body) = json_response(&app, get_request(&g("/queries"), "t-invoke")).await; assert_eq!(status, StatusCode::OK, "body: {body}"); let entries = body["queries"].as_array().unwrap(); @@ -303,7 +355,7 @@ async fn list_queries_is_read_gated_so_a_non_invoker_can_list() { INVOKE_POLICY_YAML, ) .await; - let (status, body) = json_response(&app, get_request("/queries", "t-noinvoke")).await; + let (status, body) = json_response(&app, get_request(&g("/queries"), "t-noinvoke")).await; assert_eq!(status, StatusCode::OK, "read-gated catalog; body: {body}"); let names: Vec<&str> = body["queries"] .as_array() @@ -320,7 +372,7 @@ async fn list_queries_is_read_gated_so_a_non_invoker_can_list() { #[tokio::test(flavor = "multi_thread")] async fn list_queries_is_empty_when_no_registry() { let (_temp, app) = app_for_loaded_graph_with_auth("demo-token").await; - let (status, body) = json_response(&app, get_request("/queries", "demo-token")).await; + let (status, body) = json_response(&app, get_request(&g("/queries"), "demo-token")).await; assert_eq!(status, StatusCode::OK, "body: {body}"); assert!( body["queries"].as_array().unwrap().is_empty(), diff --git a/crates/omnigraph-server/tests/support/mod.rs b/crates/omnigraph-server/tests/support/mod.rs index 0e32410..157c58e 100644 --- a/crates/omnigraph-server/tests/support/mod.rs +++ b/crates/omnigraph-server/tests/support/mod.rs @@ -248,9 +248,17 @@ rules: pub const FIND_PERSON_GQ: &str = "query find_person($name: String) { match { $p: Person { name: $name } } return { $p.age } }"; +/// RFC-011 cluster-only: the single-graph convenience apps built by the +/// `app_for_loaded_graph*` helpers serve the graph under the reserved id +/// `default`. This prefixes a flat per-graph path (e.g. `/snapshot`) with +/// the cluster route prefix so tests address `/graphs/default/snapshot`. +pub fn g(path: &str) -> String { + format!("/graphs/default{path}") +} + pub fn invoke_request(name: &str, token: &str, body: Value) -> Request { Request::builder() - .uri(format!("/queries/{name}")) + .uri(g(&format!("/queries/{name}"))) .method(Method::POST) .header("content-type", "application/json") .header("authorization", format!("Bearer {token}")) @@ -265,7 +273,7 @@ pub fn invoke_request_bytes( content_type: Option<&str>, ) -> Request { let mut builder = Request::builder() - .uri(format!("/queries/{name}")) + .uri(g(&format!("/queries/{name}"))) .method(Method::POST) .header("authorization", format!("Bearer {token}")); if let Some(content_type) = content_type { @@ -656,7 +664,7 @@ pub mod matrix { .clone() .oneshot( Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -686,7 +694,7 @@ pub mod matrix { .clone() .oneshot( Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -728,7 +736,7 @@ pub mod matrix { .clone() .oneshot( Request::builder() - .uri(format!("/snapshot?branch={}", branch)) + .uri(g(&format!("/snapshot?branch={}", branch))) .method(Method::GET) .body(Body::empty()) .unwrap(), @@ -766,7 +774,7 @@ pub mod matrix { .clone() .oneshot( Request::builder() - .uri("/read") + .uri(g("/read")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -833,7 +841,7 @@ pub mod matrix { .clone() .oneshot( Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -874,7 +882,7 @@ pub mod matrix { let response = app .oneshot( Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -910,7 +918,7 @@ pub mod matrix { let response = app .oneshot( Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -943,7 +951,7 @@ pub mod matrix { let response = app .oneshot( Request::builder() - .uri("/branches") + .uri(g("/branches")) .method(Method::POST) .header("content-type", "application/json") .body(Body::from(body)) @@ -970,7 +978,7 @@ pub mod matrix { let response = app .oneshot( Request::builder() - .uri(format!("/branches/{}", name)) + .uri(g(&format!("/branches/{}", name))) .method(Method::DELETE) .body(Body::empty()) .unwrap(), @@ -1091,7 +1099,7 @@ pub async fn http_change_decision( let (status, _body) = json_response( &app, Request::builder() - .uri("/change") + .uri(g("/change")) .method(Method::POST) .header(AUTHORIZATION, format!("Bearer {token}")) .header("content-type", "application/json") @@ -1141,7 +1149,7 @@ pub async fn http_merge_decision( let (status, _body) = json_response( &app, Request::builder() - .uri("/branches/merge") + .uri(g("/branches/merge")) .method(Method::POST) .header(AUTHORIZATION, format!("Bearer {token}")) .header("content-type", "application/json") @@ -1191,5 +1199,5 @@ graphs: } pub async fn cluster_settings(dir: &Path) -> color_eyre::eyre::Result { - omnigraph_server::load_server_settings(None, Some(&dir.to_path_buf()), None, None, None, true).await + omnigraph_server::load_server_settings(Some(&dir.to_path_buf()), None, true).await } diff --git a/crates/omnigraph/src/changes/mod.rs b/crates/omnigraph/src/changes/mod.rs index d4a3fe7..2e9bc02 100644 --- a/crates/omnigraph/src/changes/mod.rs +++ b/crates/omnigraph/src/changes/mod.rs @@ -248,12 +248,12 @@ async fn diff_table_same_lineage( // Inserts + Updates: use _row_last_updated_at_version to find all rows // touched since Vf, then classify by checking whether the ID existed at Vf. // - // Why not _row_created_at_version for inserts: Lance's merge_insert stamps - // new rows with _row_created_at_version = dataset_creation_version (v1), - // not the merge_insert commit version. This makes _row_created_at_version - // unreliable for detecting inserts from merge_insert writes. Using - // _row_last_updated_at_version catches all touched rows regardless of - // write mode, and ID-set membership distinguishes inserts from updates. + // We key on _row_last_updated_at_version because one scan over it catches + // every row touched in the window — inserts and updates alike — regardless + // of write mode, and ID-set membership at Vf then distinguishes inserts from + // updates. (lance#6774 made merge_insert stamp new rows' _row_created_at_version + // with the commit version, so created_at became reliable too; last_updated + // stays the right key since it also covers updates.) if wants_inserts || wants_updates { let filter_sql = format!( "_row_last_updated_at_version > {} AND _row_last_updated_at_version <= {}", diff --git a/crates/omnigraph/src/db/commit_graph.rs b/crates/omnigraph/src/db/commit_graph.rs index 9531a64..3d90e54 100644 --- a/crates/omnigraph/src/db/commit_graph.rs +++ b/crates/omnigraph/src/db/commit_graph.rs @@ -57,6 +57,8 @@ impl CommitGraph { mode: WriteMode::Create, enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; let dataset = Dataset::write(reader, &uri as &str, Some(params)) @@ -430,6 +432,8 @@ async fn create_commit_actor_dataset(root_uri: &str) -> Result { mode: WriteMode::Create, enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; match Dataset::write(reader, &uri as &str, Some(params)).await { diff --git a/crates/omnigraph/src/db/manifest.rs b/crates/omnigraph/src/db/manifest.rs index 6cd271a..f130523 100644 --- a/crates/omnigraph/src/db/manifest.rs +++ b/crates/omnigraph/src/db/manifest.rs @@ -34,10 +34,10 @@ pub(crate) use namespace::open_table_head_for_write; use namespace::{branch_manifest_namespace, staged_table_namespace}; use publisher::{GraphNamespacePublisher, ManifestBatchPublisher}; pub(crate) use recovery::{ - RecoveryMode, RecoverySidecar, RecoverySidecarHandle, SidecarKind, SidecarTablePin, - SidecarTableRegistration, SidecarTombstone, delete_sidecar, has_schema_apply_sidecar, - heal_pending_sidecars_roll_forward, list_sidecars, new_sidecar, recover_manifest_drift, - schema_apply_serial_queue_key, write_sidecar, + RecoveryMode, RecoverySidecarHandle, SidecarKind, SidecarTablePin, SidecarTableRegistration, + SidecarTombstone, delete_sidecar, has_schema_apply_sidecar, heal_pending_sidecars_roll_forward, + list_sidecars, new_sidecar, recover_manifest_drift, schema_apply_serial_queue_key, + write_sidecar, }; pub use state::SubTableEntry; #[cfg(test)] diff --git a/crates/omnigraph/src/db/manifest/graph.rs b/crates/omnigraph/src/db/manifest/graph.rs index 6c414aa..da2c641 100644 --- a/crates/omnigraph/src/db/manifest/graph.rs +++ b/crates/omnigraph/src/db/manifest/graph.rs @@ -31,6 +31,8 @@ pub(super) async fn init_manifest_graph( mode: WriteMode::Create, enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; let manifest_path = manifest_uri(root); @@ -127,6 +129,8 @@ async fn create_empty_dataset(uri: &str, schema: &SchemaRef) -> Result enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; Dataset::write(reader, uri, Some(params)) diff --git a/crates/omnigraph/src/db/manifest/migrations.rs b/crates/omnigraph/src/db/manifest/migrations.rs index e2801fe..2a65079 100644 --- a/crates/omnigraph/src/db/manifest/migrations.rs +++ b/crates/omnigraph/src/db/manifest/migrations.rs @@ -113,20 +113,47 @@ pub(super) async fn migrate_internal_schema(dataset: &mut Dataset) -> Result<()> /// so the merge-insert conflict resolver enforces row-level CAS at commit /// time, then bump the stamp. /// -/// Both steps are idempotent under retry: re-applying the field annotation -/// at its current value is a no-op-ish bump in Lance, and the stamp is a -/// simple key-value write. A crash between the two leaves the field set -/// without a stamp; the next open re-runs this fn and only the stamp lands. +/// Idempotent under crash-retry by construction. Lance 7 makes the unenforced +/// primary key **immutable once set**: any write that touches the reserved +/// `lance-schema:unenforced-primary-key` field metadata after the PK is set +/// errors ("cannot be changed once set", `lance::dataset::transaction`), even +/// re-applying the same value. A crash between the field-set and the stamp +/// bump leaves the field set without a stamp, so the next open re-enters here +/// with the PK already present — we must therefore set it only when absent. +/// (Fresh graphs bake the PK into `manifest_schema()` at init and never run +/// this migration; only genuine pre-v0.4.0 graphs do.) async fn migrate_v1_to_v2(dataset: &mut Dataset) -> Result<()> { - dataset - .update_field_metadata() - .update( - "object_id", - [(OBJECT_ID_PK_KEY.to_string(), "true".to_string())], - ) - .map_err(|e| OmniError::Lance(e.to_string()))? - .await - .map_err(|e| OmniError::Lance(e.to_string()))?; + // The guard is over the *specific* field, not just "any PK is set": skipping + // when `object_id` is already the PK is the idempotent crash-recovery path, + // but a manifest whose PK is some *other* field has the wrong CAS key — and + // Lance 7 won't let us change it. Refuse loudly rather than silently leave + // merge-insert conflict detection keyed on the wrong column. + let pk_fields: Vec<&str> = dataset + .schema() + .unenforced_primary_key() + .iter() + .map(|field| field.name.as_str()) + .collect(); + match pk_fields.as_slice() { + ["object_id"] => {} // already migrated (or a crash re-entry) — idempotent no-op + [] => { + dataset + .update_field_metadata() + .update( + "object_id", + [(OBJECT_ID_PK_KEY.to_string(), "true".to_string())], + ) + .map_err(|e| OmniError::Lance(e.to_string()))? + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + } + other => { + return Err(OmniError::manifest_internal(format!( + "__manifest unenforced primary key is {other:?}, expected [\"object_id\"]; \ + refusing to migrate a manifest with an unexpected CAS key" + ))); + } + } set_stamp(dataset, 2).await } diff --git a/crates/omnigraph/src/db/manifest/namespace.rs b/crates/omnigraph/src/db/manifest/namespace.rs index 80d206f..5e907ba 100644 --- a/crates/omnigraph/src/db/manifest/namespace.rs +++ b/crates/omnigraph/src/db/manifest/namespace.rs @@ -10,7 +10,9 @@ use lance_namespace::models::{ }; use lance_namespace::{Error as LanceNamespaceError, LanceNamespace, NamespaceError}; use lance_table::io::commit::ManifestNamingScheme; -use object_store::{Error as ObjectStoreError, ObjectStore as _, PutMode, PutOptions, path::Path}; +use object_store::{ + Error as ObjectStoreError, ObjectStore as _, ObjectStoreExt, PutMode, PutOptions, path::Path, +}; use crate::error::{OmniError, Result}; diff --git a/crates/omnigraph/src/db/manifest/publisher.rs b/crates/omnigraph/src/db/manifest/publisher.rs index d13dd08..288f4be 100644 --- a/crates/omnigraph/src/db/manifest/publisher.rs +++ b/crates/omnigraph/src/db/manifest/publisher.rs @@ -381,6 +381,12 @@ impl GraphNamespacePublisher { // the publisher loop above, where each attempt re-runs the pre-check. merge_builder.conflict_retries(0); merge_builder.use_index(false); + // Skip Lance's auto-cleanup hook: `__manifest` versions are the snapshot + // / time-travel authority and must never be GC'd by Lance's per-commit + // hook. A `__manifest` created before the v7 bump (6.0.1 defaulted + // auto_cleanup ON) still carries the stored config, so this skip is + // load-bearing on upgraded graphs, not just defensive. + merge_builder.skip_auto_cleanup(true); let (new_dataset, _stats) = merge_builder .try_build() .map_err(|e| OmniError::Lance(e.to_string()))? diff --git a/crates/omnigraph/src/db/manifest/recovery.rs b/crates/omnigraph/src/db/manifest/recovery.rs index d49e86a..968d3f4 100644 --- a/crates/omnigraph/src/db/manifest/recovery.rs +++ b/crates/omnigraph/src/db/manifest/recovery.rs @@ -793,10 +793,10 @@ pub(crate) fn schema_apply_serial_queue_key() -> crate::db::write_queue::TableQu /// same table append extra Lance restore commits which `omnigraph /// cleanup` reclaims. /// -/// Concurrency: today recovery runs synchronously in `Omnigraph::open` -/// *before* the engine is wrapped in the server's `Arc>`. -/// No request handlers can race, so this sweep does NOT acquire write -/// queues. In-process callers (refresh, write entry points) must use +/// Concurrency: the open-time sweep runs synchronously in `Omnigraph::open` +/// before the engine handle is published to any caller, so no request +/// handler can race it and it does NOT acquire write queues. In-process +/// callers (refresh, write entry points) must use /// [`heal_pending_sidecars_roll_forward`] instead, which serializes /// against live writers via per-(table_key, branch) queue acquisition. pub(crate) async fn recover_manifest_drift( diff --git a/crates/omnigraph/src/db/manifest/tests.rs b/crates/omnigraph/src/db/manifest/tests.rs index 885a2a8..0e00505 100644 --- a/crates/omnigraph/src/db/manifest/tests.rs +++ b/crates/omnigraph/src/db/manifest/tests.rs @@ -336,40 +336,77 @@ async fn test_directory_namespace_direct_publish_cannot_replace_native_omnigraph .await .unwrap(); - let versions = namespace - .list_table_versions(ListTableVersionsRequest { - id: Some(vec!["node:Person".to_string()]), - descending: Some(true), - ..Default::default() - }) - .await - .unwrap(); - assert_eq!( - versions.versions[0].version as u64, - person_entry.table_version + // Lance 7: the native `DirectoryNamespace` no longer recognizes omnigraph's + // manifest-tracked tables, so list / describe / create_table_version all + // return `TableNotFound`. The mechanism is *contingent on omnigraph's legacy + // boolean PK key*, not an unconditional v7 property: v7's namespace eagerly + // rewrites any `__manifest` whose `object_id` lacks the new + // `lance-schema:unenforced-primary-key:position` key, omnigraph declares the + // PK with the legacy boolean key, and v7 forbids changing a PK once set — so + // `ensure_manifest_table_up_to_date` errors, the namespace silently falls + // back to directory listing (disabled here), and `check_table_status` reports + // the table absent. omnigraph keeps the boolean key deliberately: Lance + // honors it permanently (it maps to PK position 0) and one uniform on-disk + // format beats a new-vs-old split, since existing graphs can't be re-keyed to + // the position key under that same immutability rule. The decoupling is + // therefore an accepted, production-irrelevant tradeoff (omnigraph never uses + // the native namespace — its publisher writes `__manifest` via merge_insert + // and its reads go through its own `LanceNamespace` impls), and it only + // strengthens this guard's thesis: native tooling cannot enumerate, inspect, + // or publish over omnigraph's tables, let alone replace the write path. + let assert_table_not_found = |what: &str, dbg: String| { + assert!( + dbg.contains("TableNotFound") && dbg.contains("node:Person"), + "{what}: expected TableNotFound for node:Person, got: {dbg}" + ); + }; + assert_table_not_found( + "list_table_versions", + format!( + "{:?}", + namespace + .list_table_versions(ListTableVersionsRequest { + id: Some(vec!["node:Person".to_string()]), + descending: Some(true), + ..Default::default() + }) + .await + .unwrap_err() + ), + ); + assert_table_not_found( + "describe_table_version", + format!( + "{:?}", + namespace + .describe_table_version(DescribeTableVersionRequest { + id: Some(vec!["node:Person".to_string()]), + version: Some(person_version as i64), + ..Default::default() + }) + .await + .unwrap_err() + ), + ); + assert_table_not_found( + "create_table_version", + format!( + "{:?}", + namespace + .create_table_version(version_metadata.to_create_table_version_request( + "node:Person", + person_version, + 1, + None, + )) + .await + .unwrap_err() + ), ); - let err = namespace - .describe_table_version(DescribeTableVersionRequest { - id: Some(vec!["node:Person".to_string()]), - version: Some(person_version as i64), - ..Default::default() - }) - .await - .unwrap_err(); - assert!(err.to_string().contains("not found")); - - let err = namespace - .create_table_version(version_metadata.to_create_table_version_request( - "node:Person", - person_version, - 1, - None, - )) - .await - .unwrap_err(); - assert!(err.to_string().contains("already exists")); - + // omnigraph's manifest stays authoritative: refresh ignores the direct + // `person_ds.append` above (it was never manifest-published), so the row + // count stays 0 and the version is unchanged. mc.refresh().await.unwrap(); assert_eq!( mc.snapshot().entry("node:Person").unwrap().table_version, diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index 000602a..f382908 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -11,9 +11,9 @@ pub use graph_coordinator::{GraphCoordinator, ReadTarget, ResolvedTarget, Snapsh pub use manifest::{Snapshot, SubTableEntry, SubTableUpdate}; pub(crate) use omnigraph::ensure_public_branch_ref; pub use omnigraph::{ - CleanupPolicyOptions, InitOptions, MergeOutcome, Omnigraph, OpenMode, RepairAction, - RepairClassification, RepairOptions, RepairStats, SchemaApplyOptions, SchemaApplyResult, - SkipReason, TableCleanupStats, TableOptimizeStats, TableRepairStats, + CleanupPolicyOptions, InitOptions, MergeOutcome, Omnigraph, OpenMode, PendingIndex, + RepairAction, RepairClassification, RepairOptions, RepairStats, SchemaApplyOptions, + SchemaApplyResult, SkipReason, TableCleanupStats, TableOptimizeStats, TableRepairStats, }; pub(crate) const SCHEMA_APPLY_LOCK_BRANCH: &str = "__schema_apply_lock__"; diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 779a2e0..48be274 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -16,7 +16,7 @@ use lance::dataset::scanner::ColumnOrdering; use lance::datatypes::BlobKind; use omnigraph_compiler::catalog::{Catalog, EdgeType, NodeType}; use omnigraph_compiler::schema::parser::parse_schema; -use omnigraph_compiler::types::ScalarType; +use omnigraph_compiler::types::{PropType, ScalarType}; use omnigraph_compiler::{ DropMode, SchemaIR, SchemaMigrationPlan, SchemaMigrationStep, SchemaTypeKind, build_catalog_from_ir, build_schema_ir, plan_schema_migration, @@ -40,6 +40,7 @@ pub use repair::{ RepairAction, RepairClassification, RepairOptions, RepairStats, TableRepairStats, }; pub use schema_apply::SchemaApplyOptions; +pub use table_ops::PendingIndex; use super::commit_graph::GraphCommit; use super::manifest::{ @@ -113,10 +114,11 @@ pub struct Omnigraph { /// Read-heavy on schema introspection paths, written only by /// `apply_schema`. Same ArcSwap rationale as `catalog`. schema_source: Arc>, - /// Per-`(table_key, branch)` writer queues. Reachable from engine - /// internals (mutation finalize, schema_apply, branch_merge, - /// ensure_indices, delete_where) and from future MR-870 recovery - /// reconciler. PR 1b adds the field; callers acquire in commits 4+. + /// Per-`(table_key, branch)` writer queues — the engine's + /// write-serialization mechanism (the server holds the engine as a + /// lockless `Arc`). Reachable from engine internals + /// (mutation finalize, schema_apply, branch_merge, ensure_indices, + /// delete_where, the fork path, recovery reconciler). write_queue: Arc, /// Process-wide mutex held across the swap → operate → restore window /// in `branch_merge_impl`. Two concurrent merges with distinct targets @@ -156,6 +158,17 @@ pub struct Omnigraph { /// `apply_schema_as` consults this field (PR #2 proof-of-concept); /// PR #3 fans the `enforce()` call out to the remaining writers. policy: Option>, + /// Lazily-built, reused-across-queries embedding client. Built on the first + /// `nearest($v, "string")` that needs server-side embedding (so a graph that + /// never embeds needs no provider key), then shared by every later query — + /// avoids the per-query `from_env()` rebuild and keeps the provider HTTP + /// connection pool warm. `OnceCell` guarantees a single initialization. + embedding: Arc>, + /// Optional pre-resolved embedding config (RFC-012 Phase 5), injected from an + /// applied cluster `providers.embedding` profile via [`Omnigraph::with_embedding_config`]. + /// When set, the embedding cell builds its client from this instead of + /// `EmbeddingClient::from_env()`; `None` keeps the env fallback. + embedding_config: Option>, } /// Whether [`Omnigraph::open`] runs the open-time recovery sweep. @@ -319,6 +332,8 @@ impl Omnigraph { write_queue: Arc::new(crate::db::write_queue::WriteQueueManager::new()), merge_exclusive: Arc::new(tokio::sync::Mutex::new(())), policy: None, + embedding: Arc::new(tokio::sync::OnceCell::new()), + embedding_config: None, }) } @@ -418,6 +433,8 @@ impl Omnigraph { write_queue: Arc::new(crate::db::write_queue::WriteQueueManager::new()), merge_exclusive: Arc::new(tokio::sync::Mutex::new(())), policy: None, + embedding: Arc::new(tokio::sync::OnceCell::new()), + embedding_config: None, }) } @@ -465,6 +482,29 @@ impl Omnigraph { self } + /// The lazily-initialized, reused-across-queries embedding client cell + /// (see the `embedding` field doc). The query executor resolves the client + /// through this on the first `nearest($v, "string")` that needs embedding. + pub(crate) fn embedding_cell( + &self, + ) -> &tokio::sync::OnceCell { + &self.embedding + } + + /// Install a pre-resolved embedding config (RFC-012 Phase 5). Builder-style, + /// mirroring [`Omnigraph::with_policy`]: a graph served from a cluster + /// embedding provider profile injects it here; an embedded/CLI caller that doesn't + /// call this keeps the `EmbeddingClient::from_env()` fallback. + pub fn with_embedding_config(mut self, config: Arc) -> Self { + self.embedding_config = Some(config); + self + } + + /// The injected embedding config, if any (see the `embedding_config` field). + pub(crate) fn embedding_config_ref(&self) -> Option<&crate::embedding::EmbeddingConfig> { + self.embedding_config.as_deref() + } + /// Engine-layer policy enforcement gate (MR-722 chassis core). /// /// * If no policy is installed → no-op (returns `Ok(())`). @@ -1069,11 +1109,15 @@ impl Omnigraph { /// unbranched subtables keep inheriting `main`, while subtables inherited /// from an ancestor branch are first forked into the active branch before /// their index metadata is updated. - pub async fn ensure_indices(&self) -> Result<()> { + /// Returns the declared indexes that could not be materialized on this + /// pass (today: vector columns with no trainable vectors yet). They are + /// deferred, not errors; a later `ensure_indices`/`optimize` builds them + /// once the column is trainable. Reads stay correct (brute-force) meanwhile. + pub async fn ensure_indices(&self) -> Result> { table_ops::ensure_indices(self).await } - pub async fn ensure_indices_on(&self, branch: &str) -> Result<()> { + pub async fn ensure_indices_on(&self, branch: &str) -> Result> { table_ops::ensure_indices_on(self, branch).await } @@ -1479,6 +1523,13 @@ impl Omnigraph { table_ops::open_for_mutation_on_branch(self, branch, table_key, op_kind).await } + /// Fork `table_key` onto `active_branch` from the given source state, + /// self-healing a manifest-unreferenced leftover fork if one is in the + /// way. Callers that reach this MUST already hold the per-`(table_key, + /// active_branch)` write queue (so the reclaim cannot race an in-process + /// fork) and must have confirmed via the live manifest that the table is + /// not yet on `active_branch`. Both the first-write fork path + /// (`open_owned_dataset_for_branch_write`) and `branch_merge` satisfy this. pub(crate) async fn fork_dataset_from_entry_state( &self, table_key: &str, @@ -1487,7 +1538,7 @@ impl Omnigraph { source_version: u64, active_branch: &str, ) -> Result { - table_ops::fork_dataset_from_entry_state( + match table_ops::fork_dataset_from_entry_state( self, table_key, full_path, @@ -1495,7 +1546,21 @@ impl Omnigraph { source_version, active_branch, ) - .await + .await? + { + crate::storage_layer::ForkOutcome::Created(ds) => Ok(ds), + crate::storage_layer::ForkOutcome::RefAlreadyExists => { + table_ops::reclaim_orphaned_fork_and_refork( + self, + table_key, + full_path, + source_branch, + source_version, + active_branch, + ) + .await + } + } } pub(crate) async fn reopen_for_mutation( @@ -1530,19 +1595,10 @@ impl Omnigraph { &self, table_key: &str, ds: &mut SnapshotHandle, - ) -> Result<()> { + ) -> Result> { table_ops::build_indices_on_dataset(self, table_key, ds).await } - pub(crate) async fn build_indices_on_dataset_for_catalog( - &self, - catalog: &Catalog, - table_key: &str, - ds: &mut SnapshotHandle, - ) -> Result<()> { - table_ops::build_indices_on_dataset_for_catalog(self, catalog, table_key, ds).await - } - // Used only by in-tree tests (`#[cfg(test)]`); the runtime path now // uses `commit_updates_on_branch_with_expected` exclusively. #[cfg(test)] @@ -2498,25 +2554,49 @@ edge WorksAt: Person -> Company } #[tokio::test] - async fn test_apply_schema_adds_index_for_existing_property() { + async fn test_apply_schema_defers_index_then_reconciler_builds_it() { + // iss-848: schema apply records the @index intent but builds nothing + // inline; a later ensure_indices materializes it once the table has + // rows. (Use `age`, which is unindexed in TEST_SCHEMA — `name @key` is + // already FTS-indexed at seed, so it can't show the deferral.) let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let mut db = Omnigraph::init(uri, TEST_SCHEMA).await.unwrap(); + seed_person_row(&mut db, "Alice", Some(30)).await; - let desired = TEST_SCHEMA.replace("name: String @key", "name: String @key @index"); + let desired = TEST_SCHEMA.replace("age: I32?", "age: I32? @index"); db.apply_schema(&desired).await.unwrap(); + // Apply built nothing — the BTREE on `age` is deferred. let snapshot = db.snapshot().await; let ds = db .storage() .open_snapshot_at_table(&snapshot, "node:Person") .await .unwrap(); - assert!(db.storage().has_fts_index(&ds, "name").await.unwrap()); + assert!( + !db.storage().has_btree_index(&ds, "age").await.unwrap(), + "apply must not build the index inline (deferred to the reconciler)" + ); + + // The reconciler materializes it (Person has a row). + db.ensure_indices().await.unwrap(); + let snapshot = db.snapshot().await; + let ds = db + .storage() + .open_snapshot_at_table(&snapshot, "node:Person") + .await + .unwrap(); + assert!( + db.storage().has_btree_index(&ds, "age").await.unwrap(), + "ensure_indices must build the deferred index" + ); } #[tokio::test] - async fn test_apply_schema_rewrite_preserves_existing_indices() { + async fn test_apply_schema_rewrite_defers_index_then_reconciler_restores() { + // iss-848: an AddProperty rewrite writes a new dataset version without + // rebuilding indexes inline (deferred); ensure_indices restores them. let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); let initial_schema = TEST_SCHEMA.replace("name: String @key", "name: String @key @index"); @@ -2529,6 +2609,8 @@ edge WorksAt: Person -> Company ); db.apply_schema(&desired).await.unwrap(); + // After the rewrite the reconciler restores index coverage. + db.ensure_indices().await.unwrap(); let snapshot = db.snapshot().await; let ds = db .storage() diff --git a/crates/omnigraph/src/db/omnigraph/optimize.rs b/crates/omnigraph/src/db/omnigraph/optimize.rs index 21629a8..9181822 100644 --- a/crates/omnigraph/src/db/omnigraph/optimize.rs +++ b/crates/omnigraph/src/db/omnigraph/optimize.rs @@ -32,6 +32,8 @@ use lance::dataset::cleanup::{CleanupPolicy, RemovalStats}; use lance::dataset::optimize::{ CompactionMetrics, CompactionOptions, compact_files, plan_compaction, }; +use lance::index::DatasetIndexExt; +use lance_index::optimize::OptimizeOptions; use super::*; @@ -138,6 +140,12 @@ pub struct TableOptimizeStats { /// Lance HEAD version observed by optimize for drift skips. `None` for /// normal compaction/no-op/blob skips. pub lance_head_version: Option, + /// Declared `@index` columns on this table the reconciler could not build + /// this run, each with the `reason` (today: a vector column with no + /// trainable vectors yet). Empty on the common path. Reported, not fatal — a + /// later `optimize` retries; the `list_indices`/`indisvalid` analog so + /// operators can see which index is pending and why. + pub pending_indexes: Vec, } impl TableOptimizeStats { @@ -151,6 +159,7 @@ impl TableOptimizeStats { skipped: None, manifest_version: None, lance_head_version: None, + pending_indexes: Vec::new(), } } @@ -164,6 +173,7 @@ impl TableOptimizeStats { skipped: Some(reason), manifest_version: None, lance_head_version: None, + pending_indexes: Vec::new(), } } @@ -181,6 +191,7 @@ impl TableOptimizeStats { skipped: Some(SkipReason::DriftNeedsRepair), manifest_version: Some(manifest_version), lance_head_version: Some(lance_head_version), + pending_indexes: Vec::new(), } } } @@ -257,9 +268,7 @@ pub async fn optimize_all_tables(db: &Omnigraph) -> Result 0; + // Even when there is nothing to compact, the table may still have index + // work: rows appended since the index was built (e.g. via `ingest --mode + // merge`) are scanned unindexed until folded in (needs_reindex), OR a + // declared `@index` was never built — schema apply records the intent but + // defers the physical build (iss-848), so optimize is the operator-facing + // reconciler that materializes it (needs_index_create). Any of the three is + // enough to enter the publish path. If NONE, this table is a no-op and must + // NOT be pinned in a sidecar — a zero-commit pin classifies NoMovement on + // recovery and forces an all-or-nothing rollback of sibling tables' + // legitimate work. Uncovered pre-existing manifest/HEAD drift is skipped + // above and goes through explicit repair, so this only runs on a healthy + // table under the per-table queue + sidecar. + let needs_reindex = TableStore::has_unindexed_fragments(&ds).await?; + // needs_index_work_* checks "a declared index is missing AND row_count > 0", + // so empty tables stay no-ops (never pinned). It re-reads the head under the + // queue we already hold, so it is consistent with `ds`. + let needs_index_create = if let Some(type_name) = table_key.strip_prefix("node:") { + super::table_ops::needs_index_work_node(db, type_name, &table_key, &full_path, None).await? + } else { + super::table_ops::needs_index_work_edge(db, &table_key, &full_path, None).await? + }; + if !will_compact && !needs_reindex && !needs_index_create { return Ok(TableOptimizeStats::compacted( table_key, &CompactionMetrics::default(), @@ -378,8 +405,9 @@ async fn optimize_one_table( )); } - // Phase A: recovery sidecar BEFORE compaction advances the Lance HEAD, so a - // crash before the manifest publish rolls forward on next open. + // Phase A: recovery sidecar BEFORE any HEAD-advancing op (compaction or + // index optimize), so a crash before the manifest publish rolls forward on + // next open. let sidecar = crate::db::manifest::new_sidecar( crate::db::manifest::SidecarKind::Optimize, None, @@ -398,12 +426,50 @@ async fn optimize_one_table( let handle = crate::db::manifest::write_sidecar(db.root_uri(), db.storage_adapter(), &sidecar).await?; - // Phase B: compaction (reserve-fragments + rewrite commits advance HEAD). + // Phase B: compaction (if any) then incremental index optimize — both + // advance Lance HEAD inside the sidecar window. `compact_files` rewrites + // fragments and drops them from existing index segments' coverage; + // `optimize_indices` folds the rewritten and any previously-unindexed + // fragments back in (Lance's incremental merge, not a full retrain). This + // is the same compact -> optimize_indices sequencing LanceDB's `optimize()` + // uses. `optimize_indices` is an inline-commit residual: lance-6.0.1 + // exposes no uncommitted variant, so like `compact_files` it commits + // directly and relies on the sidecar for recovery. let version_before = ds.version().version; - let metrics: CompactionMetrics = compact_files(&mut ds, options, None) + let metrics: CompactionMetrics = if will_compact { + compact_files(&mut ds, options, None) + .await + .map_err(|e| OmniError::Lance(e.to_string()))? + } else { + CompactionMetrics::default() + }; + ds.optimize_indices(&OptimizeOptions::default()) .await - .map_err(|e| OmniError::Lance(e.to_string()))?; - let version_after = ds.version().version; + .map_err(|e| OmniError::Lance(format!("optimize_indices on {}: {}", table_key, e)))?; + + // Materialize any declared-but-missing index over the just-compacted layout, + // reusing the build chokepoint (idempotent: skips existing indexes; fault- + // isolates an untrainable vector column into `pending` rather than failing). + // Run it UNCONDITIONALLY now that we are past the no-op gate — not only when + // `needs_index_create`. A table can enter this path for compaction or + // reindex while its sole missing index is an untrainable Vector column + // (which `needs_index_work_*` does not count as buildable work); calling the + // build here is what surfaces that column in `pending_indexes`, so optimize + // can't compact a table yet silently drop the deferred-index signal. + // Idempotent + cheap when there is nothing to build. Vector index creation + // is an inline-commit residual; the Optimize sidecar's loose post_commit_pin + // covers the extra commits. + let catalog = db.catalog(); + let mut snapshot = crate::storage_layer::SnapshotHandle::new(ds); + let pending_indexes: Vec = + super::table_ops::build_indices_on_dataset_for_catalog( + db, + &catalog, + &table_key, + &mut snapshot, + ) + .await?; + let version_after = snapshot.dataset().version().version; let committed = version_after != version_before; // Pin the per-writer Phase B → Phase C residual for optimize: Lance HEAD has @@ -414,9 +480,6 @@ async fn optimize_one_table( // expected = the version observed under the queue). On failure the sidecar // is intentionally left for the open-time recovery sweep to roll forward. if committed { - // Re-wrap the post-compaction dataset to read its state through the - // trait surface (`table_state` is a read; no HEAD advance). - let snapshot = crate::storage_layer::SnapshotHandle::new(ds); let state = db.storage().table_state(&full_path, &snapshot).await?; let update = crate::db::SubTableUpdate { table_key: table_key.clone(), @@ -443,7 +506,9 @@ async fn optimize_one_table( ); } - Ok(TableOptimizeStats::compacted(table_key, &metrics, committed)) + let mut stat = TableOptimizeStats::compacted(table_key, &metrics, committed); + stat.pending_indexes = pending_indexes; + Ok(stat) } /// Run Lance `cleanup_old_versions` on every node + edge table on `main`, @@ -575,27 +640,37 @@ pub struct BranchReconcileStats { pub failures: Vec<(String, String)>, } -/// Drop every per-table and commit-graph Lance branch that the manifest no -/// longer references. +/// Drop every per-table and commit-graph Lance branch fork the manifest does +/// not reference. /// -/// Orphaned forks arise when a `branch_delete` flips the manifest authority -/// (atomic) but a downstream best-effort reclaim does not complete. They are -/// unreachable through any snapshot — no manifest entry can name them — yet -/// they pin their `tree/{branch}/` storage and can block reusing the branch -/// name. This is the guaranteed convergence backstop: it is idempotent and -/// derived purely from the manifest authority, so it no-ops once everything is -/// reconciled, and it would harmlessly find nothing if a future Lance atomic -/// multi-dataset branch op prevented orphans from forming. +/// Two origins produce a manifest-unreferenced fork: +/// 1. A `branch_delete` flips the manifest authority (atomic) but a +/// downstream best-effort reclaim does not complete — the whole branch is +/// gone from the manifest, but a `tree/{branch}/` ref lingers. +/// 2. A first-write fork (or a merge fork) creates the branch ref before the +/// manifest publish, then the writer dies / is cancelled — the branch is +/// still a live manifest branch, but the manifest's snapshot of it does +/// not place *this table* on the branch. /// -/// The keep-set is the full (unfiltered) manifest branch list, so system -/// branches' forks are never reclaimed; `main`/default is not a named Lance -/// branch and so is never a candidate. Referencing children are dropped before -/// parents (Lance refuses to delete a referenced parent) by ordering longest -/// branch names first. +/// The write path self-heals (2) on the next write to the table +/// (`reclaim_orphaned_fork_and_refork`); this is the guaranteed-convergence +/// backstop that also covers (1) and any table the write path never revisits. +/// +/// The orphan test is therefore **per-table**, not per-branch-name: a Lance +/// branch `B` on table `T` is an orphan iff `B` is not a live manifest branch +/// at all (origin 1) OR the manifest's branch-`B` snapshot does not place `T` +/// on `B` (origin 2). A legitimately-forked table (`table_branch == Some(B)`) +/// is kept. `main` and internal/system branches are never candidates. Lance +/// refuses to force-delete a branch with referencing descendants, so children +/// are dropped before parents (longest name first). Idempotent and authority- +/// derived: no-ops once reconciled, and degrades to finding nothing if a future +/// Lance atomic multi-dataset branch op prevents orphans from forming. pub async fn reconcile_orphaned_branches(db: &Omnigraph) -> Result { - use std::collections::HashSet; + use std::collections::{HashMap, HashSet}; - let keep: HashSet = db + // Live manifest branches: the set whose per-table placements are + // authoritative. A branch absent here is a whole-branch (origin-1) orphan. + let live_branches: HashSet = db .coordinator .read() .await @@ -616,6 +691,12 @@ pub async fn reconcile_orphaned_branches(db: &Omnigraph) -> Result = HashMap::new(); + let mut failed_branch_snapshots: HashSet = HashSet::new(); // Per-table fault isolation: one table's transient failure is recorded and // logged, never aborting the rest of the sweep. @@ -634,7 +715,104 @@ pub async fn reconcile_orphaned_branches(db: &Omnigraph) -> Result = Vec::new(); + for branch in listed { + // `main` is not a named Lance branch; system/internal branches + // (e.g. the schema-apply lock) own legitimate forks — never touch. + if branch == "main" || crate::db::is_internal_system_branch(&branch) { + continue; + } + let is_orphan = if !live_branches.contains(&branch) { + true // origin 1: whole branch gone from the manifest + } else { + // origin 2: live branch, but does the manifest place THIS + // table on it? Resolve (and cache) the branch's snapshot. + if failed_branch_snapshots.contains(&branch) { + continue; + } + if !branch_snapshots.contains_key(&branch) { + let branch_snapshot = + match crate::failpoints::maybe_fail("cleanup.resolve_branch_snapshot") { + Ok(()) => db.snapshot_for_branch(Some(&branch)).await, + Err(injected) => Err(injected), + }; + match branch_snapshot { + Ok(snap) => { + branch_snapshots.insert(branch.clone(), snap); + } + Err(err) => { + tracing::warn!( + target: "omnigraph::cleanup", + table = %table_key, + branch = %branch, + error = %err, + "resolving branch snapshot failed during reconcile; skipping", + ); + stats.failures.push((table_key.clone(), err.to_string())); + failed_branch_snapshots.insert(branch.clone()); + continue; + } + } + } + branch_snapshots[&branch] + .entry(&table_key) + .map(|e| e.table_branch.as_deref() != Some(branch.as_str())) + .unwrap_or(true) + }; + if is_orphan { + orphans.push(branch); + } + } + // Children before parents (longest name first) so Lance's referenced- + // parent RefConflict cannot block reclamation. + orphans.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b))); + + for branch in orphans { + // Serialize against in-process live writers before destroying a ref. + // A first-write fork holds the per-(table, branch) write queue from + // before the fork through the manifest publish; on a LIVE branch its + // in-flight fork looks exactly like an origin-2 orphan (manifest not + // yet advanced). Acquire the same queue so cleanup waits for any such + // writer, then RE-VALIDATE under the queue with a fresh read: if the + // writer published in the meantime (table now placed on the branch), + // it is no longer an orphan — skip it. (Cross-process writers remain + // the documented one-winner-CAS gap.) One key held at a time → no + // lock-order inversion against multi-table `acquire_many` writers. + let _guard = db + .write_queue() + .acquire(&(table_key.clone(), Some(branch.clone()))) + .await; + // Decide under the queue from FRESH authority via the shared + // classifier (same decision the write-path reclaim uses) — never + // from the sweep-start `live_branches` capture. A branch created + // AFTER that capture is absent from the stale set yet may already + // carry a legitimately-published fork (an in-process writer held + // this queue through its fork+publish; we just waited on it), so a + // stale "origin-1 ⇒ delete" shortcut would destroy a live fork. + // Only `Orphan` is reclaimed; `Indeterminate` (transient read) is + // skipped and recorded. (Cross-process writers remain the documented + // one-winner-CAS gap.) One key held at a time → no lock-order + // inversion vs multi-table `acquire_many` writers. + match super::table_ops::classify_fork_ref(db, &table_key, &branch).await { + super::table_ops::ForkRefStatus::Orphan => {} + super::table_ops::ForkRefStatus::Legitimate => continue, + super::table_ops::ForkRefStatus::Indeterminate => { + tracing::warn!( + target: "omnigraph::cleanup", + table = %table_key, + branch = %branch, + "fresh re-check inconclusive during reconcile; skipping to avoid \ + destroying a possibly-live fork (will retry next cleanup)", + ); + stats.failures.push(( + table_key.clone(), + format!("indeterminate fork status for {branch}"), + )); + continue; + } + } let outcome = match crate::failpoints::maybe_fail("cleanup.reconcile_fork") { Ok(()) => storage.force_delete_branch(&full_path, &branch).await, Err(injected) => Err(injected), @@ -655,15 +833,17 @@ pub async fn reconcile_orphaned_branches(db: &Omnigraph) -> Result keys.sort(); keys } + +#[cfg(all(test, feature = "failpoints"))] +mod tests { + use super::*; + use crate::failpoints::ScopedFailPoint; + use crate::loader::{LoadMode, load_jsonl}; + + fn node_table_uri(root: &str, type_name: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for &b in type_name.as_bytes() { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100_0000_01b3); + } + format!("{}/nodes/{hash:016x}", root.trim_end_matches('/')) + } + + #[tokio::test] + async fn reconcile_caches_live_branch_snapshot_resolution_failure() { + let _scenario = fail::FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let schema = "node Person { name: String @key }\nnode Company { name: String @key }\n"; + let mut db = Omnigraph::init(uri, schema).await.unwrap(); + load_jsonl( + &mut db, + "{\"type\":\"Person\",\"data\":{\"name\":\"Alice\"}}\n\ + {\"type\":\"Company\",\"data\":{\"name\":\"Acme\"}}", + LoadMode::Merge, + ) + .await + .unwrap(); + db.branch_create("feature").await.unwrap(); + + for type_name in ["Person", "Company"] { + let table_uri = node_table_uri(uri, type_name); + let mut ds = lance::Dataset::open(&table_uri).await.unwrap(); + let base = ds.version().version; + ds.create_branch("feature", base, None).await.unwrap(); + } + + let _fp = ScopedFailPoint::new("cleanup.resolve_branch_snapshot", "return"); + let stats = reconcile_orphaned_branches(&db).await.unwrap(); + + assert_eq!( + stats.failures.len(), + 1, + "one live-branch snapshot resolution failure should be reported once, \ + not once per table: {:?}", + stats.failures + ); + assert!( + stats.failures[0] + .1 + .contains("cleanup.resolve_branch_snapshot"), + "the recorded failure should be the branch-snapshot resolution failure: {:?}", + stats.failures + ); + assert!( + stats.reclaimed.is_empty(), + "unreadable live-branch refs must be left for the next cleanup run" + ); + } +} diff --git a/crates/omnigraph/src/db/omnigraph/schema_apply.rs b/crates/omnigraph/src/db/omnigraph/schema_apply.rs index f965ad4..48f8099 100644 --- a/crates/omnigraph/src/db/omnigraph/schema_apply.rs +++ b/crates/omnigraph/src/db/omnigraph/schema_apply.rs @@ -193,7 +193,6 @@ where let mut added_tables = BTreeSet::new(); let mut renamed_tables = HashMap::new(); let mut rewritten_tables = BTreeSet::new(); - let mut indexed_tables = BTreeSet::new(); let mut dropped_tables = BTreeSet::new(); // Hard-drop cleanup targets: (table_key, full_dataset_uri). // Populated for DropProperty { Hard } and DropType { Hard }; the @@ -252,14 +251,14 @@ where .or_default() .insert(to.clone(), from.clone()); } - SchemaMigrationStep::AddConstraint { - type_kind, - type_name, - .. - } => { - indexed_tables.insert(schema_table_key(*type_kind, type_name)); - } - SchemaMigrationStep::UpdateTypeMetadata { .. } + // AddConstraint is only ever an `@index` addition (every other + // added constraint plans as UnsupportedChange). It records intent + // in the desired catalog/IR; the physical index is built off the + // critical path by ensure_indices/optimize (iss-848), so the apply + // does no table work for it — a pure metadata change like the two + // metadata steps below. + SchemaMigrationStep::AddConstraint { .. } + | SchemaMigrationStep::UpdateTypeMetadata { .. } | SchemaMigrationStep::UpdatePropertyMetadata { .. } => {} SchemaMigrationStep::DropProperty { type_kind, @@ -347,18 +346,15 @@ where let mut table_updates = HashMap::::new(); let mut table_tombstones = HashMap::::new(); - // Recovery sidecar: protect the per-table commit_staged loop in - // rewritten_tables + indexed_tables. The post_commit_pin we record - // here is a lower bound (expected + 1); the classifier loose-matches - // for SidecarKind::SchemaApply because the actual N depends on how - // many indices need building. See classify_table's loose-match arm. + // Recovery sidecar: protect the per-table `stage_overwrite` + + // `commit_staged` in rewritten_tables — the only tables that advance Lance + // HEAD inline now that index building is deferred to the reconciler + // (iss-848). Each rewritten table is exactly one commit, so + // `post_commit_pin = expected + 1` is now exact (it was a loose lower bound + // when index builds added extra commits); the classifier's loose-match for + // SidecarKind::SchemaApply still accepts it. let recovery_pins: Vec = rewritten_tables .iter() - .chain(indexed_tables.iter().filter(|t| { - !rewritten_tables.contains(*t) - && !added_tables.contains(*t) - && !renamed_tables.contains_key(*t) - })) .filter_map(|table_key| { let entry = snapshot.entry(table_key)?; Some(crate::db::manifest::SidecarTablePin { @@ -432,10 +428,10 @@ where // manifest publish via `commit_changes_with_actor` below. // // Schema-apply already holds the graph-wide `__schema_apply_lock__` - // sentinel branch, so under PR 1b's intermediate state these - // per-table acquisitions are uncontended. They exist for symmetry - // with future MR-870 recovery, which will need queue acquisition - // before any `Dataset::restore` it issues for SchemaApply sidecars. + // sentinel branch, so these per-table acquisitions are uncontended in + // practice. They exist for symmetry with the recovery reconciler, which + // acquires the same queues before any `Dataset::restore` it issues for + // SchemaApply sidecars. let mut schema_apply_queue_keys: Vec<(String, Option)> = recovery_pins .iter() .map(|pin| (pin.table_key.clone(), pin.table_branch.clone())) @@ -490,10 +486,11 @@ where let table_path = table_path_for_table_key(table_key)?; let dataset_uri = db.storage().dataset_uri(&table_path); let schema = schema_for_table_key(&desired_catalog, table_key)?; - let mut ds = + let ds = SnapshotHandle::new(TableStore::create_empty_dataset(&dataset_uri, &schema).await?); - db.build_indices_on_dataset_for_catalog(&desired_catalog, table_key, &mut ds) - .await?; + // Indexes for the new table are materialized off the critical path by + // ensure_indices/optimize (iss-848); a 0-row table is never trainable + // anyway. The @index intent is recorded in the persisted catalog/IR. let state = db.storage().table_state(&dataset_uri, &ds).await?; table_registrations.insert(table_key.clone(), table_path); table_updates.insert( @@ -533,10 +530,9 @@ where .await?; let table_path = table_path_for_table_key(target_table_key)?; let dataset_uri = db.storage().dataset_uri(&table_path); - let mut target_ds = + let target_ds = SnapshotHandle::new(TableStore::write_dataset(&dataset_uri, batch).await?); - db.build_indices_on_dataset_for_catalog(&desired_catalog, target_table_key, &mut target_ds) - .await?; + // Indexes on the renamed table are reconciled later (iss-848). let state = db.storage().table_state(&dataset_uri, &target_ds).await?; table_registrations.insert(target_table_key.clone(), table_path); table_updates.insert( @@ -593,9 +589,10 @@ where .open_dataset_head_for_write(table_key, &dataset_uri, entry.table_branch.as_deref()) .await?; let staged = db.storage().stage_overwrite(&existing, batch).await?; - let mut target_ds = db.storage().commit_staged(existing, staged).await?; - db.build_indices_on_dataset_for_catalog(&desired_catalog, table_key, &mut target_ds) - .await?; + let target_ds = db.storage().commit_staged(existing, staged).await?; + // The rewrite drops the table's existing index coverage; it is + // restored off the critical path by optimize's optimize_indices / + // ensure_indices (iss-848). Reads scan uncovered fragments meanwhile. let state = db.storage().table_state(&dataset_uri, &target_ds).await?; table_updates.insert( table_key.clone(), @@ -609,41 +606,12 @@ where ); } - for table_key in &indexed_tables { - if added_tables.contains(table_key) - || renamed_tables.contains_key(table_key) - || rewritten_tables.contains(table_key) - { - continue; - } - let entry = snapshot.entry(table_key).ok_or_else(|| { - OmniError::manifest(format!( - "missing table '{}' for schema index apply", - table_key - )) - })?; - ensure_snapshot_entry_head_matches(db, entry).await?; - let dataset_uri = db.storage().dataset_uri(&entry.table_path); - let mut ds = db - .storage() - .open_dataset_head_for_write(table_key, &dataset_uri, entry.table_branch.as_deref()) - .await?; - db.storage() - .ensure_expected_version(&ds, table_key, entry.table_version)?; - db.build_indices_on_dataset_for_catalog(&desired_catalog, table_key, &mut ds) - .await?; - let state = db.storage().table_state(&dataset_uri, &ds).await?; - table_updates.insert( - table_key.clone(), - crate::db::SubTableUpdate { - table_key: table_key.clone(), - table_version: state.version, - table_branch: None, - row_count: state.row_count, - version_metadata: state.version_metadata, - }, - ); - } + // Index-only changes (AddConstraint, i.e. adding an `@index`) are pure + // metadata: the new `@index` intent is recorded in the desired catalog/IR + // persisted below, and the physical index is materialized off the critical + // path by `ensure_indices`/`optimize` (iss-848). Schema apply touches no + // table data for them, so there is no per-table loop here and no recovery + // pin (no Lance HEAD advances). Reads stay correct meanwhile via a scan. let mut manifest_changes = Vec::new(); for (table_key, table_path) in table_registrations { diff --git a/crates/omnigraph/src/db/omnigraph/table_ops.rs b/crates/omnigraph/src/db/omnigraph/table_ops.rs index f7a365a..d30acff 100644 --- a/crates/omnigraph/src/db/omnigraph/table_ops.rs +++ b/crates/omnigraph/src/db/omnigraph/table_ops.rs @@ -21,7 +21,7 @@ pub(super) async fn graph_index_for_resolved( db.runtime_cache.graph_index(resolved, &catalog).await } -pub(super) async fn ensure_indices(db: &Omnigraph) -> Result<()> { +pub(super) async fn ensure_indices(db: &Omnigraph) -> Result> { let current_branch = db .coordinator .read() @@ -31,7 +31,7 @@ pub(super) async fn ensure_indices(db: &Omnigraph) -> Result<()> { ensure_indices_for_branch(db, current_branch.as_deref()).await } -pub(super) async fn ensure_indices_on(db: &Omnigraph, branch: &str) -> Result<()> { +pub(super) async fn ensure_indices_on(db: &Omnigraph, branch: &str) -> Result> { let branch = normalize_branch_name(branch)?; ensure_indices_for_branch(db, branch.as_deref()).await } @@ -73,12 +73,16 @@ pub(super) async fn failpoint_publish_table_head_without_index_rebuild_for_test( .await } -pub(super) async fn ensure_indices_for_branch(db: &Omnigraph, branch: Option<&str>) -> Result<()> { +pub(super) async fn ensure_indices_for_branch( + db: &Omnigraph, + branch: Option<&str>, +) -> Result> { db.ensure_schema_state_valid().await?; db.ensure_schema_apply_idle("ensure_indices").await?; let resolved = db.resolved_branch_target(branch).await?; let snapshot = resolved.snapshot; let mut updates = Vec::new(); + let mut pending = Vec::new(); let active_branch = resolved.branch; let catalog = db.catalog(); @@ -160,9 +164,8 @@ pub(super) async fn ensure_indices_for_branch(db: &Omnigraph, branch: Option<&st // that needs index work. Held across the per-table commit loop and // the manifest publish at the end of this function. Sorted-order // acquisition prevents lock-order inversion against concurrent - // multi-table writers (mutation finalize, branch_merge, future - // MR-870 recovery). Under PR 1b's intermediate state (global server - // RwLock still in place), this acquisition is uncontended. + // multi-table writers (mutation finalize, branch_merge, the fork + // path, recovery). let queue_keys: Vec<(String, Option)> = recovery_pins .iter() .map(|pin| (pin.table_key.clone(), pin.table_branch.clone())) @@ -217,7 +220,7 @@ pub(super) async fn ensure_indices_for_branch(db: &Omnigraph, branch: Option<&st }; let row_count = db.storage().count_rows(&ds, None).await.unwrap_or(0); if row_count > 0 { - build_indices_on_dataset(db, &table_key, &mut ds).await?; + pending.extend(build_indices_on_dataset(db, &table_key, &mut ds).await?); } let state = db.storage().table_state(&full_path, &ds).await?; @@ -265,7 +268,7 @@ pub(super) async fn ensure_indices_for_branch(db: &Omnigraph, branch: Option<&st }; let row_count = db.storage().count_rows(&ds, None).await.unwrap_or(0); if row_count > 0 { - build_indices_on_dataset(db, &table_key, &mut ds).await?; + pending.extend(build_indices_on_dataset(db, &table_key, &mut ds).await?); } let state = db.storage().table_state(&full_path, &ds).await?; @@ -307,7 +310,69 @@ pub(super) async fn ensure_indices_for_branch(db: &Omnigraph, branch: Option<&st } } - Ok(()) + Ok(pending) +} + +/// The single scalar/vector index a node property receives from a one-column +/// `@index`/`@key` declaration, or `None` when the property type is not +/// indexable here (a list column or `Blob`). +/// +/// Shared by `build_indices_on_dataset_for_catalog` (which builds the index) +/// and `needs_index_work_node` (which checks coverage to decide recovery- +/// sidecar pinning) so the two cannot drift: an enum or orderable scalar the +/// builder gives a BTREE must also be reported as "needs work" until that +/// BTREE exists, or the HEAD-advancing build would run without sidecar cover. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum NodePropIndexKind { + Btree, + Fts, + Vector, +} + +fn node_prop_index_kind(prop_type: &PropType) -> Option { + if prop_type.list { + return None; + } + // Enums are physically `String` but filtered by equality, so they take a + // scalar BTREE, not an FTS inverted index (Lance never consults an inverted + // index for `=`/range). Free-text Strings keep FTS for + // `search()`/`match_text`/`bm25`. + let is_enum = prop_type.enum_values.is_some(); + match prop_type.scalar { + ScalarType::String if !is_enum => Some(NodePropIndexKind::Fts), + ScalarType::Vector(_) => Some(NodePropIndexKind::Vector), + ScalarType::String + | ScalarType::DateTime + | ScalarType::Date + | ScalarType::I32 + | ScalarType::I64 + | ScalarType::U32 + | ScalarType::U64 + | ScalarType::F32 + | ScalarType::F64 + | ScalarType::Bool => Some(NodePropIndexKind::Btree), + ScalarType::Blob => None, + } +} + +/// Whether a vector column currently has at least one non-null vector — the +/// minimum for Lance IVF k-means to train (the `ivf_flat(1)` index we build +/// needs >=1 vector). Used identically by `needs_index_work_node` (so an +/// untrainable column is not pinned for recovery — avoiding a zero-commit pin +/// that would roll back a sibling's index work) and by the vector build arm (so +/// `create_vector_index` is only attempted when it can succeed, keeping its +/// genuine errors fatal instead of swallowed as pending). If index params +/// become size-aware (dev-graph iss-687), this threshold moves with them. +async fn vector_column_trainable( + db: &Omnigraph, + ds: &SnapshotHandle, + column: &str, +) -> Result { + Ok(db + .storage() + .count_rows(ds, Some(format!("{column} IS NOT NULL"))) + .await? + > 0) } /// Returns true if the node table is missing at least one declared @@ -318,12 +383,13 @@ pub(super) async fn ensure_indices_for_branch(db: &Omnigraph, branch: Option<&st /// would force `NoMovement` classification on recovery and trigger the /// all-or-nothing rollback of sibling tables' legitimate index work). /// -/// Per the actual `build_indices_on_dataset_for_catalog` implementation -/// (this file, ~line 419-491), nodes get BTree (id) + per-prop FTS -/// (@search String) + per-prop Vector indices; edges get BTree only -/// (id, src, dst). The two helpers mirror that asymmetry — see the -/// `needs_index_work_edge` doc comment. -async fn needs_index_work_node( +/// Per `build_indices_on_dataset_for_catalog`, nodes get BTree (id) plus, for +/// each one-column `@index`/`@key` property, the index `node_prop_index_kind` +/// assigns: a scalar BTREE for enums and orderable scalars +/// (DateTime/Date/numeric/Bool), FTS for free-text Strings, or a Vector index. +/// Edges get BTree only (id, src, dst). This helper and the builder share +/// `node_prop_index_kind` so they cannot drift — see its doc comment. +pub(super) async fn needs_index_work_node( db: &Omnigraph, type_name: &str, table_key: &str, @@ -359,14 +425,30 @@ async fn needs_index_work_node( let Some(prop_type) = node_type.properties.get(prop_name) else { continue; }; - if matches!(prop_type.scalar, ScalarType::String) && !prop_type.list { - if !db.storage().has_fts_index(&ds, prop_name).await? { - return Ok(true); + match node_prop_index_kind(prop_type) { + Some(NodePropIndexKind::Fts) => { + if !db.storage().has_fts_index(&ds, prop_name).await? { + return Ok(true); + } } - } else if matches!(prop_type.scalar, ScalarType::Vector(_)) && !prop_type.list { - if !db.storage().has_vector_index(&ds, prop_name).await? { - return Ok(true); + Some(NodePropIndexKind::Vector) => { + // Only count a missing vector index as buildable *work* when the + // column is trainable (>=1 non-null vector). An untrainable + // column would defer in the build and commit nothing; pinning it + // for recovery would be a zero-commit pin that classifies + // NoMovement and rolls back a sibling table's index work. + if !db.storage().has_vector_index(&ds, prop_name).await? + && vector_column_trainable(db, &ds, prop_name).await? + { + return Ok(true); + } } + Some(NodePropIndexKind::Btree) => { + if !db.storage().has_btree_index(&ds, prop_name).await? { + return Ok(true); + } + } + None => {} } } Ok(false) @@ -382,7 +464,7 @@ async fn needs_index_work_node( /// /// Empty edge tables are skipped by the ensure_indices loop the same /// way node tables are; see `needs_index_work_node`. -async fn needs_index_work_edge( +pub(super) async fn needs_index_work_edge( db: &Omnigraph, table_key: &str, full_path: &str, @@ -499,8 +581,14 @@ pub(super) async fn open_owned_dataset_for_branch_write( )); } } - fork_dataset_from_entry_state( - db, + // The fork advances Lance state before the manifest publish. The + // caller holds the per-(table, active_branch) write queue from + // before this fork through the publish, so a leftover ref is a + // manifest-unreferenced fork (interrupted prior fork, or + // delete+recreate), not a live in-process fork. The wrapper + // self-heals it (reclaim + re-fork); see + // `Omnigraph::fork_dataset_from_entry_state`. + db.fork_dataset_from_entry_state( table_key, full_path, source_branch, @@ -528,7 +616,7 @@ pub(super) async fn fork_dataset_from_entry_state( source_branch: Option<&str>, source_version: u64, active_branch: &str, -) -> Result { +) -> Result> { db.storage() .fork_branch_from_state( full_path, @@ -540,6 +628,172 @@ pub(super) async fn fork_dataset_from_entry_state( .await } +/// Classification of a Lance branch ref `B` on table `T` against FRESH manifest +/// authority — the single decision both fork-ref reclaim sites share: the +/// write-path reclaim ([`reclaim_orphaned_fork_and_refork`]) and the cleanup +/// reconciler (`optimize::reconcile_orphaned_branches`). Having one classifier +/// keeps the two destructive sites from drifting (the bug history: each was +/// hardened separately and the other lagged). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ForkRefStatus { + /// The manifest places `T` on `B` — a legitimate fork. Never destroy. + Legitimate, + /// The manifest does not reference this fork (`T` not on `B`, or `B` absent + /// from the manifest entirely). Reclaimable. + Orphan, + /// Fresh authority could not be established (a transient read failure on a + /// live branch). Ambiguous — do not destroy; the caller retries / converges. + Indeterminate, +} + +/// Classify a fork ref from FRESH manifest authority (bypasses the coordinator +/// cache). MUST be called with the per-`(table, branch)` write queue held, so +/// the classification is stable against in-process writers for the caller's +/// critical section. Both reclaim sites map the result to their own action +/// (write path: reclaim vs retryable; cleanup: delete vs skip), but the +/// destroy-only-on-`Orphan` rule is enforced here, once. +pub(crate) async fn classify_fork_ref( + db: &Omnigraph, + table_key: &str, + branch: &str, +) -> ForkRefStatus { + // `classify.fresh_read` failpoint: simulate a transient failure of the + // fresh-authority read (no-op without the `failpoints` feature). Lets a + // test exercise the Indeterminate path — a read failure on a live branch + // must classify as Indeterminate (skip), never Orphan (destroy). + let fresh = match crate::failpoints::maybe_fail("classify.fresh_read") { + Ok(()) => db.fresh_snapshot_for_branch(Some(branch)).await, + Err(injected) => Err(injected), + }; + match fresh { + Ok(snap) => { + let placed = snap + .entry(table_key) + .map(|e| e.table_branch.as_deref() == Some(branch)) + .unwrap_or(false); + if placed { + ForkRefStatus::Legitimate + } else { + // Branch resolves but the manifest does not place this table on + // it — a manifest-unreferenced fork. + ForkRefStatus::Orphan + } + } + // Branch did not resolve. `all_branches` lists `_refs/branches/` live, so + // absent there = genuinely no such manifest branch (origin-1 orphan); + // present (or a list error) = transient read — never destroy on that. + Err(_) => match db.coordinator.read().await.all_branches().await { + Ok(fresh) if !fresh.iter().any(|b| b == branch) => ForkRefStatus::Orphan, + _ => ForkRefStatus::Indeterminate, + }, + } +} + +/// Reclaim a manifest-unreferenced fork and re-fork in its place. +/// +/// Reached when `fork_branch_from_state` reports `RefAlreadyExists`. This is a +/// destructive op (it force-deletes a Lance branch ref), so it owns its own +/// safety precondition rather than trusting the caller's: it re-derives, via +/// [`classify_fork_ref`], that the manifest does not place this table on +/// `active_branch`. The caller's earlier proof may have come from the +/// coordinator's *cached* branch snapshot (`resolved_branch_target` returns +/// the cache when the handle is bound to `active_branch` — an embedded handle +/// on the branch, or `branch_merge`'s target swap); trusting it could +/// force-delete a fork a concurrent writer just legitimately published. Only +/// once fresh authority confirms the ref is unreferenced does it drop the ref +/// (idempotent `force_delete_branch`) and re-fork, exactly once. +/// +/// If fresh authority shows the table IS on `active_branch` (a legitimate +/// concurrent fork), or a second collision occurs after reclaim (a foreign- +/// process writer recreated the ref — the documented one-winner-CAS gap), it +/// surfaces a retryable conflict; on retry the winner's fork is visible and +/// the no-fork path runs. +pub(super) async fn reclaim_orphaned_fork_and_refork( + db: &Omnigraph, + table_key: &str, + full_path: &str, + source_branch: Option<&str>, + source_version: u64, + active_branch: &str, +) -> Result { + // Self-validate against FRESH authority before destroying anything. Only an + // Orphan is reclaimable; a Legitimate status (a concurrent writer published + // a real fork despite the caller's possibly-cached proof) or an + // Indeterminate one (transient read) surfaces a retryable conflict rather + // than stranding the manifest at a version the recreated ref won't have. + match classify_fork_ref(db, table_key, active_branch).await { + ForkRefStatus::Orphan => {} + ForkRefStatus::Legitimate => { + let actual = db + .fresh_snapshot_for_branch(Some(active_branch)) + .await + .ok() + .and_then(|s| s.entry(table_key).map(|e| e.table_version)) + .unwrap_or(source_version); + return Err(OmniError::manifest_expected_version_mismatch( + table_key, + source_version, + actual, + )); + } + ForkRefStatus::Indeterminate => { + return Err(OmniError::manifest_conflict(format!( + "could not verify whether branch '{active_branch}' still owns an orphaned \ + fork for table '{table_key}' because fresh manifest authority was \ + unavailable; refresh and retry" + ))); + } + } + + crate::failpoints::maybe_fail("fork.before_reclaim")?; + db.storage() + .force_delete_branch(full_path, active_branch) + .await + .map_err(|e| { + // Lance refuses to delete a branch with dependent child branches + // even under force (RefConflict). Unreachable for a leaf first-write + // fork (the cleanup reconciler also drops children before parents), + // but surface it actionably if it ever happens. We match loosely on + // "referenc" rather than the exact prose, which is not a Lance API + // contract; a typed RefConflict variant through `force_delete_branch` + // is the durable follow-up. + if e.to_string().contains("referenc") { + OmniError::manifest_conflict(format!( + "branch '{active_branch}' cannot reclaim the leftover fork for \ + table '{table_key}' because it has dependent child branches; \ + delete the child branches (or run `omnigraph cleanup`) first" + )) + } else { + e + } + })?; + + match fork_dataset_from_entry_state( + db, + table_key, + full_path, + source_branch, + source_version, + active_branch, + ) + .await? + { + crate::storage_layer::ForkOutcome::Created(ds) => Ok(ds), + crate::storage_layer::ForkOutcome::RefAlreadyExists => { + let live = db.fresh_snapshot_for_branch(Some(active_branch)).await?; + let actual = live + .entry(table_key) + .map(|e| e.table_version) + .unwrap_or(source_version); + Err(OmniError::manifest_expected_version_mismatch( + table_key, + source_version, + actual, + )) + } + } +} + pub(super) async fn reopen_for_mutation( db: &Omnigraph, table_key: &str, @@ -580,11 +834,25 @@ pub(super) async fn open_dataset_at_state( .await } +/// A declared index the builder could not materialize on this pass. Today the +/// only such case is a vector (IVF) column with no trainable vectors yet +/// (KMeans needs >=1 vector), e.g. the load-before-embed window. Reported, not +/// fatal: a later `ensure_indices`/`optimize` retries once the column is +/// buildable, and reads stay correct via brute-force meanwhile. Surfacing +/// pending index *status* rather than failing the operation is the database +/// norm (Postgres `indisvalid`, LanceDB `list_indices`). +#[derive(Debug, Clone)] +pub struct PendingIndex { + pub table_key: String, + pub column: String, + pub reason: String, +} + pub(super) async fn build_indices_on_dataset( db: &Omnigraph, table_key: &str, ds: &mut SnapshotHandle, -) -> Result<()> { +) -> Result> { let catalog = db.catalog(); build_indices_on_dataset_for_catalog(db, &catalog, table_key, ds).await } @@ -594,8 +862,9 @@ pub(super) async fn build_indices_on_dataset_for_catalog( catalog: &Catalog, table_key: &str, ds: &mut SnapshotHandle, -) -> Result<()> { +) -> Result> { if let Some(type_name) = table_key.strip_prefix("node:") { + let mut pending = Vec::new(); if !db.storage().has_btree_index(ds, "id").await? { stage_and_commit_btree(db, table_key, ds, &["id"]).await?; } @@ -615,35 +884,79 @@ pub(super) async fn build_indices_on_dataset_for_catalog( } let prop_name = &index_cols[0]; if let Some(prop_type) = node_type.properties.get(prop_name) { - if matches!(prop_type.scalar, ScalarType::String) && !prop_type.list { - if !db.storage().has_fts_index(ds, prop_name).await? { - stage_and_commit_inverted(db, table_key, ds, prop_name.as_str()) - .await?; + match node_prop_index_kind(prop_type) { + Some(NodePropIndexKind::Fts) => { + if !db.storage().has_fts_index(ds, prop_name).await? { + stage_and_commit_inverted(db, table_key, ds, prop_name.as_str()) + .await?; + } } - } else if matches!(prop_type.scalar, ScalarType::Vector(_)) && !prop_type.list { - if !db.storage().has_vector_index(ds, prop_name).await? { - // Inline-commit residual: lance-6.0.1 does not - // expose `build_index_metadata_from_segments` as - // `pub`, so vector indices cannot be staged from - // outside the lance crate. Document at the call - // site; companion ticket to lance-format/lance#6658. - let new_snap = db - .storage_inline_residual() - .create_vector_index(ds.clone(), prop_name.as_str()) - .await - .map_err(|e| { - OmniError::Lance(format!( - "create Vector index on {}({}): {}", - table_key, prop_name, e - )) - })?; - *ds = new_snap; + Some(NodePropIndexKind::Vector) => { + if !db.storage().has_vector_index(ds, prop_name).await? { + // A vector (IVF) index trains k-means over the column, + // so it needs >=1 non-null vector (KMeans errors + // "cannot train N centroids with 0 vectors"). Precheck + // trainability: a column with no vectors yet (e.g. rows + // loaded before `embed`) is recorded as a *pending* + // index and skipped — deferred, not failed. The SAME + // predicate gates `needs_index_work_node`, so an + // untrainable column is never pinned for recovery (no + // zero-commit pin that would roll back a sibling + // table's index work). This function is the chokepoint + // every write path funnels through (load/mutate, schema + // apply, ensure_indices, optimize, merge), realizing + // the governing principle — physical index state never + // fails a logical operation. Only when trainable do we + // attempt the build, and then we PROPAGATE any error: a + // genuine I/O/manifest/Lance failure must stay fatal, + // not be hidden as pending. (Vector creation is an + // inline-commit residual until lance#6666; iss-951.) + if vector_column_trainable(db, ds, prop_name).await? { + let new_snap = db + .storage_inline_residual() + .create_vector_index(ds.clone(), prop_name.as_str()) + .await + .map_err(|e| { + OmniError::Lance(format!( + "create Vector index on {}({}): {}", + table_key, prop_name, e + )) + })?; + *ds = new_snap; + } else { + tracing::info!( + target: "omnigraph::index", + table = %table_key, + column = %prop_name, + "deferring Vector index: column has no \ + trainable vectors yet", + ); + pending.push(PendingIndex { + table_key: table_key.to_string(), + column: prop_name.clone(), + reason: "column has no non-null vectors to \ + train on yet" + .to_string(), + }); + } + } } + // Enum + orderable scalars (DateTime/Date/numeric/Bool) + // get a BTREE so `=`, range, IN, and IS NULL are index- + // accelerated instead of degrading to a full scan. + Some(NodePropIndexKind::Btree) => { + if !db.storage().has_btree_index(ds, prop_name).await? { + stage_and_commit_btree(db, table_key, ds, &[prop_name.as_str()]) + .await?; + } + } + // List or Blob column: not indexable as a scalar here. + None => {} } } } } - return Ok(()); + return Ok(pending); } if table_key.starts_with("edge:") { @@ -656,7 +969,9 @@ pub(super) async fn build_indices_on_dataset_for_catalog( if !db.storage().has_btree_index(ds, "dst").await? { stage_and_commit_btree(db, table_key, ds, &["dst"]).await?; } - return Ok(()); + // Edge tables only get BTree (id/src/dst), which build at any + // cardinality; no pending state is possible here. + return Ok(Vec::new()); } Err(OmniError::manifest(format!( @@ -778,7 +1093,11 @@ async fn prepare_updates_for_commit( crate::db::MutationOpKind::SchemaRewrite, ) .await?; - build_indices_on_dataset(db, &prepared_update.table_key, &mut ds).await?; + // Any column not yet buildable (e.g. a vector column whose rows + // have null embeddings) is deferred and logged inside + // build_indices; a later ensure_indices/optimize materializes it. + // The load/mutate/merge commit must not fail on it. + let _pending = build_indices_on_dataset(db, &prepared_update.table_key, &mut ds).await?; let state = db.storage().table_state(&full_path, &ds).await?; prepared_update.table_version = state.version; prepared_update.row_count = state.row_count; @@ -979,3 +1298,78 @@ pub(super) async fn ensure_commit_graph_initialized(db: &Omnigraph) -> Result<() pub(super) async fn invalidate_graph_index(db: &Omnigraph) { db.runtime_cache.invalidate_all().await; } + +#[cfg(test)] +mod classify_fork_ref_tests { + //! Direct coverage of [`classify_fork_ref`] — the single fresh-authority + //! decision both fork-ref reclaim sites (write-path reclaim + cleanup + //! reconciler) route through. Pins each deterministic status so reverting + //! the fresh-authority logic at either site fails here. (The `Indeterminate` + //! arm needs an injected transient read and is covered under the + //! `failpoints` suite.) + use super::*; + use crate::db::Omnigraph; + use crate::loader::LoadMode; + + const SCHEMA: &str = "node Person { name: String @key }\nnode Company { name: String @key }\n"; + + /// On-disk dataset path for a node table, taken from the manifest entry + /// (the same path the engine uses) so the test forges against the real ref. + async fn node_path(db: &Omnigraph, branch: &str, table_key: &str) -> String { + let snap = db.snapshot_for_branch(Some(branch)).await.unwrap(); + let entry = snap.entry(table_key).unwrap(); + format!("{}/{}", db.root_uri, entry.table_path) + } + + #[tokio::test] + async fn classify_distinguishes_legitimate_unreferenced_and_ghost() { + let dir = tempfile::tempdir().unwrap(); + let db = Omnigraph::init(dir.path().to_str().unwrap(), SCHEMA) + .await + .unwrap(); + db.branch_create("feature").await.unwrap(); + + // Legitimate: a real write forks Company onto `feature`, and the + // manifest places Company on `feature`. + db.load_as( + "feature", + None, + r#"{"type":"Company","data":{"name":"Acme"}}"#, + LoadMode::Merge, + None, + ) + .await + .unwrap(); + assert_eq!( + classify_fork_ref(&db, "node:Company", "feature").await, + ForkRefStatus::Legitimate, + "a manifest-placed fork must classify as Legitimate (never destroyed)" + ); + + // Orphan (manifest-unreferenced): forge a `feature` ref on Person, which + // the manifest's `feature` snapshot still places on main. + let person = node_path(&db, "feature", "node:Person").await; + { + let mut ds = lance::Dataset::open(&person).await.unwrap(); + let v = ds.version().version; + ds.create_branch("feature", v, None).await.unwrap(); + } + assert_eq!( + classify_fork_ref(&db, "node:Person", "feature").await, + ForkRefStatus::Orphan, + "a ref the manifest does not place on the branch must classify as Orphan" + ); + + // Orphan (ghost): a ref for a branch the manifest does not have at all. + { + let mut ds = lance::Dataset::open(&person).await.unwrap(); + let v = ds.version().version; + ds.create_branch("ghost", v, None).await.unwrap(); + } + assert_eq!( + classify_fork_ref(&db, "node:Person", "ghost").await, + ForkRefStatus::Orphan, + "a ref for a branch absent from the manifest must classify as Orphan" + ); + } +} diff --git a/crates/omnigraph/src/db/recovery_audit.rs b/crates/omnigraph/src/db/recovery_audit.rs index 2aab6bc..05d84b8 100644 --- a/crates/omnigraph/src/db/recovery_audit.rs +++ b/crates/omnigraph/src/db/recovery_audit.rs @@ -189,6 +189,8 @@ async fn create_recoveries_dataset(root_uri: &str) -> Result { mode: WriteMode::Create, enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; match Dataset::write(reader, &uri as &str, Some(params)).await { diff --git a/crates/omnigraph/src/db/write_queue.rs b/crates/omnigraph/src/db/write_queue.rs index 1f0c53a..18a14d1 100644 --- a/crates/omnigraph/src/db/write_queue.rs +++ b/crates/omnigraph/src/db/write_queue.rs @@ -1,12 +1,15 @@ -//! Per-`(table_key, branch)` writer queues — MR-686 scaffolding. +//! Per-`(table_key, branch)` writer queues. //! -//! Today every server-layer write serializes on the global -//! `Arc>` in `AppState`. MR-686 replaces that with -//! per-`(table_key, branch_ref)` queues so disjoint-key writes proceed -//! concurrently. This module owns the queue data structure; callers in -//! `MutationStaging::commit_all`, `branch_merge`, `schema_apply`, -//! `ensure_indices`, `delete_where`, and the future MR-870 recovery -//! reconciler acquire guards before any per-table Lance commit. +//! These queues are the engine's write-serialization mechanism: the server +//! holds the engine as a lockless `Arc` (writes are `&self`), so +//! disjoint-key writes proceed concurrently and only writes to the same +//! `(table_key, branch_ref)` serialize here. This module owns the queue +//! data structure; callers in `MutationStaging::commit_all`, `branch_merge`, +//! `schema_apply`, `ensure_indices`, `delete_where`, the fork path (first +//! write to a table on a branch — acquired before the fork, held through the +//! manifest publish), and the recovery reconciler acquire guards before any +//! per-table Lance commit. Serialization is in-process only; cross-process +//! writers on one graph remain one-winner-CAS at the manifest publish. //! //! ## Why exclusive `tokio::sync::Mutex<()>` per key //! diff --git a/crates/omnigraph/src/embedding.rs b/crates/omnigraph/src/embedding.rs index cfd4071..246836c 100644 --- a/crates/omnigraph/src/embedding.rs +++ b/crates/omnigraph/src/embedding.rs @@ -8,29 +8,157 @@ use tokio::time::sleep; use crate::error::{OmniError, Result}; -const GEMINI_EMBED_MODEL: &str = "gemini-embedding-2-preview"; +const DEFAULT_OPENROUTER_BASE_URL: &str = "https://openrouter.ai/api/v1"; +const DEFAULT_OPENROUTER_MODEL: &str = "openai/text-embedding-3-large"; +const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; +const DEFAULT_OPENAI_MODEL: &str = "text-embedding-3-large"; const DEFAULT_GEMINI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta"; +const DEFAULT_GEMINI_MODEL: &str = "gemini-embedding-2"; const DEFAULT_TIMEOUT_MS: u64 = 30_000; const DEFAULT_RETRY_ATTEMPTS: usize = 4; const DEFAULT_RETRY_BACKOFF_MS: u64 = 200; -const QUERY_TASK_TYPE: &str = "RETRIEVAL_QUERY"; -const DOCUMENT_TASK_TYPE: &str = "RETRIEVAL_DOCUMENT"; +const DEFAULT_DEADLINE_MS: u64 = 60_000; +const GEMINI_QUERY_TASK_TYPE: &str = "RETRIEVAL_QUERY"; +const GEMINI_DOCUMENT_TASK_TYPE: &str = "RETRIEVAL_DOCUMENT"; -#[derive(Clone, Debug)] -enum EmbeddingTransport { +/// Which embedding API a client speaks. Each variant owns its request shape, +/// auth, and response parsing; everything else (retry, deadline, normalization, +/// tracing) is provider-independent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Provider { + /// OpenAI-compatible (`POST {base}/embeddings`, bearer auth, + /// `{model, input, dimensions}`). Covers OpenRouter (the default gateway), + /// OpenAI direct, and self-hosted endpoints (vLLM/Ollama/LM Studio). + OpenAiCompatible, + /// Google Gemini `generativelanguage` (`POST {base}/models/{model}:embedContent`, + /// `x-goog-api-key`), with `RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT` task types. + Gemini, + /// Deterministic, offline. No network, no key. Mock, - Gemini { +} + +/// Whether the text being embedded is a search query or a stored document. +/// Only Gemini distinguishes these (`RETRIEVAL_QUERY` vs `RETRIEVAL_DOCUMENT`); +/// OpenAI-compatible providers and Mock produce the identical request for both, +/// which is also the same-space property a query relies on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum EmbedRole { + Query, + Document, +} + +/// The single source of truth for how embedding text becomes a vector: +/// provider + model + endpoint + key. Resolved once (from env for direct +/// engine/CLI callers, or from an applied cluster `providers.embedding` profile +/// at server boot) and shared by the query path and the offline CLI so stored +/// and query vectors stay same-space by construction. +#[derive(Clone, Debug)] +pub struct EmbeddingConfig { + pub provider: Provider, + pub model: String, + pub base_url: String, + pub api_key: String, +} + +impl EmbeddingConfig { + /// Resolve from the environment. Precedence: + /// 1. `OMNIGRAPH_EMBEDDINGS_MOCK` → Mock. + /// 2. `OMNIGRAPH_EMBED_PROVIDER` (`openai-compatible`|`openai`|`gemini`|`mock`); + /// unset defaults to `openai-compatible` (OpenRouter). + /// 3. `OMNIGRAPH_EMBED_BASE_URL` else the provider default. + /// 4. `OMNIGRAPH_EMBED_MODEL` else the provider default. + /// 5. provider api-key env (`OPENROUTER_API_KEY`/`OPENAI_API_KEY`, or `GEMINI_API_KEY`). + pub fn from_env() -> Result { + if env_flag("OMNIGRAPH_EMBEDDINGS_MOCK") { + return Ok(Self::mock()); + } + + let alias = env_string("OMNIGRAPH_EMBED_PROVIDER"); + if alias.as_deref() == Some("mock") { + return Ok(Self::mock()); + } + + let (provider, default_base, default_model, key_envs) = provider_profile(alias.as_deref())?; + let base_url = env_string("OMNIGRAPH_EMBED_BASE_URL") + .unwrap_or_else(|| default_base.to_string()) + .trim_end_matches('/') + .to_string(); + let model = + env_string("OMNIGRAPH_EMBED_MODEL").unwrap_or_else(|| default_model.to_string()); + + let api_key = key_envs.iter().copied().find_map(env_string).ok_or_else(|| { + OmniError::manifest_internal(format!( + "{} is required for the {} embedding provider", + key_envs.join(" or "), + alias.as_deref().unwrap_or("openai-compatible") + )) + })?; + + Ok(Self { + provider, + model, + base_url, + api_key, + }) + } + + /// Build a config from explicit parts — the cluster `providers.embedding` profile path + /// (RFC-012 Phase 5). `provider`/`base_url`/`model` default exactly as + /// `from_env` does (shared `provider_profile`); `api_key` is already resolved + /// (the cluster path resolves a `${NAME}` ref before calling this). + pub fn from_parts( + provider: Option<&str>, + base_url: Option, + model: Option, api_key: String, - base_url: String, - http: Client, - }, + ) -> Result { + if provider == Some("mock") { + // An explicit `model` (e.g. a cluster `providers.embedding` profile) is + // authoritative — it is what the same-space check compares against — + // so honor it; fall back to `mock()`'s env-based model only when the + // caller supplied none. Without this, a profile's `model` is silently + // dropped and the same-space check resolves to OMNIGRAPH_EMBED_MODEL. + let mut config = Self::mock(); + if let Some(model) = model { + config.model = model; + } + return Ok(config); + } + let (provider, default_base, default_model, _key_envs) = provider_profile(provider)?; + let base_url = base_url + .unwrap_or_else(|| default_base.to_string()) + .trim_end_matches('/') + .to_string(); + let model = model.unwrap_or_else(|| default_model.to_string()); + Ok(Self { + provider, + model, + base_url, + api_key, + }) + } + + fn mock() -> Self { + Self { + provider: Provider::Mock, + // Honor OMNIGRAPH_EMBED_MODEL so the same-space check is exercisable + // under mock; the mock vectors themselves don't depend on the model. + model: env_string("OMNIGRAPH_EMBED_MODEL").unwrap_or_default(), + base_url: String::new(), + api_key: String::new(), + } + } } #[derive(Clone, Debug)] pub struct EmbeddingClient { + config: EmbeddingConfig, + http: Client, retry_attempts: usize, retry_backoff_ms: u64, - transport: EmbeddingTransport, + /// Total wall-clock budget for one embed call, across all retries + /// (`OMNIGRAPH_EMBED_DEADLINE_MS`). `0` = unbounded. + deadline_ms: u64, } struct EmbedCallError { @@ -58,35 +186,39 @@ struct GoogleErrorBody { message: String, } +#[derive(Debug, Deserialize)] +struct OpenAiEmbeddingResponse { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct OpenAiEmbeddingDatum { + index: usize, + embedding: Vec, +} + +#[derive(Debug, Deserialize)] +struct OpenAiErrorEnvelope { + error: OpenAiErrorBody, +} + +#[derive(Debug, Deserialize)] +struct OpenAiErrorBody { + message: String, +} + impl EmbeddingClient { pub fn from_env() -> Result { + Self::new(EmbeddingConfig::from_env()?) + } + + pub fn new(config: EmbeddingConfig) -> Result { let retry_attempts = parse_env_usize("OMNIGRAPH_EMBED_RETRY_ATTEMPTS", DEFAULT_RETRY_ATTEMPTS); let retry_backoff_ms = parse_env_u64("OMNIGRAPH_EMBED_RETRY_BACKOFF_MS", DEFAULT_RETRY_BACKOFF_MS); - - if env_flag("OMNIGRAPH_EMBEDDINGS_MOCK") { - return Ok(Self { - retry_attempts, - retry_backoff_ms, - transport: EmbeddingTransport::Mock, - }); - } - - let api_key = std::env::var("GEMINI_API_KEY") - .ok() - .map(|v| v.trim().to_string()) - .filter(|v| !v.is_empty()) - .ok_or_else(|| { - OmniError::manifest_internal( - "GEMINI_API_KEY is required when nearest() needs a string embedding", - ) - })?; - let base_url = std::env::var("OMNIGRAPH_GEMINI_BASE_URL") - .ok() - .map(|v| v.trim_end_matches('/').to_string()) - .filter(|v| !v.is_empty()) - .unwrap_or_else(|| DEFAULT_GEMINI_BASE_URL.to_string()); + let deadline_ms = + parse_env_u64_allow_zero("OMNIGRAPH_EMBED_DEADLINE_MS", DEFAULT_DEADLINE_MS); let timeout_ms = parse_env_u64("OMNIGRAPH_EMBED_TIMEOUT_MS", DEFAULT_TIMEOUT_MS); let http = Client::builder() .timeout(Duration::from_millis(timeout_ms)) @@ -96,39 +228,36 @@ impl EmbeddingClient { })?; Ok(Self { + config, + http, retry_attempts, retry_backoff_ms, - transport: EmbeddingTransport::Gemini { - api_key, - base_url, - http, - }, + deadline_ms, }) } + pub fn config(&self) -> &EmbeddingConfig { + &self.config + } + #[cfg(test)] fn mock_for_tests() -> Self { - Self { - retry_attempts: DEFAULT_RETRY_ATTEMPTS, - retry_backoff_ms: DEFAULT_RETRY_BACKOFF_MS, - transport: EmbeddingTransport::Mock, - } + Self::new(EmbeddingConfig::mock()).expect("mock client builds") } pub async fn embed_query_text(&self, input: &str, expected_dim: usize) -> Result> { - self.embed_text(input, expected_dim, QUERY_TASK_TYPE).await + self.embed_text(input, expected_dim, EmbedRole::Query).await } pub async fn embed_document_text(&self, input: &str, expected_dim: usize) -> Result> { - self.embed_text(input, expected_dim, DOCUMENT_TASK_TYPE) - .await + self.embed_text(input, expected_dim, EmbedRole::Document).await } async fn embed_text( &self, input: &str, expected_dim: usize, - task_type: &'static str, + role: EmbedRole, ) -> Result> { if expected_dim == 0 { return Err(OmniError::manifest_internal( @@ -136,10 +265,71 @@ impl EmbeddingClient { )); } - match &self.transport { - EmbeddingTransport::Mock => Ok(mock_embedding(input, expected_dim)), - EmbeddingTransport::Gemini { .. } => { - self.with_retry(|| self.embed_text_gemini_once(input, expected_dim, task_type)) + let started = std::time::Instant::now(); + let result = self + .run_with_deadline(self.embed_text_inner(input, expected_dim, role)) + .await; + let elapsed_ms = started.elapsed().as_millis() as u64; + + match &result { + Ok(_) => tracing::info!( + target: "omnigraph::embedding", + provider = ?self.config.provider, + model = %self.config.model, + dim = expected_dim, + elapsed_ms, + outcome = "ok", + "embedding succeeded" + ), + Err(err) => tracing::warn!( + target: "omnigraph::embedding", + provider = ?self.config.provider, + model = %self.config.model, + dim = expected_dim, + elapsed_ms, + outcome = "error", + error = %err, + "embedding failed" + ), + } + result + } + + /// Bound the whole embed operation (all retries + backoff) by `deadline_ms`, + /// so a degraded provider can never hang the caller for the full retry + /// envelope. Applies to every embed call (query and document). `0` = + /// unbounded. Embedding has no Lance/manifest side effects, so cancelling the + /// in-flight request future on elapse is safe. + async fn run_with_deadline(&self, fut: F) -> Result> + where + F: Future>>, + { + if self.deadline_ms == 0 { + return fut.await; + } + match tokio::time::timeout(Duration::from_millis(self.deadline_ms), fut).await { + Ok(res) => res, + Err(_elapsed) => Err(OmniError::manifest_internal(format!( + "embedding deadline exceeded after {} ms (provider={:?}, model={})", + self.deadline_ms, self.config.provider, self.config.model + ))), + } + } + + async fn embed_text_inner( + &self, + input: &str, + expected_dim: usize, + role: EmbedRole, + ) -> Result> { + match self.config.provider { + Provider::Mock => Ok(mock_embedding(input, expected_dim)), + Provider::Gemini => { + self.with_retry(|| self.embed_gemini_once(input, expected_dim, role)) + .await + } + Provider::OpenAiCompatible => { + self.with_retry(|| self.embed_openai_once(input, expected_dim)) .await } } @@ -160,6 +350,14 @@ impl EmbeddingClient { if !err.retryable || attempt >= max_attempt { return Err(OmniError::manifest_internal(err.message)); } + tracing::warn!( + target: "omnigraph::embedding", + provider = ?self.config.provider, + model = %self.config.model, + attempt, + error = %err.message, + "embedding attempt failed, retrying" + ); let shift = (attempt - 1).min(10) as u32; let delay = self.retry_backoff_ms.saturating_mul(1u64 << shift); sleep(Duration::from_millis(delay)).await; @@ -168,25 +366,27 @@ impl EmbeddingClient { } } - async fn embed_text_gemini_once( + async fn embed_gemini_once( &self, input: &str, expected_dim: usize, - task_type: &'static str, + role: EmbedRole, ) -> std::result::Result, EmbedCallError> { - let (api_key, base_url, http) = match &self.transport { - EmbeddingTransport::Gemini { - api_key, - base_url, - http, - } => (api_key, base_url, http), - EmbeddingTransport::Mock => unreachable!("mock transport should not call Gemini"), + let task_type = match role { + EmbedRole::Query => GEMINI_QUERY_TASK_TYPE, + EmbedRole::Document => GEMINI_DOCUMENT_TASK_TYPE, }; - let response = http - .post(gemini_endpoint(base_url)) - .header("x-goog-api-key", api_key) - .json(&build_gemini_request(input, expected_dim, task_type)) + let response = self + .http + .post(gemini_endpoint(&self.config.base_url, &self.config.model)) + .header("x-goog-api-key", &self.config.api_key) + .json(&build_gemini_request( + &self.config.model, + input, + expected_dim, + task_type, + )) .send() .await; let response = match response { @@ -205,10 +405,7 @@ impl EmbeddingClient { Ok(body) => body, Err(err) => { return Err(EmbedCallError { - message: format!( - "embedding response read failed (status {}): {}", - status, err - ), + message: format!("embedding response read failed (status {}): {}", status, err), retryable: status.is_server_error() || status.as_u16() == 429, }); } @@ -217,10 +414,7 @@ impl EmbeddingClient { if !status.is_success() { let message = parse_google_error_message(&body).unwrap_or(body); return Err(EmbedCallError { - message: format!( - "embedding request failed with status {}: {}", - status, message - ), + message: format!("embedding request failed with status {}: {}", status, message), retryable: status.is_server_error() || status.as_u16() == 429, }); } @@ -238,19 +432,85 @@ impl EmbeddingClient { } }) } + + async fn embed_openai_once( + &self, + input: &str, + expected_dim: usize, + ) -> std::result::Result, EmbedCallError> { + let response = self + .http + .post(format!("{}/embeddings", self.config.base_url)) + .bearer_auth(&self.config.api_key) + .json(&build_openai_request(&self.config.model, input, expected_dim)) + .send() + .await; + let response = match response { + Ok(response) => response, + Err(err) => { + let retryable = err.is_timeout() || err.is_connect() || err.is_request(); + return Err(EmbedCallError { + message: format!("embedding request failed: {}", err), + retryable, + }); + } + }; + + let status = response.status(); + let body = match response.text().await { + Ok(body) => body, + Err(err) => { + return Err(EmbedCallError { + message: format!("embedding response read failed (status {}): {}", status, err), + retryable: status.is_server_error() || status.as_u16() == 429, + }); + } + }; + + if !status.is_success() { + let message = parse_openai_error_message(&body).unwrap_or(body); + return Err(EmbedCallError { + message: format!("embedding request failed with status {}: {}", status, message), + retryable: status.is_server_error() || status.as_u16() == 429, + }); + } + + let parsed: OpenAiEmbeddingResponse = + serde_json::from_str(&body).map_err(|err| EmbedCallError { + message: format!("embedding response decode failed: {}", err), + retryable: false, + })?; + + // The query path embeds exactly one string, so expect one datum at index 0. + let datum = parsed + .data + .into_iter() + .find(|d| d.index == 0) + .ok_or_else(|| EmbedCallError { + message: "embedding response missing data[0]".to_string(), + retryable: false, + })?; + + validate_and_normalize_embedding(datum.embedding, expected_dim).map_err(|message| { + EmbedCallError { + message, + retryable: false, + } + }) + } } -fn gemini_endpoint(base_url: &str) -> String { +fn gemini_endpoint(base_url: &str, model: &str) -> String { format!( "{}/models/{}:embedContent", base_url.trim_end_matches('/'), - GEMINI_EMBED_MODEL + model ) } -fn build_gemini_request(input: &str, expected_dim: usize, task_type: &'static str) -> Value { +fn build_gemini_request(model: &str, input: &str, expected_dim: usize, task_type: &str) -> Value { json!({ - "model": format!("models/{}", GEMINI_EMBED_MODEL), + "model": format!("models/{}", model), "content": { "parts": [ { @@ -263,6 +523,14 @@ fn build_gemini_request(input: &str, expected_dim: usize, task_type: &'static st }) } +fn build_openai_request(model: &str, input: &str, expected_dim: usize) -> Value { + json!({ + "model": model, + "input": [input], + "dimensions": expected_dim, + }) +} + fn validate_and_normalize_embedding( values: Vec, expected_dim: usize, @@ -298,6 +566,57 @@ fn parse_google_error_message(body: &str) -> Option { .filter(|msg| !msg.trim().is_empty()) } +fn parse_openai_error_message(body: &str) -> Option { + serde_json::from_str::(body) + .ok() + .map(|e| e.error.message) + .filter(|msg| !msg.trim().is_empty()) +} + +/// Map a provider alias to `(provider, default base URL, default model, ordered +/// api-key envs)`. Shared by `from_env` and `from_parts` so both apply identical +/// defaults: `openai-compatible`/unset → the OpenRouter gateway, `openai` → +/// OpenAI's own host. `mock` is handled by callers before this is reached. The +/// `Provider` enum alone would collapse the two openai aliases, so the alias +/// (not the enum) determines the key-env order here. +fn provider_profile( + alias: Option<&str>, +) -> Result<(Provider, &'static str, &'static str, &'static [&'static str])> { + Ok(match alias { + None | Some("openai-compatible") => ( + Provider::OpenAiCompatible, + DEFAULT_OPENROUTER_BASE_URL, + DEFAULT_OPENROUTER_MODEL, + &["OPENROUTER_API_KEY", "OPENAI_API_KEY"], + ), + Some("openai") => ( + Provider::OpenAiCompatible, + DEFAULT_OPENAI_BASE_URL, + DEFAULT_OPENAI_MODEL, + &["OPENAI_API_KEY"], + ), + Some("gemini") => ( + Provider::Gemini, + DEFAULT_GEMINI_BASE_URL, + DEFAULT_GEMINI_MODEL, + &["GEMINI_API_KEY"], + ), + Some(other) => { + return Err(OmniError::manifest_internal(format!( + "unknown embedding provider '{}' (expected openai-compatible|openai|gemini|mock)", + other + ))); + } + }) +} + +fn env_string(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) +} + fn parse_env_usize(name: &str, default: usize) -> usize { std::env::var(name) .ok() @@ -314,6 +633,15 @@ fn parse_env_u64(name: &str, default: u64) -> u64 { .unwrap_or(default) } +/// Like [`parse_env_u64`] but accepts `0` as a meaningful value (the deadline +/// uses `0` for "unbounded"). +fn parse_env_u64_allow_zero(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(default) +} + fn env_flag(name: &str) -> bool { std::env::var(name) .ok() @@ -395,6 +723,25 @@ mod tests { } } + // Every test that calls `EmbeddingConfig::from_env` clears the full set of + // embedding env vars first so the host environment can't leak in. + const EMBED_ENV: &[&str] = &[ + "OMNIGRAPH_EMBEDDINGS_MOCK", + "OMNIGRAPH_EMBED_PROVIDER", + "OMNIGRAPH_EMBED_BASE_URL", + "OMNIGRAPH_EMBED_MODEL", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + "GEMINI_API_KEY", + ]; + + fn cleared_env(extra: &[(&'static str, Option<&str>)]) -> EnvGuard { + let mut vars: Vec<(&'static str, Option<&str>)> = + EMBED_ENV.iter().map(|n| (*n, None)).collect(); + vars.extend_from_slice(extra); + EnvGuard::set(&vars) + } + #[tokio::test] async fn mock_embeddings_are_deterministic() { let client = EmbeddingClient::mock_for_tests(); @@ -407,18 +754,30 @@ mod tests { } #[test] - fn gemini_request_uses_preview_model_retrieval_query_and_dimension() { - let request = build_gemini_request("alpha", 4, QUERY_TASK_TYPE); - assert_eq!(request["model"], "models/gemini-embedding-2-preview"); - assert_eq!(request["taskType"], QUERY_TASK_TYPE); + fn gemini_request_uses_model_retrieval_query_and_dimension() { + let request = + build_gemini_request("gemini-embedding-2", "alpha", 4, GEMINI_QUERY_TASK_TYPE); + assert_eq!(request["model"], "models/gemini-embedding-2"); + assert_eq!(request["taskType"], GEMINI_QUERY_TASK_TYPE); assert_eq!(request["outputDimensionality"], 4); assert_eq!(request["content"]["parts"][0]["text"], "alpha"); } #[test] fn gemini_document_request_uses_retrieval_document_task_type() { - let request = build_gemini_request("alpha", 4, DOCUMENT_TASK_TYPE); - assert_eq!(request["taskType"], DOCUMENT_TASK_TYPE); + let request = + build_gemini_request("gemini-embedding-2", "alpha", 4, GEMINI_DOCUMENT_TASK_TYPE); + assert_eq!(request["taskType"], GEMINI_DOCUMENT_TASK_TYPE); + } + + #[test] + fn openai_request_uses_model_input_array_and_dimensions() { + let request = build_openai_request("openai/text-embedding-3-large", "alpha", 4); + assert_eq!(request["model"], "openai/text-embedding-3-large"); + assert_eq!(request["input"][0], "alpha"); + assert!(request["input"].is_array()); + assert_eq!(request["dimensions"], 4); + assert!(request.get("taskType").is_none()); } #[test] @@ -475,15 +834,202 @@ mod tests { assert!(err.to_string().contains("do not retry")); } + #[tokio::test] + async fn run_with_deadline_aborts_slow_future() { + let mut client = EmbeddingClient::mock_for_tests(); + client.deadline_ms = 20; + let slow = async { + tokio::time::sleep(Duration::from_secs(5)).await; + Ok(vec![0.0_f32]) + }; + let err = client.run_with_deadline(slow).await.unwrap_err(); + assert!(err.to_string().contains("deadline exceeded")); + } + + #[tokio::test] + async fn run_with_deadline_passes_through_fast_future() { + let client = EmbeddingClient::mock_for_tests(); + let ok = client + .run_with_deadline(async { Ok(vec![1.0_f32, 2.0]) }) + .await + .unwrap(); + assert_eq!(ok, vec![1.0, 2.0]); + } + + #[tokio::test] + async fn run_with_deadline_zero_is_unbounded() { + let mut client = EmbeddingClient::mock_for_tests(); + client.deadline_ms = 0; + let ok = client + .run_with_deadline(async { Ok(vec![3.0_f32]) }) + .await + .unwrap(); + assert_eq!(ok, vec![3.0]); + } + #[test] #[serial] - fn from_env_requires_gemini_api_key_when_not_mocking() { - let _guard = EnvGuard::set(&[ - ("OMNIGRAPH_EMBEDDINGS_MOCK", None), - ("GEMINI_API_KEY", None), - ]); + fn from_env_defaults_to_openai_compatible_openrouter() { + let _guard = cleared_env(&[("OPENROUTER_API_KEY", Some("sk-test"))]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.provider, Provider::OpenAiCompatible); + assert_eq!(config.base_url, DEFAULT_OPENROUTER_BASE_URL); + assert_eq!(config.model, DEFAULT_OPENROUTER_MODEL); + assert_eq!(config.api_key, "sk-test"); + } - let err = EmbeddingClient::from_env().unwrap_err(); - assert!(err.to_string().contains("GEMINI_API_KEY")); + #[test] + #[serial] + fn from_env_openai_alias_uses_openai_host_not_openrouter() { + let _guard = cleared_env(&[ + ("OMNIGRAPH_EMBED_PROVIDER", Some("openai")), + ("OPENAI_API_KEY", Some("k")), + ]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.provider, Provider::OpenAiCompatible); + assert_eq!(config.base_url, DEFAULT_OPENAI_BASE_URL); // api.openai.com, not OpenRouter + assert_eq!(config.model, DEFAULT_OPENAI_MODEL); // text-embedding-3-large, no openai/ prefix + assert_eq!(config.api_key, "k"); + } + + #[test] + #[serial] + fn from_env_openai_alias_prefers_openai_key_over_openrouter() { + // `openai` targets api.openai.com, so an OpenRouter key must not be sent there. + let _guard = cleared_env(&[ + ("OMNIGRAPH_EMBED_PROVIDER", Some("openai")), + ("OPENROUTER_API_KEY", Some("router")), + ("OPENAI_API_KEY", Some("openai")), + ]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.base_url, DEFAULT_OPENAI_BASE_URL); + assert_eq!(config.api_key, "openai"); + } + + #[test] + #[serial] + fn from_env_openai_alias_errors_when_only_openrouter_key_is_set() { + let _guard = cleared_env(&[ + ("OMNIGRAPH_EMBED_PROVIDER", Some("openai")), + ("OPENROUTER_API_KEY", Some("router")), + ]); + let err = EmbeddingConfig::from_env().unwrap_err(); + assert!(err.to_string().contains("OPENAI_API_KEY"), "got: {err}"); + } + + #[test] + fn from_parts_applies_provider_defaults_and_overrides() { + let openrouter = EmbeddingConfig::from_parts(None, None, None, "k".to_string()).unwrap(); + assert_eq!(openrouter.provider, Provider::OpenAiCompatible); + assert_eq!(openrouter.base_url, DEFAULT_OPENROUTER_BASE_URL); + assert_eq!(openrouter.model, DEFAULT_OPENROUTER_MODEL); + assert_eq!(openrouter.api_key, "k"); + + let gemini = + EmbeddingConfig::from_parts(Some("gemini"), None, None, "g".to_string()).unwrap(); + assert_eq!(gemini.provider, Provider::Gemini); + assert_eq!(gemini.base_url, DEFAULT_GEMINI_BASE_URL); + + let overridden = EmbeddingConfig::from_parts( + Some("openai"), + Some("https://x/v1/".to_string()), + Some("custom".to_string()), + "k".to_string(), + ) + .unwrap(); + assert_eq!(overridden.base_url, "https://x/v1"); // trailing slash trimmed + assert_eq!(overridden.model, "custom"); + + let err = + EmbeddingConfig::from_parts(Some("cohere"), None, None, "k".to_string()).unwrap_err(); + assert!( + err.to_string().contains("unknown embedding provider"), + "got: {err}" + ); + } + + #[test] + #[serial] + fn from_parts_mock_honors_an_explicit_model() { + // A cluster `providers.embedding` profile that sets `kind: mock, model: X` + // must resolve to model X — it is what the query-time same-space check + // compares against. Env cleared so the assertion isolates the arg. + let _guard = cleared_env(&[]); + let pinned = + EmbeddingConfig::from_parts(Some("mock"), None, Some("recorded-x".to_string()), String::new()) + .unwrap(); + assert_eq!(pinned.provider, Provider::Mock); + assert_eq!(pinned.model, "recorded-x"); + // With no explicit model, mock falls back to its env-based default (here + // empty, since the env is cleared). + let bare = EmbeddingConfig::from_parts(Some("mock"), None, None, String::new()).unwrap(); + assert_eq!(bare.provider, Provider::Mock); + assert_eq!(bare.model, ""); + } + + #[test] + #[serial] + fn from_env_openai_compatible_prefers_openrouter_key() { + let _guard = cleared_env(&[ + ("OPENROUTER_API_KEY", Some("router")), + ("OPENAI_API_KEY", Some("openai")), + ]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.api_key, "router"); + } + + #[test] + #[serial] + fn from_env_explicit_gemini_provider() { + let _guard = cleared_env(&[ + ("OMNIGRAPH_EMBED_PROVIDER", Some("gemini")), + ("GEMINI_API_KEY", Some("g-key")), + ]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.provider, Provider::Gemini); + assert_eq!(config.base_url, DEFAULT_GEMINI_BASE_URL); + assert_eq!(config.model, DEFAULT_GEMINI_MODEL); + assert_eq!(config.api_key, "g-key"); + } + + #[test] + #[serial] + fn from_env_base_url_and_model_overrides_apply() { + let _guard = cleared_env(&[ + ("OMNIGRAPH_EMBED_PROVIDER", Some("openai-compatible")), + ("OMNIGRAPH_EMBED_BASE_URL", Some("https://example.test/v1/")), + ("OMNIGRAPH_EMBED_MODEL", Some("custom/model")), + ("OPENAI_API_KEY", Some("k")), + ]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.base_url, "https://example.test/v1"); // trailing slash trimmed + assert_eq!(config.model, "custom/model"); + } + + #[test] + #[serial] + fn from_env_unknown_provider_errors() { + let _guard = cleared_env(&[("OMNIGRAPH_EMBED_PROVIDER", Some("cohere"))]); + let err = EmbeddingConfig::from_env().unwrap_err(); + assert!(err.to_string().contains("unknown embedding provider")); + } + + #[test] + #[serial] + fn from_env_errors_when_no_key_present() { + let _guard = cleared_env(&[]); + let err = EmbeddingConfig::from_env().unwrap_err(); + assert!(err.to_string().contains("OPENROUTER_API_KEY or OPENAI_API_KEY")); + } + + #[test] + #[serial] + fn from_env_mock_flag_wins() { + let _guard = cleared_env(&[ + ("OMNIGRAPH_EMBEDDINGS_MOCK", Some("1")), + ("OMNIGRAPH_EMBED_PROVIDER", Some("gemini")), + ]); + let config = EmbeddingConfig::from_env().unwrap(); + assert_eq!(config.provider, Provider::Mock); } } diff --git a/crates/omnigraph/src/exec/merge.rs b/crates/omnigraph/src/exec/merge.rs index ea16b15..5d0be74 100644 --- a/crates/omnigraph/src/exec/merge.rs +++ b/crates/omnigraph/src/exec/merge.rs @@ -1323,9 +1323,9 @@ impl Omnigraph { // branch_merge writes only to the target branch. // // Held across the per-table publish loop and the manifest - // commit + record_merge_commit calls below. Under PR 1b's - // intermediate state (global server RwLock still in place), - // this acquisition is uncontended. + // commit + record_merge_commit calls below, so no concurrent + // writer to a touched (table, target_branch) can interleave + // between our commit_staged and our publish. let active_branch_for_keys = self.active_branch().await; let merge_queue_keys: Vec<(String, Option)> = ordered_table_keys .iter() diff --git a/crates/omnigraph/src/exec/mutation.rs b/crates/omnigraph/src/exec/mutation.rs index e9051c4..9fcff45 100644 --- a/crates/omnigraph/src/exec/mutation.rs +++ b/crates/omnigraph/src/exec/mutation.rs @@ -741,14 +741,45 @@ impl Omnigraph { // tables. Branch is threaded explicitly — no coordinator swap. let mut staging = MutationStaging::default(); + // Lower + validate up front so the touched-table set is known before + // execution. A lowering/validation error returns exactly as it did + // when this happened inside execute_named_mutation. + let ir = self.lower_named_mutation(query_source, query_name)?; + + // Up-front fork-queue acquisition (see the loader for the full + // rationale): if this mutation will fork any touched table onto a + // non-main branch, acquire the per-(table, branch) write queues for + // every touched table before the first fork and hold them through the + // publish, so the orphan-fork reclaim can't race a concurrent + // in-process fork. The touched set is derived from the lowered IR. + let fork_queue_guards: Option<( + Vec<(String, Option)>, + Vec>, + )> = if let Some(active) = requested.as_deref() { + let snapshot = self.snapshot_for_branch(Some(active)).await?; + let touched: Vec<(String, Option)> = self + .touched_table_keys(&ir) + .into_iter() + .map(|k| (k, Some(active.to_string()))) + .collect(); + let needs_fork = touched.iter().any(|(table_key, _)| { + snapshot + .entry(table_key) + .map(|e| e.table_branch.as_deref() != Some(active)) + .unwrap_or(false) + }); + if needs_fork { + let guards = self.write_queue().acquire_many(&touched).await; + Some((touched, guards)) + } else { + None + } + } else { + None + }; + let exec_result = self - .execute_named_mutation( - query_source, - query_name, - &resolved_params, - requested.as_deref(), - &mut staging, - ) + .execute_named_mutation(&ir, &resolved_params, requested.as_deref(), &mut staging) .await; match exec_result { @@ -768,6 +799,7 @@ impl Omnigraph { requested.as_deref(), crate::db::manifest::SidecarKind::Mutation, actor_id, + fork_queue_guards, ) .await?; // Failpoint that wedges the documented finalize→publisher @@ -817,14 +849,19 @@ impl Omnigraph { } } - async fn execute_named_mutation( + /// Lower + validate a named mutation query into its IR. + /// + /// Hoisted out of [`Self::execute_named_mutation`] so the caller can + /// inspect the IR before execution — specifically to compute the + /// touched-table set (see [`Self::touched_table_keys`]) for up-front + /// write-queue acquisition. Performs the same find → typecheck → lower + /// → D₂ checks that execution previously did inline, so error behavior + /// is unchanged. + fn lower_named_mutation( &self, query_source: &str, query_name: &str, - params: &ParamMap, - branch: Option<&str>, - staging: &mut MutationStaging, - ) -> Result { + ) -> Result { let query_decl = omnigraph_compiler::find_named_query(query_source, query_name) .map_err(|e| OmniError::manifest(e.to_string()))?; @@ -841,7 +878,61 @@ impl Omnigraph { let ir = lower_mutation_query(&query_decl)?; // D₂: reject mixed insert/update + delete before any I/O. enforce_no_mixed_destructive_constructive(&ir)?; + Ok(ir) + } + /// The COMPLETE set of `(node|edge):{type}` table keys a mutation IR can + /// touch at execution time, keyed as `MutationStaging`/`commit_all` key + /// them. Must be a superset of everything execution forks/commits, since + /// it drives the up-front fork-queue acquisition and `commit_all`'s + /// held-guard coverage check — a miss means an unserialized fork/commit. + /// + /// The set is a pure function of (IR ops + catalog). For each op it mirrors + /// the execute path's node-vs-edge dispatch (`node_types` first, then + /// `edge_types`). A `delete ` additionally **cascades** to every edge + /// type whose endpoint is that node (see `execute_delete_node`), forking + /// those edge tables during execution — so they are included here, derived + /// the same way the executor derives them (`from_type`/`to_type` match). + /// Unknown types are skipped (the execute path surfaces the error). + /// Sorted + deduped for one-shot `acquire_many`. + fn touched_table_keys(&self, ir: &omnigraph_compiler::ir::MutationIR) -> Vec { + use omnigraph_compiler::ir::MutationOpIR; + let catalog = self.catalog(); + let mut keys: Vec = Vec::new(); + for op in &ir.ops { + let type_name = match op { + MutationOpIR::Insert { type_name, .. } + | MutationOpIR::Update { type_name, .. } + | MutationOpIR::Delete { type_name, .. } => type_name, + }; + if catalog.node_types.contains_key(type_name) { + keys.push(format!("node:{type_name}")); + // A node delete cascades to every edge touching this node type, + // forking those edge tables. Include them so the up-front + // acquisition covers the cascade (mirrors execute_delete_node). + if matches!(op, MutationOpIR::Delete { .. }) { + for (edge_name, edge_type) in &catalog.edge_types { + if edge_type.from_type == *type_name || edge_type.to_type == *type_name { + keys.push(format!("edge:{edge_name}")); + } + } + } + } else if catalog.edge_types.contains_key(type_name) { + keys.push(format!("edge:{type_name}")); + } + } + keys.sort(); + keys.dedup(); + keys + } + + async fn execute_named_mutation( + &self, + ir: &omnigraph_compiler::ir::MutationIR, + params: &ParamMap, + branch: Option<&str>, + staging: &mut MutationStaging, + ) -> Result { let mut total = MutationResult::default(); for op in &ir.ops { let result = match op { diff --git a/crates/omnigraph/src/exec/projection.rs b/crates/omnigraph/src/exec/projection.rs index 7280ec5..bb6e665 100644 --- a/crates/omnigraph/src/exec/projection.rs +++ b/crates/omnigraph/src/exec/projection.rs @@ -72,7 +72,11 @@ fn evaluate_expr(batch: &RecordBatch, expr: &IRExpr, params: &ParamMap) -> Resul } /// Create a constant array from a literal value. -fn literal_to_array(lit: &Literal, num_rows: usize) -> Result { +/// +/// `pub(super)` so the pushdown arm (`query.rs::literal_to_typed_expr`) can build +/// a literal in the same natural Arrow type and cast it to the column type through +/// the identical `arrow_cast` path used here, keeping the two filter arms in sync. +pub(super) fn literal_to_array(lit: &Literal, num_rows: usize) -> Result { Ok(match lit { Literal::Null => arrow_array::new_null_array(&DataType::Utf8, num_rows), Literal::String(s) => Arc::new(StringArray::from(vec![s.as_str(); num_rows])) as ArrayRef, diff --git a/crates/omnigraph/src/exec/query.rs b/crates/omnigraph/src/exec/query.rs index 5bc18f2..b12e26b 100644 --- a/crates/omnigraph/src/exec/query.rs +++ b/crates/omnigraph/src/exec/query.rs @@ -2,6 +2,30 @@ use super::*; use super::projection::{apply_filter, apply_ordering, project_return}; +/// Bundles the per-handle embedding client cell with the optional injected +/// config (RFC-012 Phase 5) so the lazy init uses the injected config when +/// present, else `EmbeddingClient::from_env()`. Threaded through the query path +/// in place of the bare cell, preserving laziness (a graph that never embeds +/// builds no client and needs no key). +pub(crate) struct EmbeddingResolver<'a> { + cell: &'a tokio::sync::OnceCell, + config: Option<&'a crate::embedding::EmbeddingConfig>, +} + +impl EmbeddingResolver<'_> { + async fn resolve(&self) -> Result<&EmbeddingClient> { + let config = self.config.cloned(); + self.cell + .get_or_try_init(|| async move { + match config { + Some(cfg) => EmbeddingClient::new(cfg), + None => EmbeddingClient::from_env(), + } + }) + .await + } +} + impl Omnigraph { /// Run a named query against an explicit branch or snapshot target. pub async fn query( @@ -31,7 +55,18 @@ impl Omnigraph { GraphIndexHandle::none() }; - execute_query(&ir, params, &resolved.snapshot, &graph_index, &catalog).await + execute_query( + &ir, + params, + &resolved.snapshot, + &graph_index, + &catalog, + &EmbeddingResolver { + cell: self.embedding_cell(), + config: self.embedding_config_ref(), + }, + ) + .await } /// Run a named query against the graph as it existed at a prior manifest version. @@ -72,7 +107,18 @@ impl Omnigraph { GraphIndexHandle::none() }; - execute_query(&ir, params, &snapshot, &graph_index, &catalog).await + execute_query( + &ir, + params, + &snapshot, + &graph_index, + &catalog, + &EmbeddingResolver { + cell: self.embedding_cell(), + config: self.embedding_config_ref(), + }, + ) + .await } } @@ -102,6 +148,7 @@ async fn extract_search_mode( ir: &QueryIR, params: &ParamMap, catalog: &Catalog, + embedding: &EmbeddingResolver<'_>, ) -> Result { if ir.order_by.is_empty() { return Ok(SearchMode::default()); @@ -114,7 +161,8 @@ async fn extract_search_mode( query, } => { let vec = - resolve_nearest_query_vec(ir, catalog, variable, property, query, params).await?; + resolve_nearest_query_vec(ir, catalog, variable, property, query, params, embedding) + .await?; let k = ir.limit.ok_or_else(|| { OmniError::manifest("nearest() ordering requires a limit clause".to_string()) })? as usize; @@ -157,9 +205,10 @@ async fn extract_search_mode( .unwrap_or(60) as u32; let primary_mode = - extract_sub_search_mode(ir, primary, params, catalog, ir.limit).await?; + extract_sub_search_mode(ir, primary, params, catalog, ir.limit, embedding).await?; let secondary_mode = - extract_sub_search_mode(ir, secondary, params, catalog, ir.limit).await?; + extract_sub_search_mode(ir, secondary, params, catalog, ir.limit, embedding) + .await?; Ok(SearchMode { rrf: Some(RrfMode { @@ -182,6 +231,7 @@ async fn extract_sub_search_mode( params: &ParamMap, catalog: &Catalog, limit: Option, + embedding: &EmbeddingResolver<'_>, ) -> Result { match expr { IRExpr::Nearest { @@ -190,7 +240,8 @@ async fn extract_sub_search_mode( query, } => { let vec = - resolve_nearest_query_vec(ir, catalog, variable, property, query, params).await?; + resolve_nearest_query_vec(ir, catalog, variable, property, query, params, embedding) + .await?; let k = limit.unwrap_or(100) as usize; Ok(SearchMode { nearest: Some((variable.clone(), property.clone(), vec, k)), @@ -229,15 +280,34 @@ async fn resolve_nearest_query_vec( property: &str, expr: &IRExpr, params: &ParamMap, + embedding: &EmbeddingResolver<'_>, ) -> Result> { let lit = resolve_literal_or_param(expr, params)?; match lit { Literal::List(_) => literal_to_f32_vec(&lit), Literal::String(text) => { - let expected_dim = nearest_property_dimension(ir, catalog, variable, property)?; - EmbeddingClient::from_env()? - .embed_query_text(&text, expected_dim) - .await + let (expected_dim, recorded_model) = + nearest_property_dim_and_model(ir, catalog, variable, property)?; + // Lazily resolve the per-handle client once, then reuse it across + // queries (keeps the provider connection pool warm); a graph that + // never embeds never builds a client and needs no provider key. + let client = embedding.resolve().await?; + // Same-space guarantee: if the property recorded the model that + // produced its stored vectors (`@embed("…", model="…")`), the query + // embedder must resolve to that same model — otherwise the comparison + // is across vector spaces. Reject loudly instead of ranking garbage. + if let Some(recorded) = &recorded_model { + let resolved = &client.config().model; + if resolved != recorded { + return Err(OmniError::manifest(format!( + "nearest() on '{property}': its stored vectors were embedded with model \ + '{recorded}', but the query embedder resolves to '{resolved}'. Set \ + OMNIGRAPH_EMBED_MODEL='{recorded}' (and the matching provider) or re-embed \ + the stored vectors." + ))); + } + } + client.embed_query_text(&text, expected_dim).await } _ => Err(OmniError::manifest( "nearest query must be a string or list of floats".to_string(), @@ -279,12 +349,14 @@ fn literal_to_f32_vec(lit: &Literal) -> Result> { } } -fn nearest_property_dimension( +/// Resolve the nearest() target property's vector dimension and the embedding +/// model recorded for it via `@embed("…", model="…")` (`None` if unrecorded). +fn nearest_property_dim_and_model( ir: &QueryIR, catalog: &Catalog, variable: &str, property: &str, -) -> Result { +) -> Result<(usize, Option)> { let type_name = resolve_binding_type_name(&ir.pipeline, variable).ok_or_else(|| { OmniError::manifest_internal(format!( "nearest() variable '${}' is not bound to a node type in the lowered pipeline", @@ -303,13 +375,20 @@ fn nearest_property_dimension( type_name, property )) })?; - match prop.scalar { - ScalarType::Vector(dim) if !prop.list => Ok(dim as usize), - _ => Err(OmniError::manifest_internal(format!( - "nearest() property '{}.{}' is not a scalar vector", - type_name, property - ))), - } + let dim = match prop.scalar { + ScalarType::Vector(dim) if !prop.list => dim as usize, + _ => { + return Err(OmniError::manifest_internal(format!( + "nearest() property '{}.{}' is not a scalar vector", + type_name, property + ))); + } + }; + let recorded_model = node_type + .embed_sources + .get(property) + .and_then(|embed| embed.model.clone()); + Ok((dim, recorded_model)) } fn resolve_binding_type_name<'a>(pipeline: &'a [IROp], variable: &str) -> Option<&'a str> { @@ -341,8 +420,9 @@ pub async fn execute_query( snapshot: &Snapshot, graph_index: &GraphIndexHandle<'_>, catalog: &Catalog, + embedding: &EmbeddingResolver<'_>, ) -> Result { - let search_mode = extract_search_mode(ir, params, catalog).await?; + let search_mode = extract_search_mode(ir, params, catalog, embedding).await?; // RRF requires forked execution if let Some(ref rrf) = search_mode.rrf { @@ -763,7 +843,7 @@ fn traversal_indexed_override() -> Option { /// Max source-row frontier for which Expand uses the BTREE-indexed path. /// Larger frontiers fall back to the in-memory CSR (dense / whole-graph). See -/// `docs/user/constants.md`. +/// `docs/user/reference/constants.md`. const DEFAULT_EXPAND_INDEXED_MAX_FRONTIER: usize = 1024; /// Max hop count for the indexed path (each hop is one indexed scan; very deep /// traversals fan out toward whole-graph and are better served by CSR). @@ -1289,10 +1369,12 @@ async fn expand_hydrate_and_align( params: &ParamMap, ) -> Result<()> { // Pushable destination filters are applied by `hydrate_nodes`; the rest - // (`ir_filter_to_expr` → None) are applied in memory after hconcat. + // (`ir_filter_to_expr` → None) are applied in memory after hconcat. The + // schema arg only affects a pushable literal's TYPE, never Some-vs-None, so + // `None` here yields the same pushable/non-pushable split as `hydrate_nodes`. let non_pushable: Vec<&IRFilter> = dst_filters .iter() - .filter(|f| ir_filter_to_expr(f, params).is_none()) + .filter(|f| ir_filter_to_expr(f, params, None).is_none()) .collect(); // Unique destination ids (first-seen order) for one batched hydration. @@ -1506,7 +1588,8 @@ async fn hydrate_nodes( // `id IN (ids)` AND any pushable destination filters, as a structured Expr. let id_list: Vec = ids.iter().map(|id| lit(id.clone())).collect(); let mut filter_expr = col("id").in_list(id_list, false); - if let Some(dst_expr) = build_lance_filter_expr(dst_filters, params) { + if let Some(dst_expr) = build_lance_filter_expr(dst_filters, params, Some(&node_type.arrow_schema)) + { filter_expr = filter_expr.and(dst_expr); } @@ -1747,21 +1830,23 @@ async fn execute_node_scan( let table_key = format!("node:{}", type_name); let ds = snapshot.open(&table_key).await?; + let node_type = &catalog.node_types[type_name]; + // Lower the IR filters to a DataFusion `Expr` and apply via // `Scanner::filter_expr` inside the configure closure. The string // pushdown path (`build_lance_filter` → `scanner.filter(&str)`) is // gone for node scans — structured Expr unlocks `CompOp::Contains` // pushdown (via `array_has`) and lets DF 53's optimizer rules // (vectorized IN-list, PhysicalExprSimplifier, CASE-NULL shortcut) - // reach our predicates. Other call sites that still take string SQL - // (hydrate_nodes for the Expand pushdown, count_rows, the mutation - // delete path) migrate in follow-up MRs. - let filter_expr = build_lance_filter_expr(filters, params); + // reach our predicates. Passing the node's `arrow_schema` lets the lowering + // coerce literals to each column's exact type so narrow-numeric BTREEs are + // used. Other call sites that still take string SQL (count_rows, the + // mutation delete path) migrate in follow-up MRs. + let filter_expr = build_lance_filter_expr(filters, params, Some(&node_type.arrow_schema)); // Blob columns must be excluded from scan when a filter is present // (Lance bug: BlobsDescriptions + filter triggers a projection assertion). // We exclude blob columns and add metadata post-scan via take_blobs_by_indices. - let node_type = &catalog.node_types[type_name]; let has_blobs = !node_type.blob_properties.is_empty(); let non_blob_cols: Vec<&str> = node_type .arrow_schema @@ -1990,13 +2075,14 @@ pub(super) fn literal_to_sql(lit: &Literal) -> String { pub(super) fn build_lance_filter_expr( filters: &[IRFilter], params: &ParamMap, + schema: Option<&Schema>, ) -> Option { use datafusion::logical_expr::Operator; use datafusion::prelude::Expr; let mut acc: Option = None; for f in filters { - let Some(e) = ir_filter_to_expr(f, params) else { + let Some(e) = ir_filter_to_expr(f, params, schema) else { continue; }; acc = Some(match acc { @@ -2017,6 +2103,7 @@ pub(super) fn build_lance_filter_expr( pub(super) fn ir_filter_to_expr( filter: &IRFilter, params: &ParamMap, + schema: Option<&Schema>, ) -> Option { use datafusion::functions_nested::expr_fn::array_has; @@ -2027,14 +2114,22 @@ pub(super) fn ir_filter_to_expr( // List-contains: `prop CONTAINS value` lowers to `array_has(prop, value)`. // This is the case the old SQL-string pushdown had to return None for // ("Can't pushdown list contains"); with structured Expr it pushes down fine. + // (Element-type coercion for the contained value is deferred — list columns + // are not scalar-indexed, so the index-eligibility concern below does not apply.) if matches!(filter.op, CompOp::Contains) { - let left = ir_expr_to_expr(&filter.left, params)?; - let right = ir_expr_to_expr(&filter.right, params)?; + let left = ir_expr_to_expr(&filter.left, params, None)?; + let right = ir_expr_to_expr(&filter.right, params, None)?; return Some(array_has(left, right)); } - let left = ir_expr_to_expr(&filter.left, params)?; - let right = ir_expr_to_expr(&filter.right, params)?; + // A literal/param operand is coerced to the OTHER operand's column type so + // the predicate stays a direct `col OP literal` and the scalar index is used. + // Without this, DataFusion widens a narrow column (`CAST(col AS Int64)`), + // which defeats the BTREE (validated by `probe_scalar_index_use_under_literal_type`). + let left_col_type = prop_data_type(&filter.left, schema); + let right_col_type = prop_data_type(&filter.right, schema); + let left = ir_expr_to_expr(&filter.left, params, right_col_type.as_ref())?; + let right = ir_expr_to_expr(&filter.right, params, left_col_type.as_ref())?; Some(match filter.op { CompOp::Eq => left.eq(right), CompOp::Ne => left.not_eq(right), @@ -2052,19 +2147,91 @@ pub(super) fn ir_filter_to_expr( pub(super) fn ir_expr_to_expr( expr: &IRExpr, params: &ParamMap, + target: Option<&arrow_schema::DataType>, ) -> Option { - use datafusion::prelude::{col, lit}; + use datafusion::prelude::col; match expr { IRExpr::PropAccess { property, .. } => Some(col(property)), - IRExpr::Literal(l) => literal_to_expr(l), - IRExpr::Param(name) => params.get(name).and_then(literal_to_expr), + IRExpr::Literal(l) => literal_to_expr_coerced(l, target), + IRExpr::Param(name) => params + .get(name) + .and_then(|l| literal_to_expr_coerced(l, target)), _ => None, } } -/// Convert a Literal to a DataFusion `Expr`. Returns `None` for List -/// (which the existing SQL path also can't pushdown — falls through to -/// post-scan in-memory application). +/// The Arrow type of a `PropAccess` operand, looked up in the scan's schema, or +/// `None` if the expr is not a column or the schema/field is unavailable. +fn prop_data_type(expr: &IRExpr, schema: Option<&Schema>) -> Option { + match expr { + IRExpr::PropAccess { property, .. } => schema? + .field_with_name(property) + .ok() + .map(|f| f.data_type().clone()), + _ => None, + } +} + +/// Lower a literal for pushdown, coercing it to `target` (the comparison +/// column's Arrow type) when known. Falls back to the natural-type +/// `literal_to_expr` on a missing target or any coercion failure, so a filter is +/// never demoted to `None` by coercion (a node scan has no in-memory fallback for +/// inline filters — see `execute_node_scan`). +fn literal_to_expr_coerced( + lit: &Literal, + target: Option<&arrow_schema::DataType>, +) -> Option { + if let Some(target) = target { + if let Some(e) = literal_to_typed_expr(lit, target) { + return Some(e); + } + } + literal_to_expr(lit) +} + +/// Build a literal as a typed Arrow scalar matching `target`, reusing the same +/// `literal_to_array` + `arrow_cast` path as the in-memory arm +/// (`projection.rs::evaluate_filter`) so the two arms agree. Returns `None` on +/// any failure (unbuildable literal, incompatible cast) — the caller then falls +/// back to the natural-type literal. +/// +/// Lossless-only for integer targets: typecheck permits numeric cross-type +/// comparisons (`types_compatible`), so a fractional float or out-of-range +/// integer can reach here. Casting those to a narrower integer would truncate +/// (`2.7 -> 2`) or overflow to null, silently changing which rows match. We +/// round-trip the cast and, on mismatch, return `None` so the caller keeps the +/// natural literal — correct via DataFusion coercion, the index just goes unused +/// for that out-of-domain predicate. Float targets are exempt: narrowing +/// `F64 -> F32` is the column's own precision domain, not a value error. +fn literal_to_typed_expr( + lit: &Literal, + target: &arrow_schema::DataType, +) -> Option { + use datafusion::prelude::lit as df_lit; + use datafusion::scalar::ScalarValue; + + let arr = super::projection::literal_to_array(lit, 1).ok()?; + if arr.data_type() == target { + return Some(df_lit(ScalarValue::try_from_array(&arr, 0).ok()?)); + } + let casted = arrow_cast::cast::cast(&arr, target).ok()?; + if target.is_integer() { + let back = arrow_cast::cast::cast(&casted, arr.data_type()).ok()?; + let original = ScalarValue::try_from_array(&arr, 0).ok()?; + let round_tripped = ScalarValue::try_from_array(&back, 0).ok()?; + if original != round_tripped { + return None; + } + } + Some(df_lit(ScalarValue::try_from_array(&casted, 0).ok()?)) +} + +/// Convert a Literal to a DataFusion `Expr` in its NATURAL Arrow type. This is +/// the fallback used when the comparison column's type is unknown (no schema) or +/// when coercion to it fails; the typed, column-matched coercion that keeps +/// scalar indexes usable lives in `literal_to_typed_expr`. Returns `None` for +/// List (the SQL path also could not pushdown it — falls through to post-scan +/// in-memory application). fn literal_to_expr(lit: &Literal) -> Option { use datafusion::prelude::lit as df_lit; Some(match lit { @@ -2073,9 +2240,12 @@ fn literal_to_expr(lit: &Literal) -> Option { Literal::Integer(n) => df_lit(*n), Literal::Float(f) => df_lit(*f), Literal::Bool(b) => df_lit(*b), - // Date/DateTime stored as strings; pass through as string literals - // — Lance/DataFusion handles the comparison against typed columns - // via implicit cast, matching the existing string-SQL behavior. + // Date/DateTime pass through as strings here. Against a typed Date + // column DataFusion casts the LITERAL (`CAST(Utf8 AS Date32)`), which is + // index-safe (proven by `scalar_index_use_requires_matched_literal_type`). + // At real pushdown sites the schema is known, so `literal_to_typed_expr` + // produces a typed Date32/Date64 anyway; this branch is only the + // no-schema fallback. Literal::Date(s) => df_lit(s.clone()), Literal::DateTime(s) => df_lit(s.clone()), Literal::List(_) => return None, @@ -2285,3 +2455,205 @@ mod expand_chooser_tests { assert_eq!(choose_expand_mode(&i), ExpandMode::Csr); } } + +#[cfg(test)] +mod literal_lowering_tests { + use super::*; + use datafusion::prelude::Expr; + use datafusion::scalar::ScalarValue; + + // With the column type known, the generic coercion types a date literal to + // the column's Date32/Date64 (the live pushdown path). Without a target it + // is the natural Utf8 fallback, which is still index-safe for dates because + // DataFusion casts the LITERAL, not the column (proven by + // `lance_surface_guards::scalar_index_use_requires_matched_literal_type`). + #[test] + fn date_literals_coerce_to_typed_arrow_scalars() { + use arrow_schema::DataType; + let dt = literal_to_expr_coerced( + &Literal::DateTime("2024-06-01T12:00:00Z".into()), + Some(&DataType::Date64), + ) + .unwrap(); + assert!( + matches!(dt, Expr::Literal(ScalarValue::Date64(Some(_)), ..)), + "DateTime vs Date64 column must coerce to a typed Date64, got {dt:?}" + ); + let d = literal_to_expr_coerced(&Literal::Date("2024-06-01".into()), Some(&DataType::Date32)) + .unwrap(); + assert!( + matches!(d, Expr::Literal(ScalarValue::Date32(Some(_)), ..)), + "Date vs Date32 column must coerce to a typed Date32, got {d:?}" + ); + let nat = literal_to_expr_coerced(&Literal::Date("2024-06-01".into()), None).unwrap(); + assert!( + matches!(nat, Expr::Literal(ScalarValue::Utf8(Some(_)), ..)), + "no target should keep the natural Utf8 date literal, got {nat:?}" + ); + } + + // A malformed date string makes coercion fail, so it falls back to the + // natural Utf8 literal rather than dropping the predicate to None. + #[test] + fn malformed_date_literal_falls_back_to_string() { + use arrow_schema::DataType; + let bad = literal_to_expr_coerced( + &Literal::DateTime("not-a-date".into()), + Some(&DataType::Date64), + ) + .unwrap(); + assert!( + matches!(bad, Expr::Literal(ScalarValue::Utf8(Some(_)), ..)), + "malformed DateTime literal should fall back to a Utf8 literal, got {bad:?}" + ); + } + + // With a column target, a literal lowers to the column's EXACT Arrow type + // (not its natural width), so DataFusion does not widen and cast the column + // — keeping the scalar BTREE usable. See + // `lance_surface_guards::scalar_index_use_requires_matched_literal_type`. + #[test] + fn integer_literal_coerces_to_narrow_column_type() { + use arrow_schema::DataType; + let i32_lit = literal_to_expr_coerced(&Literal::Integer(5), Some(&DataType::Int32)).unwrap(); + assert!( + matches!(i32_lit, Expr::Literal(ScalarValue::Int32(Some(5)), ..)), + "integer literal vs Int32 column must lower to Int32, got {i32_lit:?}" + ); + let u32_lit = literal_to_expr_coerced(&Literal::Integer(7), Some(&DataType::UInt32)).unwrap(); + assert!( + matches!(u32_lit, Expr::Literal(ScalarValue::UInt32(Some(7)), ..)), + "integer literal vs UInt32 column must lower to UInt32, got {u32_lit:?}" + ); + } + + #[test] + fn float_literal_coerces_to_f32_column_type() { + use arrow_schema::DataType; + let f32_lit = + literal_to_expr_coerced(&Literal::Float(1.5), Some(&DataType::Float32)).unwrap(); + assert!( + matches!(f32_lit, Expr::Literal(ScalarValue::Float32(Some(_)), ..)), + "float literal vs Float32 column must lower to Float32, got {f32_lit:?}" + ); + } + + // Lossless guard: a fractional float against an integer column must NOT + // truncate (2.7 -> 2). Fall back to the natural Float64 so the comparison + // stays exact (no integer equals 2.7). + #[test] + fn fractional_float_vs_int_column_falls_back_not_truncate() { + use arrow_schema::DataType; + let e = literal_to_expr_coerced(&Literal::Float(2.7), Some(&DataType::Int32)).unwrap(); + assert!( + matches!(e, Expr::Literal(ScalarValue::Float64(Some(_)), ..)), + "fractional float vs Int32 must fall back to natural Float64, got {e:?}" + ); + } + + // A whole-number float IS lossless against an integer column, so it coerces. + #[test] + fn whole_float_vs_int_column_coerces() { + use arrow_schema::DataType; + let e = literal_to_expr_coerced(&Literal::Float(2.0), Some(&DataType::Int32)).unwrap(); + assert!( + matches!(e, Expr::Literal(ScalarValue::Int32(Some(2)), ..)), + "whole-number float vs Int32 is lossless and must coerce to Int32(2), got {e:?}" + ); + } + + // Lossless guard: an integer literal outside the column's range must NOT + // overflow to null; fall back to the natural Int64 (correct via DataFusion). + #[test] + fn out_of_range_int_vs_narrow_column_falls_back() { + use arrow_schema::DataType; + let e = literal_to_expr_coerced(&Literal::Integer(3_000_000_000), Some(&DataType::Int32)) + .unwrap(); + assert!( + matches!(e, Expr::Literal(ScalarValue::Int64(Some(3_000_000_000)), ..)), + "out-of-range integer vs Int32 must fall back to natural Int64, got {e:?}" + ); + } + + // Float targets are exempt from the lossless guard: narrowing to the column's + // own precision is the correct comparison domain, even when the value is not + // exactly representable in F32 (0.1). + #[test] + fn float_vs_f32_column_coerces_even_when_not_exactly_representable() { + use arrow_schema::DataType; + let e = literal_to_expr_coerced(&Literal::Float(0.1), Some(&DataType::Float32)).unwrap(); + assert!( + matches!(e, Expr::Literal(ScalarValue::Float32(Some(_)), ..)), + "float target must coerce 0.1 to Float32 (exempt from lossless guard), got {e:?}" + ); + } + + // No target (caller without a schema) keeps the natural width — the existing + // fallback, so behavior never regresses where the column type is unknown. + #[test] + fn literal_without_target_keeps_natural_width() { + let nat = literal_to_expr_coerced(&Literal::Integer(5), None).unwrap(); + assert!( + matches!(nat, Expr::Literal(ScalarValue::Int64(Some(5)), ..)), + "no target should keep the natural Int64 width, got {nat:?}" + ); + } + + // True if either operand of a binary comparison is an Int32 literal. + fn binary_has_int32_literal(e: &Expr) -> bool { + if let Expr::BinaryExpr(b) = e { + [b.left.as_ref(), b.right.as_ref()] + .iter() + .any(|side| matches!(side, Expr::Literal(ScalarValue::Int32(Some(_)), ..))) + } else { + false + } + } + + fn int32_schema() -> arrow_schema::Schema { + use arrow_schema::{DataType, Field}; + arrow_schema::Schema::new(vec![Field::new("count", DataType::Int32, true)]) + } + + fn count_prop() -> IRExpr { + IRExpr::PropAccess { + variable: "m".into(), + property: "count".into(), + } + } + + // Coercion is operator-independent: a range comparison's literal coerces to + // the column type just like equality does, so range filters on a narrow + // numeric column keep the BTREE. + #[test] + fn ir_filter_coerces_literal_for_range_op() { + let schema = int32_schema(); + let filter = IRFilter { + left: count_prop(), + op: CompOp::Ge, + right: IRExpr::Literal(Literal::Integer(2)), + }; + let expr = ir_filter_to_expr(&filter, &ParamMap::new(), Some(&schema)).unwrap(); + assert!( + binary_has_int32_literal(&expr), + "range-op literal must coerce to the Int32 column type, got {expr:?}" + ); + } + + // The column may be on either side; the literal coerces to the opposite + // operand's column type regardless of order (`5 < count`). + #[test] + fn ir_filter_coerces_literal_when_column_is_on_the_right() { + let schema = int32_schema(); + let filter = IRFilter { + left: IRExpr::Literal(Literal::Integer(2)), + op: CompOp::Lt, + right: count_prop(), + }; + let expr = ir_filter_to_expr(&filter, &ParamMap::new(), Some(&schema)).unwrap(); + assert!( + binary_has_int32_literal(&expr), + "reversed-operand literal must coerce to the Int32 column type, got {expr:?}" + ); + } +} diff --git a/crates/omnigraph/src/exec/staging.rs b/crates/omnigraph/src/exec/staging.rs index cbfd52d..464ec34 100644 --- a/crates/omnigraph/src/exec/staging.rs +++ b/crates/omnigraph/src/exec/staging.rs @@ -463,12 +463,28 @@ impl StagedMutation { /// unreferenced (cleaned by `cleanup_old_versions`'s age sweep) /// rather than being committed and creating a Lance-HEAD-ahead /// residual. + /// `held_guards`: when the caller already holds the per-`(table_key, + /// branch)` write queues for every touched table (the fork path acquires + /// them up front, before the fork, and holds them through the manifest + /// publish), it passes `(acquired_keys, guards)` here so `commit_all` + /// reuses them instead of re-acquiring — the queue is a non-re-entrant + /// `tokio::Mutex`, so re-acquiring a held key would self-deadlock. + /// `None` (the steady-state path) means `commit_all` acquires them + /// itself. `acquired_keys` must cover every key `commit_all` would + /// acquire (debug-asserted below) — the guards from `acquire_many` don't + /// carry their keys, so the caller hands the key set alongside them. The + /// fork path guarantees coverage by keying every touched table uniformly + /// by the resolved target branch. pub(crate) async fn commit_all( self, db: &crate::db::Omnigraph, branch: Option<&str>, sidecar_kind: SidecarKind, actor_id: Option<&str>, + held_guards: Option<( + Vec<(String, Option)>, + Vec>, + )>, ) -> Result<( Vec, HashMap, @@ -483,21 +499,18 @@ impl StagedMutation { op_kinds, } = self; - // Acquire per-(table_key, branch) queues for every touched - // table — both staged and inline-committed. Sorted by - // `acquire_many` internally so all multi-table writers - // (mutation, branch_merge, schema_apply, future MR-870 - // recovery) agree on acquisition order — prevents lock-order - // inversion deadlock. + // Per-(table_key, branch) queues for every touched table — both + // staged and inline-committed. Sorted by `acquire_many` internally + // so all multi-table writers (mutation, branch_merge, schema_apply, + // the fork path, recovery) agree on acquisition order — prevents + // lock-order inversion deadlock. // - // For inline-committed tables (delete-only mutations), Lance - // HEAD has already advanced inside `delete_where` before - // `commit_all` runs. Holding the queue here doesn't prevent - // that interleaving (commit 6 will move queue acquisition into - // `delete_where`'s call site); it does prevent another writer - // from interleaving between our delete and our publish, which - // would otherwise leave a Lance-HEAD-ahead residual the - // delete-only sidecar (added below) would have to recover. + // For inline-committed tables (delete-only mutations), Lance HEAD + // has already advanced inside `delete_where` before `commit_all` + // runs. Holding the queue here prevents another writer from + // interleaving between our delete and our publish, which would + // otherwise leave a Lance-HEAD-ahead residual the delete-only + // sidecar (added below) would have to recover. let mut queue_keys: Vec<(String, Option)> = Vec::with_capacity(staged.len() + inline_committed.len()); for entry in &staged { @@ -512,7 +525,30 @@ impl StagedMutation { })?; queue_keys.push((table_key.clone(), path.table_branch.clone())); } - let guards = db.write_queue().acquire_many(&queue_keys).await; + // Reuse the caller's guards (fork path) when handed in, else acquire + // our own. When reusing, every key we would acquire MUST already be + // covered — re-acquiring a held non-re-entrant key would deadlock, and + // a key we'd need but DON'T hold would commit unserialized. This is a + // load-bearing safety invariant, so it is checked in ALL builds (not a + // debug_assert) and fails the write loudly+safely rather than silently + // proceeding unguarded if a future execution path ever touches a table + // outside the caller's pre-computed set. + let guards = match held_guards { + Some((acquired_keys, guards)) => { + let held: std::collections::HashSet<&(String, Option)> = + acquired_keys.iter().collect(); + if let Some(missing) = queue_keys.iter().find(|k| !held.contains(k)) { + return Err(OmniError::manifest_internal(format!( + "commit_all: pre-held write-queue guards do not cover touched table \ + '{}' on branch {:?} — the caller's up-front acquisition set diverged \ + from the staged/inline set (a touched-table-set bug)", + missing.0, missing.1 + ))); + } + guards + } + None => db.write_queue().acquire_many(&queue_keys).await, + }; // Re-capture manifest pins under the queue (PR 2 / MR-686). // diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index 69ada79..2365243 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -418,6 +418,45 @@ async fn load_jsonl_reader( LoadMode::Overwrite => crate::db::MutationOpKind::SchemaRewrite, }; + // Up-front fork-queue acquisition. The first write to a table on a + // non-main branch forks it (create_branch), which advances Lance state + // before the manifest publish; the reclaim of any manifest-unreferenced + // leftover (`reclaim_orphaned_fork_and_refork`) must not race a concurrent + // in-process fork. So when this load will fork at least one touched table, + // acquire the per-(table, branch) write queues for ALL touched tables up + // front (one sorted `acquire_many`, keyed uniformly by the target branch + // so it covers what `commit_all` recomputes) and hold them through the + // publish. Main-branch loads never fork; branch loads where every touched + // table is already forked skip this and let `commit_all` acquire at commit. + let fork_queue_guards: Option<( + Vec<(String, Option)>, + Vec>, + )> = if let Some(active) = branch { + let touched: Vec<(String, Option)> = node_rows + .keys() + .map(|t| (format!("node:{t}"), Some(active.to_string()))) + .chain( + edge_rows + .keys() + .map(|e| (format!("edge:{e}"), Some(active.to_string()))), + ) + .collect(); + let needs_fork = touched.iter().any(|(table_key, _)| { + snapshot + .entry(table_key) + .map(|e| e.table_branch.as_deref() != Some(active)) + .unwrap_or(false) + }); + if needs_fork { + let guards = db.write_queue().acquire_many(&touched).await; + Some((touched, guards)) + } else { + None + } + } else { + None + }; + // Phase 2a: build and validate every node batch up front. Cheap and // synchronous — surfaces validation errors before any S3 traffic. let mut prepared_nodes: Vec<(String, String, RecordBatch, usize)> = @@ -551,7 +590,13 @@ async fn load_jsonl_reader( // across the manifest publish below — see exec/mutation.rs for // the rationale (interleaving prevention). let (updates, expected_versions, sidecar_handle, _queue_guards) = staged - .commit_all(db, branch, crate::db::manifest::SidecarKind::Load, actor_id) + .commit_all( + db, + branch, + crate::db::manifest::SidecarKind::Load, + actor_id, + fork_queue_guards, + ) .await?; // Same finalize → publisher residual as mutations: per-table // staged commits have advanced Lance HEAD, but the manifest diff --git a/crates/omnigraph/src/storage.rs b/crates/omnigraph/src/storage.rs index 1f96b39..357f990 100644 --- a/crates/omnigraph/src/storage.rs +++ b/crates/omnigraph/src/storage.rs @@ -9,7 +9,7 @@ use object_store::aws::AmazonS3Builder; use object_store::local::LocalFileSystem; use object_store::memory::InMemory; use object_store::path::Path as ObjectPath; -use object_store::{DynObjectStore, ObjectStore, PutMode, PutPayload}; +use object_store::{DynObjectStore, ObjectStore, ObjectStoreExt, PutMode, PutPayload}; use url::Url; use crate::error::{OmniError, Result}; diff --git a/crates/omnigraph/src/storage_layer.rs b/crates/omnigraph/src/storage_layer.rs index d2f6b01..7c7685d 100644 --- a/crates/omnigraph/src/storage_layer.rs +++ b/crates/omnigraph/src/storage_layer.rs @@ -184,6 +184,26 @@ pub(crate) fn staged_handles_as_writes(handles: &[StagedHandle]) -> Vec { + Created(D), + RefAlreadyExists, +} + // ─── TableStorage trait ──────────────────────────────────────────────────── /// Engine-internal trait covering every Lance dataset operation an @@ -231,7 +251,7 @@ pub trait TableStorage: sealed::Sealed + Send + Sync + Debug { table_key: &str, source_version: u64, target_branch: &str, - ) -> Result; + ) -> Result>; async fn delete_branch(&self, dataset_uri: &str, branch: &str) -> Result<()>; @@ -497,17 +517,22 @@ impl TableStorage for TableStore { table_key: &str, source_version: u64, target_branch: &str, - ) -> Result { - TableStore::fork_branch_from_state( - self, - dataset_uri, - source_branch, - table_key, - source_version, - target_branch, + ) -> Result> { + Ok( + match TableStore::fork_branch_from_state( + self, + dataset_uri, + source_branch, + table_key, + source_version, + target_branch, + ) + .await? + { + ForkOutcome::Created(ds) => ForkOutcome::Created(SnapshotHandle::new(ds)), + ForkOutcome::RefAlreadyExists => ForkOutcome::RefAlreadyExists, + }, ) - .await - .map(SnapshotHandle::new) } async fn delete_branch(&self, dataset_uri: &str, branch: &str) -> Result<()> { diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index 65123c0..5c99b01 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -26,6 +26,7 @@ use std::sync::Arc; use crate::db::manifest::{TableVersionMetadata, open_table_head_for_write}; use crate::db::{Snapshot, SubTableEntry}; use crate::error::{OmniError, Result}; +use crate::storage_layer::ForkOutcome; #[derive(Debug, Clone, PartialEq, Eq)] pub struct TableState { @@ -285,7 +286,7 @@ impl TableStore { table_key: &str, source_version: u64, target_branch: &str, - ) -> Result { + ) -> Result> { let mut source_ds = self .open_dataset_head(dataset_uri, source_branch) .await? @@ -294,31 +295,49 @@ impl TableStore { .map_err(|e| OmniError::Lance(e.to_string()))?; self.ensure_expected_version(&source_ds, table_key, source_version)?; - if source_ds + if let Err(create_err) = source_ds .create_branch(target_branch, source_version, None) .await - .is_err() { - // The target branch ref already exists. The caller - // (`open_owned_dataset_for_branch_write`) re-reads the live manifest - // before forking and returns a retryable error when a concurrent - // writer legitimately holds the fork, so reaching here means the - // manifest does NOT reference this fork: it is an orphan from an - // incomplete prior `branch_delete`. Surface the actionable cleanup - // error rather than guessing from Lance branch versions. - return Err(OmniError::manifest_conflict(format!( - "branch '{}' has orphaned table state for '{}' from an incomplete \ - prior delete; run `omnigraph cleanup` to reclaim it before reusing \ - this branch name", - target_branch, table_key - ))); + // Disambiguate the failure: only a genuinely pre-existing ref is a + // reclaim candidate. Mapping EVERY create_branch failure to + // `RefAlreadyExists` would route a transient I/O / version / Lance + // internal error into the destructive reclaim path. So check whether + // the ref actually exists; if not, the failure is real — propagate + // it (preserving error fidelity) rather than force-deleting. + // + // `list_branches` reads `_refs/branches/` from the store, so it sees + // a fully-formed manifest-unreferenced fork (our common case — a + // create_branch that completed but whose manifest publish did not). + // It does NOT see a phase-1-only Lance "zombie" (tree dir written, + // no BranchContents) — but neither does `cleanup`'s reconciler, also + // list_branches-based. A zombie only forms if create_branch is + // interrupted *between its two internal phases* (a far narrower + // window than the manifest-publish gap), and it surfaces here as the + // propagated create error requiring manual reclaim. We deliberately + // do NOT force-delete on a not-found-ref failure: it is + // indistinguishable from a transient error on a fresh create, and + // force-deleting there is the destructive overreach this guard + // removes. The caller holds the per-(table, branch) write queue, so + // no in-process writer races this fork; a cross-process create + // between our check and now is the documented one-winner-CAS gap and + // propagates as a retryable error. + let ref_exists = source_ds + .list_branches() + .await + .map(|b| b.contains_key(target_branch)) + .unwrap_or(false); + if ref_exists { + return Ok(ForkOutcome::RefAlreadyExists); + } + return Err(OmniError::Lance(create_err.to_string())); } let ds = self .open_dataset_head(dataset_uri, Some(target_branch)) .await?; self.ensure_expected_version(&ds, table_key, source_version)?; - Ok(ds) + Ok(ForkOutcome::Created(ds)) } pub async fn scan_batches(&self, ds: &Dataset) -> Result> { @@ -705,6 +724,36 @@ impl TableStore { Ok(IndexCoverage::Indexed) } + /// True if any non-system index on `ds` leaves at least one current + /// fragment uncovered, i.e. rows that the index does not yet account for + /// (appended after the index was built, or rewritten by compaction). Such + /// fragments are scanned unindexed until a reindex (`optimize_indices`) + /// folds them in. Returns false when every index covers every fragment, or + /// when the table has no (non-system) indices to optimize. A `None` + /// `fragment_bitmap` means Lance cannot report coverage for that index, so + /// we do not treat it as uncovered (mirrors `key_column_index_coverage`). + /// + /// Used by `optimize` to decide whether an otherwise-already-compacted + /// table still has index work to do. + pub async fn has_unindexed_fragments(ds: &Dataset) -> Result { + let indices = ds + .load_indices() + .await + .map_err(|e| OmniError::Lance(e.to_string()))?; + let frag_ids: Vec = ds.fragments().iter().map(|f| f.id as u32).collect(); + for index in indices.iter() { + if is_system_index(index) { + continue; + } + if let Some(bitmap) = index.fragment_bitmap.as_ref() { + if frag_ids.iter().any(|id| !bitmap.contains(*id)) { + return Ok(true); + } + } + } + Ok(false) + } + pub async fn count_rows(&self, ds: &Dataset, filter: Option) -> Result { ds.count_rows(filter) .await @@ -745,6 +794,8 @@ impl TableStore { let params = WriteParams { mode: WriteMode::Append, allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; ds.append(reader, Some(params)) @@ -764,6 +815,8 @@ impl TableStore { let params = WriteParams { mode: WriteMode::Append, allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; ds.append(reader, Some(params)) @@ -777,6 +830,8 @@ impl TableStore { enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; Dataset::write(reader, dataset_uri, Some(params)) @@ -867,6 +922,8 @@ impl TableStore { let params = WriteParams { mode: WriteMode::Append, allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; let transaction = InsertBuilder::new(Arc::new(ds.clone())) @@ -1040,7 +1097,16 @@ impl TableStore { ds: Arc, transaction: Transaction, ) -> Result { + // Skip Lance's auto-cleanup hook on every commit. OmniGraph owns version + // GC explicitly (optimize.rs::cleanup_all_tables); Lance's hook fires off + // the *dataset's stored* `lance.auto_cleanup.*` config, which graphs + // created before the v7 bump (6.0.1 defaulted auto_cleanup ON) still + // carry — so `WriteParams::auto_cleanup = None` alone does NOT stop it on + // upgraded graphs. Skipping here covers the staged write path (the main + // data path) for new and legacy datasets alike, preventing Lance from + // GC'ing versions the __manifest still pins for snapshots/time-travel. CommitBuilder::new(ds) + .with_skip_auto_cleanup(true) .execute(transaction) .await .map_err(|e| OmniError::Lance(e.to_string())) @@ -1087,6 +1153,8 @@ impl TableStore { mode: WriteMode::Overwrite, enable_stable_row_ids: true, allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; let transaction = InsertBuilder::new(Arc::new(ds.clone())) @@ -1503,6 +1571,8 @@ impl TableStore { enable_stable_row_ids: true, data_storage_version: Some(LanceFileVersion::V2_2), allow_external_blob_outside_bases: true, + auto_cleanup: None, + skip_auto_cleanup: true, ..Default::default() }; Dataset::write(reader, dataset_uri, Some(params)) diff --git a/crates/omnigraph/tests/failpoints.rs b/crates/omnigraph/tests/failpoints.rs index b45cfa0..38a60ae 100644 --- a/crates/omnigraph/tests/failpoints.rs +++ b/crates/omnigraph/tests/failpoints.rs @@ -5,7 +5,9 @@ mod helpers; use fail::FailScenario; use futures::FutureExt; use omnigraph::db::Omnigraph; +use omnigraph::error::{ManifestErrorKind, OmniError}; use omnigraph::failpoints::ScopedFailPoint; +use omnigraph::loader::LoadMode; use helpers::recovery::{ FollowUpMutation, RecoveryExpectation, TableExpectation, assert_post_recovery_invariants, @@ -127,12 +129,12 @@ async fn branch_delete_partial_failure_converges_via_cleanup() { } // Reusing a branch name whose delete left an orphaned fork (before `cleanup` -// reconciles it) must fail with a clear, actionable error pointing at -// `cleanup`, not the opaque `ExpectedVersionMismatch` that leaks from the fork -// path. The recreate itself succeeds; the first write to the previously-forked -// table is where the stale orphan collides. +// reconciles it) must SELF-HEAL on the next write — the write reclaims the +// manifest-unreferenced fork and re-forks, rather than wedging with "incomplete +// prior delete; run cleanup". (This test was the inverse before the fork-as- +// idempotent-reconcile fix; its flip is the signal the bug class is closed.) #[tokio::test] -async fn recreate_over_orphaned_fork_before_cleanup_is_actionable() { +async fn recreate_over_orphaned_fork_self_heals_without_cleanup() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap().to_string(); @@ -158,10 +160,10 @@ async fn recreate_over_orphaned_fork_before_cleanup_is_actionable() { } // Recreate the name and write to the previously-forked table WITHOUT a - // cleanup in between. + // cleanup in between. The write must self-heal the stale orphan fork. main.branch_create("feature").await.unwrap(); let mut feature2 = Omnigraph::open(&uri).await.unwrap(); - let err = helpers::mutate_branch( + helpers::mutate_branch( &mut feature2, "feature", MUTATION_QUERIES, @@ -169,20 +171,83 @@ async fn recreate_over_orphaned_fork_before_cleanup_is_actionable() { &mixed_params(&[("$name", "Frank")], &[("$age", 41)]), ) .await - .expect_err("write should collide with the stale orphaned fork"); + .expect("recreate-over-orphan write must self-heal, not require cleanup"); - let msg = err.to_string(); - assert!( - msg.contains("cleanup") - && (msg.contains("orphan") || msg.contains("incomplete prior delete")), - "expected an actionable orphaned-fork error pointing at cleanup, got: {msg}" - ); - assert!( - !msg.contains("expected manifest table version"), - "should not surface the opaque ExpectedVersionMismatch, got: {msg}" + // The recreated branch forks FRESH from main: the deleted branch's Eve is + // gone and only the new Frank is added on top of main's seed. A count of + // main + 2 would mean Eve resurrected from the stale fork (the bug). + let main_people = helpers::count_rows(&main, "node:Person").await; + let feature_people = helpers::count_rows_branch(&feature2, "feature", "node:Person").await; + assert_eq!( + feature_people, + main_people + 1, + "self-healed feature must fork fresh from main (+Frank only); \ + main={main_people}, feature={feature_people} (main+2 ⇒ Eve resurrected)" ); } +// The write-path orphan reclaim shares the same fresh-authority classifier as +// cleanup. If that classifier is Indeterminate (transient read on a live +// branch), the write must return a clear retryable authority-read conflict and +// leave the ref in place. It must not squeeze the ambiguity through +// ExpectedVersionMismatch with expected == actual, which lies about the cause. +#[tokio::test] +async fn recreate_over_orphaned_fork_reports_indeterminate_authority_read() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let db = helpers::init_and_load(&dir).await; + db.branch_create("feature").await.unwrap(); + + let person_uri = node_table_uri(&uri, "Person"); + { + let mut ds = lance::Dataset::open(&person_uri).await.unwrap(); + let base = ds.version().version; + ds.create_branch("feature", base, None).await.unwrap(); + } + + let row = r#"{"type":"Person","data":{"name":"Grace","age":37}}"#; + { + let _fp = ScopedFailPoint::new("classify.fresh_read", "return"); + let err = db + .load_as("feature", None, row, LoadMode::Merge, None) + .await + .expect_err("indeterminate authority read must fail retryably"); + + match &err { + OmniError::Manifest(manifest) => { + assert_eq!(manifest.kind, ManifestErrorKind::Conflict); + assert!( + manifest.details.is_none(), + "indeterminate authority read is not an expected-version mismatch: {manifest:?}" + ); + } + other => panic!("expected manifest conflict, got {other:?}"), + } + let message = err.to_string(); + assert!( + message.contains("could not verify") + && message.contains("fresh manifest authority was unavailable") + && message.contains("refresh and retry"), + "error should name the unavailable authority read, got: {message}" + ); + assert!( + !message.contains("expected manifest table version"), + "indeterminate authority must not be reported as a version mismatch: {message}" + ); + + let ds = lance::Dataset::open(&person_uri).await.unwrap(); + assert!( + ds.list_branches().await.unwrap().contains_key("feature"), + "ambiguous orphan status must leave the fork for a later retry" + ); + } + + db.load_as("feature", None, row, LoadMode::Merge, None) + .await + .expect("when fresh authority is available, the orphan is reclaimed and write converges"); +} + // cleanup is the guaranteed convergence backstop, so one table's transient // failure must not abort the whole sweep. Inject a one-shot version-GC failure // for a single table and assert: cleanup still succeeds, the failure is @@ -330,6 +395,68 @@ async fn cleanup_reclaims_orphaned_commit_graph_branch() { } } +// `classify_fork_ref` returns `Indeterminate` when the fresh-authority read +// fails on a LIVE branch — and a destructive caller must SKIP, never delete, on +// that ambiguity. Here the reconciler has a genuine origin-2 orphan candidate +// (a manifest-unreferenced Person fork on the live `feature` branch), but the +// `classify.fresh_read` failpoint makes the fresh re-check fail: cleanup must +// leave the ref in place (cannot confirm it is unreferenced), then reclaim it on +// the next run once the read succeeds. This pins the Indeterminate arm and the +// don't-destroy-on-ambiguity rule end-to-end through cleanup. +#[tokio::test] +async fn reconcile_skips_fork_when_fresh_recheck_is_unavailable_then_converges() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let mut db = helpers::init_and_load(&dir).await; + db.branch_create("feature").await.unwrap(); + + // Forge a manifest-unreferenced Person fork on the live `feature` branch — + // a genuine orphan the reconciler would normally reclaim. + let person_uri = node_table_uri(&uri, "Person"); + { + let mut ds = lance::Dataset::open(&person_uri).await.unwrap(); + let base = ds.version().version; + ds.create_branch("feature", base, None).await.unwrap(); + assert!( + ds.list_branches().await.unwrap().contains_key("feature"), + "precondition: forged orphan fork present" + ); + } + + // With the fresh re-check failing, the fork's status is Indeterminate (the + // branch is live but unreadable) → cleanup must SKIP it, not delete. + { + let _fp = ScopedFailPoint::new("classify.fresh_read", "return"); + db.cleanup(omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }) + .await + .unwrap(); + let ds = lance::Dataset::open(&person_uri).await.unwrap(); + assert!( + ds.list_branches().await.unwrap().contains_key("feature"), + "reconcile must NOT delete a fork whose fresh re-check is inconclusive" + ); + } + + // Read succeeds now → cleanup confirms the orphan and reclaims it (converges). + db.cleanup(omnigraph::db::CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }) + .await + .unwrap(); + { + let ds = lance::Dataset::open(&person_uri).await.unwrap(); + assert!( + !ds.list_branches().await.unwrap().contains_key("feature"), + "next cleanup (fresh read available) must reclaim the confirmed orphan" + ); + } +} + // A branch_delete whose best-effort commit-graph reclaim fails leaves a // commit-graph "zombie" branch. Recreating that name must heal the zombie and // succeed (branch_create force-deletes a stale commit-graph ref since the @@ -2619,69 +2746,66 @@ async fn finalize_publisher_residual_does_not_drift_untouched_tables() { } /// Acceptance test: a stage-step failure in the staged-index path -/// (`stage_create_btree_index` succeeded; `commit_staged` not yet -/// called) leaves NO Lance-HEAD drift on the existing tables. -/// Subsequent operations against those tables succeed without -/// `ExpectedVersionMismatch`. +/// (`stage_create_btree_index` succeeded; `commit_staged` not yet called) +/// leaves NO Lance-HEAD drift, so other tables stay writable. /// -/// Path: `apply_schema(v1 → v2)` adds a new node type. The -/// `added_tables` loop in `schema_apply` creates the empty dataset and -/// then calls `build_indices_on_dataset_for_catalog` → -/// `stage_and_commit_btree(..., &["id"])`. The failpoint fires -/// between `stage_create_btree_index` and `commit_staged`, so the -/// staged segments are written under `_indices//` but Lance HEAD -/// on the new dataset is unchanged at v=1. The schema-apply lock -/// branch is released by `apply_schema`'s outer match. Existing -/// tables (e.g. `node:Person`) are completely untouched by the new -/// node's added_tables iteration — they're outside the failed apply -/// path entirely — and we assert that mutations against them continue -/// to work. -/// -/// The orphan empty dataset from the failed apply is acceptable -/// residual: it's unreferenced by `__manifest` and will be reclaimed -/// by `cleanup_old_versions` (or removed when a future apply at the -/// same target path resolves the rename). +/// Under iss-848 schema apply no longer builds indexes inline — the build +/// happens in the reconciler (`ensure_indices`/`optimize`) and at load. So this +/// fires the failpoint where it lives now: an `ensure_indices` build of a BTREE +/// that a prior apply declared (`@index`) but deferred. The failpoint fires +/// between `stage_create_btree_index` and `commit_staged`, so the staged +/// segment is written under `_indices//` but `node:Person`'s Lance HEAD is +/// unchanged. `ensure_indices` fails and its EnsureIndices sidecar pins only +/// Person at NoMovement (a clean no-op on the next open). A write to a +/// different, unpinned table (`node:Company`) is unaffected: mutations/loads run +/// a roll-forward-only heal and proceed — they do not refuse on a pending +/// sidecar the way `optimize`/`repair` do — so the write succeeds with no drift. #[tokio::test] async fn ensure_indices_stage_btree_failure_leaves_existing_tables_writable() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap().to_string(); - - // Init with TEST_SCHEMA which declares Person + Knows. Indices on - // those tables get built during init. let mut db = Omnigraph::init(&uri, helpers::TEST_SCHEMA).await.unwrap(); - // Apply a schema that adds a new node type. The added_tables loop - // will hit the failpoint between stage and commit on the new - // node:Project table's btree-on-id build. (TEST_SCHEMA already - // has Person + Company + Knows + WorksAt — pick a name that isn't - // already declared.) - let extended_schema = format!( - "{}\nnode Project {{ name: String @key }}\n", - helpers::TEST_SCHEMA - ); - - { - let _failpoint = - ScopedFailPoint::new("ensure_indices.post_stage_pre_commit_btree", "return"); - let err = db.apply_schema(&extended_schema).await.unwrap_err(); - assert!( - err.to_string() - .contains("ensure_indices.post_stage_pre_commit_btree"), - "schema apply should fail with the synthetic failpoint error, got: {err}" - ); - } - - // Existing tables stayed at their pre-apply versions; subsequent - // mutations against them succeed (no Lance-HEAD drift). + // Seed a Person row — the load builds Person's id BTREE + name FTS. mutate_main( &mut db, helpers::MUTATION_QUERIES, "insert_person", - &mixed_params(&[("$name", "Eve")], &[("$age", 22)]), + &mixed_params(&[("$name", "Alice")], &[("$age", 30)]), ) .await - .expect("Person mutation must succeed after the failed schema apply — existing tables are not drifted"); + .expect("seed Person"); + + // Add `@index` on `age`: schema apply records the intent but defers the + // physical build (iss-848), so the BTREE on `age` is unbuilt. + let indexed_schema = helpers::TEST_SCHEMA.replace("age: I32?", "age: I32? @index"); + db.apply_schema(&indexed_schema) + .await + .expect("adding an @index is metadata-only and succeeds"); + + { + // ensure_indices builds the deferred `age` BTREE on Person; the failpoint + // fires between stage and commit, so Person's Lance HEAD does not move. + let _failpoint = + ScopedFailPoint::new("ensure_indices.post_stage_pre_commit_btree", "return"); + let err = db.ensure_indices().await.unwrap_err(); + assert!( + err.to_string() + .contains("ensure_indices.post_stage_pre_commit_btree"), + "ensure_indices should fail with the synthetic failpoint error, got: {err}" + ); + } + + // A different, unpinned table is untouched by the failed index build. + use omnigraph::loader::{LoadMode, load_jsonl}; + load_jsonl( + &mut db, + r#"{"type": "Company", "data": {"name": "Acme"}}"#, + LoadMode::Append, + ) + .await + .expect("Company write on a table untouched by the failed ensure_indices should succeed"); } fn assert_no_staging_files(graph: &std::path::Path) { diff --git a/crates/omnigraph/tests/helpers/mod.rs b/crates/omnigraph/tests/helpers/mod.rs index 295cab7..6476e1a 100644 --- a/crates/omnigraph/tests/helpers/mod.rs +++ b/crates/omnigraph/tests/helpers/mod.rs @@ -54,6 +54,19 @@ pub async fn init_and_load(dir: &tempfile::TempDir) -> Omnigraph { db } +/// On-disk Lance dataset URI for a node type, mirroring the engine's +/// `nodes/{fnv1a(type)}` layout. Used by tests that reach the raw Lance +/// dataset to forge or inspect branch state. (Local copies exist in +/// `failpoints.rs` / `maintenance.rs`; this is the shared one for new tests.) +pub fn node_table_uri(root: &str, type_name: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for &b in type_name.as_bytes() { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100_0000_01b3); + } + format!("{}/nodes/{hash:016x}", root.trim_end_matches('/')) +} + /// Read all rows from a sub-table by table_key. pub async fn read_table(db: &Omnigraph, table_key: &str) -> Vec { let snap = snapshot_main(db).await.unwrap(); diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index 370f9e7..9a0c6bd 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -32,10 +32,14 @@ use lance::dataset::builder::DatasetBuilder; use lance::dataset::optimize::{CompactionOptions, compact_files}; use lance::dataset::transaction::Operation; use lance::dataset::write::delete::DeleteResult; -use lance::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams}; +use lance::dataset::{ + CommitBuilder, InsertBuilder, MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, + WriteParams, +}; use lance::index::DatasetIndexExt; use lance_file::version::LanceFileVersion; use lance_index::IndexType; +use lance_index::optimize::OptimizeOptions; use lance_index::scalar::ScalarIndexParams; use lance_namespace::LanceNamespace; use lance_table::io::commit::ManifestNamingScheme; @@ -541,3 +545,449 @@ async fn fragment_deletion_metadata_is_available() { per-fragment deletions and would need to read the deletion vector.", ); } + +// --- Guard 14: Dataset::optimize_indices signature ---------------------------- +// +// `db/omnigraph/optimize.rs::optimize_one_table` calls +// `ds.optimize_indices(&OptimizeOptions::default())` (via `DatasetIndexExt`) to +// fold appended/compacted fragments back into existing indexes. If Lance +// changes the receiver, the options type, or the return shape, this fails to +// compile. Compile-only. + +#[allow( + dead_code, + unreachable_code, + unused_variables, + unused_mut, + clippy::diverging_sub_expression +)] +async fn _compile_optimize_indices_signature() -> lance::Result<()> { + let mut ds: Dataset = unimplemented!(); + let options = OptimizeOptions::default(); + // `&mut self`, `&OptimizeOptions`, returns `Result<()>` (mutates in place + // and commits — there is no uncommitted variant in this Lance, which is why + // optimize treats it as an inline-commit residual under a recovery sidecar). + let _: () = ds.optimize_indices(&options).await?; + Ok(()) +} + +// --- Guard 15: optimize_indices extends fragment coverage ---------------------- +// +// PR3's reindex assumes `optimize_indices` folds fragments appended AFTER an +// index was built into that index (incremental merge, not retrain). This pins +// that Lance behavior at the surface layer so a regression turns red here, the +// first smoke check on a Lance bump, before the slower engine suite. + +#[tokio::test] +async fn optimize_indices_extends_fragment_coverage() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("guard_optimize_indices.lance"); + let uri = uri.to_str().unwrap(); + + // Fragment 0: alice, bob. Build a BTREE over `value` covering only it. + let mut ds = fresh_dataset(uri).await; + ds.create_index_builder(&["value"], IndexType::BTree, &ScalarIndexParams::default()) + .replace(true) + .await + .unwrap(); + + // Append a second fragment the existing index does not cover. + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("value", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["carol"])), + Arc::new(Int32Array::from(vec![3])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let params = WriteParams { + mode: WriteMode::Append, + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }; + Dataset::write(reader, uri, Some(params)).await.unwrap(); + + let mut ds = Dataset::open(uri).await.unwrap(); + assert!( + value_index_uncovered_count(&ds).await > 0, + "appended fragment should be uncovered by the BTREE before optimize_indices" + ); + + ds.optimize_indices(&OptimizeOptions::default()) + .await + .unwrap(); + + assert_eq!( + value_index_uncovered_count(&ds).await, + 0, + "optimize_indices must fold the appended fragment into the existing index \ + (incremental coverage); if this regresses, PR3's reindex no longer keeps \ + coverage current — revisit db/omnigraph/optimize.rs and docs/dev/lance.md." + ); +} + +/// Count current fragments not covered by the single-column `value` BTREE — +/// mirrors `TableStore::has_unindexed_fragments` (load_indices + +/// `fragment_bitmap.contains`), pinned by Guard 11. +async fn value_index_uncovered_count(ds: &Dataset) -> usize { + let indices = ds.load_indices().await.unwrap(); + let frag_ids: Vec = ds.fragments().iter().map(|f| f.id as u32).collect(); + let value_fid = ds.schema().field("value").unwrap().id; + for index in indices.iter() { + if index.fields.len() == 1 && index.fields[0] == value_fid { + if let Some(bitmap) = index.fragment_bitmap.as_ref() { + return frag_ids.iter().filter(|id| !bitmap.contains(**id)).count(); + } + } + } + // No `value` index found — treat as fully uncovered so a missing index + // is never mistaken for full coverage. + frag_ids.len() +} + +// --- Guard 16: scalar index use requires a literal matching the column type --- +// +// Pins the substrate behavior the pushdown literal-coercion fix relies on +// (`query.rs::literal_to_typed_expr`): Lance uses the BTREE only when the filter +// is `column OP literal` with a matching type. A width-mismatched literal makes +// DataFusion widen and cast the COLUMN (`CAST(n32 AS Int64)`), which drops the +// scalar index and full-scans. Temporal columns are immune (DataFusion casts the +// Utf8 LITERAL to the date type, not the column). If a Lance/DataFusion bump +// changes either coercion direction, this turns red — re-validate the fix. +#[tokio::test] +async fn scalar_index_use_requires_matched_literal_type() { + use datafusion::physical_plan::displayable; + use datafusion::prelude::{col, lit}; + use datafusion::scalar::ScalarValue; + + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("probe_literal_type.lance"); + let uri = uri.to_str().unwrap(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("n32", DataType::Int32, false), + Field::new("d32", DataType::Date32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["a", "b", "c", "d"])), + Arc::new(Int32Array::from(vec![1, 5, 9, 13])), + Arc::new(arrow_array::Date32Array::from(vec![19000, 19723, 20000, 20500])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let params = WriteParams { + mode: WriteMode::Create, + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }; + let mut ds = Dataset::write(reader, uri, Some(params)).await.unwrap(); + for c in ["n32", "d32"] { + ds.create_index_builder(&[c], IndexType::BTree, &ScalarIndexParams::default()) + .replace(true) + .await + .unwrap(); + } + + async fn plan_str(ds: &Dataset, filter: datafusion::prelude::Expr) -> String { + let mut scanner = ds.scan(); + scanner.filter_expr(filter); + let plan = scanner.create_plan().await.unwrap(); + format!("{}", displayable(plan.as_ref()).indent(true)) + } + + // (label, filter, expect_index_used) + let cases = [ + ("n32 = 5i32 (matched Int32)", col("n32").eq(lit(5i32)), true), + ("n32 = 5i64 (widened Int64)", col("n32").eq(lit(5i64)), false), + ( + "d32 = Date32 (matched)", + col("d32").eq(lit(ScalarValue::Date32(Some(19723)))), + true, + ), + ( + "d32 = '2024-01-01' (Utf8 vs Date32)", + col("d32").eq(lit("2024-01-01")), + true, + ), + ]; + + for (label, filter, expect_index) in cases { + let s = plan_str(&ds, filter).await; + let uses_index = s.contains("ScalarIndexQuery"); + assert_eq!( + uses_index, expect_index, + "[{label}] expected scalar-index use = {expect_index}, got {uses_index}.\n\ + A change here means Lance/DataFusion shifted its coercion or index \ + pushdown; re-validate query.rs::literal_to_typed_expr.\nplan:\n{s}" + ); + } + + // The widened case must show the index-defeating column CAST (the precise + // shape the fix avoids by coercing the literal to the column type). + let widened = plan_str(&ds, col("n32").eq(lit(5i64))).await; + assert!( + widened.contains("CAST(n32 AS Int64)"), + "expected a column-side cast in the widened plan, got:\n{widened}" + ); +} + +// --- Guard 17: BTREE scalar-index range-boundary correctness (lance#6796) ----- +// +// lance#6796 (issue #6792) fixed a BTREE range-query bound-inclusiveness bug: +// `price <= 10 AND price > 5` returned the wrong boundary row (5.0 instead of +// 10.0). OmniGraph today builds BTREE only on string `@key` columns and queries +// them by equality/IN, not range, so its current patterns do not hit this — the +// guard protects any future BTREE-range path. It reproduces the exact #6792 shape +// (5 rows + an explicit BTREE drives the index path even on tiny data, per the +// upstream repro) and pins the corrected inclusive-`<=` / exclusive-`>` semantics. +#[tokio::test] +async fn btree_range_query_boundary_is_correct() { + use arrow_array::Float64Array; + use futures::TryStreamExt; + + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("guard17.lance"); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("price", DataType::Float64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])), + Arc::new(Float64Array::from(vec![1.0, 5.0, 10.0, 15.0, 20.0])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let params = WriteParams { + mode: WriteMode::Create, + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }; + let mut ds = Dataset::write(reader, uri.to_str().unwrap(), Some(params)) + .await + .unwrap(); + + // Build the BTREE on the numeric column so the range filter resolves through + // the scalar index (the path lance#6796 fixed). + ds.create_index_builder(&["price"], IndexType::BTree, &ScalarIndexParams::default()) + .replace(true) + .await + .unwrap(); + + let mut scanner = ds.scan(); + scanner.filter("price <= 10.0 AND price > 5.0").unwrap(); + let batches: Vec = scanner + .try_into_stream() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut got: Vec = Vec::new(); + for b in &batches { + let col = b + .column_by_name("price") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..col.len() { + got.push(col.value(i)); + } + } + got.sort_by(|a, b| a.partial_cmp(b).unwrap()); + assert_eq!( + got, + vec![10.0], + "BTREE range `price <= 10 AND price > 5` must return exactly [10.0] \ + (lance#6796 / issue #6792 boundary fix); got {got:?}. If this regressed, \ + Lance reintroduced the range-bound inclusiveness bug.", + ); +} + +// --- Guard 18: skip_auto_cleanup suppresses version GC (lance#6755 / PR #229) -- +// +// After the v7 bump, OmniGraph relies on `CommitBuilder::with_skip_auto_cleanup` +// (`commit_staged`) and `MergeInsertBuilder::skip_auto_cleanup` (the `__manifest` +// publisher) to stop Lance's per-commit auto-cleanup hook from GC'ing versions +// the `__manifest` pins for snapshots/time-travel. This is load-bearing for +// graphs created BEFORE the bump: 6.0.1 defaulted `WriteParams::auto_cleanup` ON, +// so those datasets carry `lance.auto_cleanup.*` config that `auto_cleanup = None` +// on new writes cannot retroactively clear — only the per-commit skip stops it. +// +// Pins both halves: WITHOUT the skip the aggressive config GCs v1; WITH the skip +// (the exact call `commit_staged` makes) v1 survives. +#[tokio::test] +async fn skip_auto_cleanup_suppresses_version_gc() { + use std::collections::HashMap; + + // The cleanup config 6.0.1 stored by default, made aggressive: fire on every + // commit, delete anything older than now. + async fn set_legacy_cleanup(ds: &mut Dataset) { + let mut cfg = HashMap::new(); + cfg.insert("lance.auto_cleanup.interval".to_string(), "1".to_string()); + cfg.insert("lance.auto_cleanup.older_than".to_string(), "0ms".to_string()); + ds.update_config(cfg).await.unwrap(); + } + fn row(i: i32) -> (Arc, RecordBatch) { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("value", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec![format!("k{i}")])), + Arc::new(Int32Array::from(vec![i])), + ], + ) + .unwrap(); + (schema, batch) + } + + // Negative control: WITHOUT skip, the legacy config GCs the pinned v1. + let ctrl = tempfile::tempdir().unwrap(); + let curi = ctrl.path().join("g18_ctrl.lance"); + let curi = curi.to_str().unwrap(); + let mut ds = fresh_dataset(curi).await; + let v1 = ds.version().version; + set_legacy_cleanup(&mut ds).await; + for i in 0..5 { + let (schema, batch) = row(i); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + ds.append( + reader, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + } + assert!( + ds.checkout_version(v1).await.is_err(), + "negative control: without skip_auto_cleanup, the legacy auto_cleanup \ + config should have GC'd pinned v{v1}; if this fails the config is not \ + firing and the positive assertion below proves nothing." + ); + + // The guarantee: WITH the per-commit skip, v1 survives. Mirrors + // `TableStore::commit_staged` (InsertBuilder::execute_uncommitted + + // CommitBuilder::with_skip_auto_cleanup(true)). + let keep = tempfile::tempdir().unwrap(); + let kuri = keep.path().join("g18.lance"); + let kuri = kuri.to_str().unwrap(); + let mut ds = fresh_dataset(kuri).await; + let v1 = ds.version().version; + set_legacy_cleanup(&mut ds).await; + for i in 0..5 { + let (_schema, batch) = row(i); + let tx = InsertBuilder::new(Arc::new(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + ds = CommitBuilder::new(Arc::new(ds.clone())) + .with_skip_auto_cleanup(true) + .execute(tx) + .await + .unwrap(); + } + assert!( + ds.checkout_version(v1).await.is_ok(), + "v{v1} was GC'd despite CommitBuilder::with_skip_auto_cleanup(true) — the \ + commit_staged / publisher skip is the only thing protecting \ + __manifest-pinned versions on upgraded (pre-bump) graphs." + ); +} + +// --- Guard 19: unenforced primary key is immutable once set (lance v7) ------ +// +// Lance 7 (`lance::dataset::transaction`) makes the unenforced PK reserved: +// once `lance-schema:unenforced-primary-key` is set on a field, any later write +// that touches that reserved key — even re-applying the SAME value — errors +// "the unenforced primary key is a reserved key and cannot be changed once set". +// +// This is the upstream behavior that broke +// `db/manifest/migrations.rs::migrate_v1_to_v2`'s crash-idempotency: a +// pre-v0.4.0 graph that crashed after the field-set but before the stamp bump +// re-enters the migration with the PK already present, and on Lance 6 the +// re-apply was a no-op. The migration now guards the set on the manifest's +// unenforced-PK field (`["object_id"]` → no-op, `[]` → set, anything else → +// loud refusal). If Lance ever relaxes immutability (a re-set becomes a no-op +// again), this guard goes red — revisit whether that field-guard is still +// needed, and re-pin docs/dev/lance.md. +#[tokio::test] +async fn unenforced_primary_key_is_immutable_once_set() { + use lance::datatypes::LANCE_UNENFORCED_PRIMARY_KEY; + + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("g19.lance"); + let mut ds = fresh_dataset(uri.to_str().unwrap()).await; + + // Precondition: no unenforced PK yet (mirrors a genuine pre-v0.4.0 manifest). + assert!( + ds.schema().unenforced_primary_key().is_empty(), + "fresh dataset should carry no unenforced primary key" + ); + + // First set succeeds — the genuine pre-v0.4.0 migration path. (Discard the + // returned &Schema so the &mut borrow ends before the next call.) + ds.update_field_metadata() + .update( + "id", + [(LANCE_UNENFORCED_PRIMARY_KEY.to_string(), "true".to_string())], + ) + .unwrap() + .await + .unwrap(); + let pk: Vec = ds + .schema() + .unenforced_primary_key() + .iter() + .map(|field| field.name.clone()) + .collect(); + assert_eq!( + pk, + ["id"], + "first set should install `id` as the unenforced PK" + ); + + // Re-applying the SAME reserved key must still error. Normalize the sync + // validation stage (`.update()`) and the async commit stage (`.await`) into + // one Result so the actionable diagnostic below fires whichever stage Lance + // enforces immutability at — and even if a future Lance relaxes it to `Ok`. + // Bare `.unwrap()` / `.unwrap_err()` would instead panic with a generic + // message in those cases, defeating the guard's purpose. + let outcome: lance::Result<()> = match ds.update_field_metadata().update( + "id", + [(LANCE_UNENFORCED_PRIMARY_KEY.to_string(), "true".to_string())], + ) { + Ok(builder) => builder.await.map(|_| ()), + Err(e) => Err(e), + }; + assert!( + matches!(&outcome, Err(e) if e.to_string().contains("cannot be changed once set")), + "Lance no longer rejects re-setting the unenforced PK as immutable \ + (got: {outcome:?}); immutability relaxed or moved off the commit path \ + — revisit migrate_v1_to_v2's field-guard and re-pin docs/dev/lance.md." + ); +} diff --git a/crates/omnigraph/tests/lance_version_columns.rs b/crates/omnigraph/tests/lance_version_columns.rs index b9367b9..fbe0cb4 100644 --- a/crates/omnigraph/tests/lance_version_columns.rs +++ b/crates/omnigraph/tests/lance_version_columns.rs @@ -191,14 +191,16 @@ async fn lance_merge_insert_new_row_stamps_created_at_version() { let eve = rows.iter().find(|r| r.0 == "eve").unwrap(); eprintln!("Eve: created_at_version={}, v1={}, v2={}", eve.2, v1, v2); - // Lance behavior (as of 3.0.1): merge_insert stamps new rows with - // _row_created_at_version = dataset_creation_version (v1), NOT the - // merge_insert commit version (v2). This is why Omnigraph's change - // detection uses _row_last_updated_at_version + ID set membership - // to classify inserts vs updates, not _row_created_at_version alone. + // Lance behavior (7.0.0, lance#6774): merge_insert stamps new INSERT + // rows with _row_created_at_version = the commit version (v2). Earlier + // Lance used a fallback of the dataset creation version; #6774 changed + // it so created_at reflects when the row actually entered the dataset. + // Omnigraph's change detection keys on _row_last_updated_at_version + ID + // set membership (see changes/mod.rs), so this stamping change leaves + // insert-vs-update classification unaffected. assert_eq!( - eve.2, v1, - "Lance merge_insert stamps new rows with created_at = dataset creation version, not commit version" + eve.2, v2, + "Lance merge_insert stamps new rows with created_at = commit version (lance#6774)" ); assert_eq!( eve.3, v2, @@ -258,11 +260,24 @@ async fn lance_merge_insert_update_preserves_created_at_version() { assert_eq!(alice.2, v1, "alice created_at should still be v1"); assert_eq!(alice.3, v1, "alice updated_at should still be v1"); - // Bob: updated via merge_insert - // created_at should be preserved (v1), updated_at should be bumped (v2) + // Bob: updated via merge_insert. eprintln!( "Bob: created_at={}, updated_at={}, v1={}, v2={}", bob.2, bob.3, v1, v2 ); assert_eq!(bob.1, 99, "bob's value should be updated to 99"); + // created_at is preserved across an UPDATE (lance#6774 only changed the + // INSERT-row stamping), which is what this test's name promises. + assert_eq!( + bob.2, v1, + "bob created_at must be preserved across a merge_insert UPDATE" + ); + // updated_at bumps to the commit version on UPDATE — the change-feed + // invariant OmniGraph's insert/update classification relies on + // (changes/mod.rs keys on _row_last_updated_at_version). If this regresses, + // the diff/change feed silently misses updates. + assert_eq!( + bob.3, v2, + "bob updated_at must bump to the commit version on a merge_insert UPDATE" + ); } diff --git a/crates/omnigraph/tests/literal_filters.rs b/crates/omnigraph/tests/literal_filters.rs index a0b2bd7..d486f28 100644 --- a/crates/omnigraph/tests/literal_filters.rs +++ b/crates/omnigraph/tests/literal_filters.rs @@ -19,6 +19,7 @@ node Metric { name: String @key score: F64? ratio: F32? + count: I32? active: Bool? born: Date? seen: DateTime? @@ -26,10 +27,10 @@ node Metric { "#; // Seeds partition every predicate, so a dropped filter returns all 4 rows. -const DATA: &str = r#"{"type":"Metric","data":{"name":"m1","score":2.5,"ratio":0.5,"active":true,"born":"2024-06-01","seen":"2024-06-01T12:00:00Z"}} -{"type":"Metric","data":{"name":"m2","score":1.0,"ratio":0.25,"active":false,"born":"2023-01-01","seen":"2023-01-01T00:00:00Z"}} -{"type":"Metric","data":{"name":"m3","score":3.0,"ratio":0.75,"active":true,"born":"2025-01-01","seen":"2025-01-01T00:00:00Z"}} -{"type":"Metric","data":{"name":"m4","score":0.5,"ratio":0.1,"active":false,"born":"2022-12-31","seen":"2022-01-01T00:00:00Z"}}"#; +const DATA: &str = r#"{"type":"Metric","data":{"name":"m1","score":2.5,"ratio":0.5,"count":1,"active":true,"born":"2024-06-01","seen":"2024-06-01T12:00:00Z"}} +{"type":"Metric","data":{"name":"m2","score":1.0,"ratio":0.25,"count":2,"active":false,"born":"2023-01-01","seen":"2023-01-01T00:00:00Z"}} +{"type":"Metric","data":{"name":"m3","score":3.0,"ratio":0.75,"count":3,"active":true,"born":"2025-01-01","seen":"2025-01-01T00:00:00Z"}} +{"type":"Metric","data":{"name":"m4","score":0.5,"ratio":0.1,"count":4,"active":false,"born":"2022-12-31","seen":"2022-01-01T00:00:00Z"}}"#; async fn metric_db(dir: &tempfile::TempDir) -> Omnigraph { let uri = dir.path().to_str().unwrap(); @@ -67,6 +68,50 @@ query inline() { match { $m: Metric { score: 3.0 } } return { $m.name } } assert_eq!(sorted_metric_names(&mut db, q, "inline").await, vec!["m3"]); } +// Inline-binding equality is the Lance-pushdown arm. With the literal coerced to +// the column's exact Arrow type, a narrow-numeric column (I32) and an F32 column +// must still select the right rows — the coercion changes the literal's type, not +// the result set. (The index-use win this enables is pinned at the Lance-surface +// layer by `lance_surface_guards::scalar_index_use_requires_matched_literal_type`.) +#[tokio::test] +async fn int_and_f32_literal_pushdown_coercion() { + let dir = tempfile::tempdir().unwrap(); + let mut db = metric_db(&dir).await; + let q = r#" +query count_eq() { match { $m: Metric { count: 2 } } return { $m.name } } +query ratio_eq() { match { $m: Metric { ratio: 0.25 } } return { $m.name } } +query count_ge() { match { $m: Metric $m.count >= 3 } return { $m.name } } +"#; + // I32 column, integer literal coerced Int64 -> Int32: count == 2 is m2 only. + assert_eq!(sorted_metric_names(&mut db, q, "count_eq").await, vec!["m2"]); + // F32 column, float literal coerced Float64 -> Float32: ratio == 0.25 is m2. + assert_eq!(sorted_metric_names(&mut db, q, "ratio_eq").await, vec!["m2"]); + // Range on the I32 column: count 3,4 >= 3 -> m3, m4 (coercion is op-independent). + assert_eq!( + sorted_metric_names(&mut db, q, "count_ge").await, + vec!["m3", "m4"] + ); +} + +// A fractional float against an integer column must not be truncated by the +// pushdown coercion (`2.7 -> 2` would wrongly match the count=2 row). The +// lossless guard falls back to the natural Float64 literal, so `count = 2.7` +// matches no integer and returns no rows. +#[tokio::test] +async fn fractional_float_equality_on_int_column_returns_no_rows() { + let dir = tempfile::tempdir().unwrap(); + let mut db = metric_db(&dir).await; + let q = r#" +query count_frac() { match { $m: Metric { count: 2.7 } } return { $m.name } } +"#; + assert!( + sorted_metric_names(&mut db, q, "count_frac") + .await + .is_empty(), + "count = 2.7 must match no integer rows (no truncation to count = 2)" + ); +} + #[tokio::test] async fn bool_literal_filters_execute() { let dir = tempfile::tempdir().unwrap(); @@ -88,9 +133,15 @@ async fn date_and_datetime_literal_filters_execute() { let q = r#" query born_ge() { match { $m: Metric $m.born >= date("2024-01-01") } return { $m.name } } query seen_lt() { match { $m: Metric $m.seen < datetime("2024-01-01T00:00:00Z") } return { $m.name } } +query born_eq() { match { $m: Metric { born: date("2024-06-01") } } return { $m.name } } +query seen_eq() { match { $m: Metric { seen: datetime("2024-06-01T12:00:00Z") } } return { $m.name } } "#; // born: m1 2024-06, m3 2025 >= 2024-01-01 assert_eq!(sorted_metric_names(&mut db, q, "born_ge").await, vec!["m1", "m3"]); // seen: m2 2023, m4 2022 < 2024-01-01 assert_eq!(sorted_metric_names(&mut db, q, "seen_lt").await, vec!["m2", "m4"]); + // Inline-binding equality exercises the Lance-pushdown arm with a typed + // Date32/Date64 literal: the epoch conversion must select exactly m1. + assert_eq!(sorted_metric_names(&mut db, q, "born_eq").await, vec!["m1"]); + assert_eq!(sorted_metric_names(&mut db, q, "seen_eq").await, vec!["m1"]); } diff --git a/crates/omnigraph/tests/maintenance.rs b/crates/omnigraph/tests/maintenance.rs index 13c9de7..78e31fa 100644 --- a/crates/omnigraph/tests/maintenance.rs +++ b/crates/omnigraph/tests/maintenance.rs @@ -14,9 +14,11 @@ use omnigraph::db::{ SkipReason, }; use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph::table_store::{IndexCoverage, TableStore}; use helpers::{ MUTATION_QUERIES, TEST_DATA, TEST_SCHEMA, count_rows, init_and_load, mixed_params, mutate_main, + snapshot_main, }; /// Filesystem URI of a node sub-table, mirroring the engine's layout @@ -131,6 +133,72 @@ async fn optimize_after_load_then_again_is_idempotent() { } } +// PR3 (Workstream B): an existing scalar index does not cover fragments +// appended after it was built (build_indices is existence-gated), so those +// rows are scanned unindexed. `optimize` must fold them back in via Lance's +// incremental `optimize_indices`, restoring full coverage. +#[tokio::test] +async fn optimize_reindexes_fragments_appended_after_index_build() { + const SCHEMA: &str = r#" +node Doc { + slug: String @key + rank: I32 @index +} +"#; + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, SCHEMA).await.unwrap(); + + // First load builds the id + rank BTREEs over the initial fragment. + load_jsonl( + &mut db, + "{\"type\":\"Doc\",\"data\":{\"slug\":\"d1\",\"rank\":1}}\n\ + {\"type\":\"Doc\",\"data\":{\"slug\":\"d2\",\"rank\":2}}", + LoadMode::Merge, + ) + .await + .unwrap(); + + // A second load with NEW keys appends a fragment the existing BTREEs do not + // cover (the existence gate skips re-building an index that already exists). + load_jsonl( + &mut db, + "{\"type\":\"Doc\",\"data\":{\"slug\":\"d3\",\"rank\":3}}\n\ + {\"type\":\"Doc\",\"data\":{\"slug\":\"d4\",\"rank\":4}}", + LoadMode::Merge, + ) + .await + .unwrap(); + + // Precondition: the appended fragment is unindexed. + { + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Doc").await.unwrap(); + assert!( + TableStore::has_unindexed_fragments(&ds).await.unwrap(), + "appended fragment should be unindexed before optimize" + ); + } + + db.optimize().await.unwrap(); + + // Postcondition: optimize_indices folded the appended fragment in, so every + // index covers every fragment and `rank` reports fully Indexed. + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Doc").await.unwrap(); + assert!( + !TableStore::has_unindexed_fragments(&ds).await.unwrap(), + "optimize must extend index coverage to all fragments" + ); + assert_eq!( + TableStore::key_column_index_coverage(&ds, "rank") + .await + .unwrap(), + IndexCoverage::Indexed, + "rank BTREE must cover all fragments after optimize" + ); +} + // Regression: `optimize` must not crash on a graph that has a `Blob` table. // // Lance `compact_files` forces `BlobHandling::AllBinary`, which mis-decodes @@ -775,3 +843,222 @@ async fn cleanup_reconciles_orphaned_branch_forks() { .await .unwrap(); } + +// cleanup must reclaim a manifest-unreferenced fork even when the BRANCH is +// still live (origin 2: an interrupted first-write fork), while KEEPING a table +// that is legitimately forked on that same live branch. Before the per-table +// authority broadening, the reconciler keyed only on the branch name and so +// never reclaimed a fork on a live branch — the wedge the handoff hit. +#[tokio::test] +async fn cleanup_reconciles_live_branch_orphan_fork_but_keeps_legitimate_fork() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let mut db = init_and_load(&dir).await; + + db.branch_create("feature").await.unwrap(); + + // Legitimately fork Company onto the live `feature` branch (a real write). + db.load_as( + "feature", + None, + r#"{"type":"Company","data":{"name":"Acme"}}"#, + LoadMode::Merge, + None, + ) + .await + .unwrap(); + + // Forge a manifest-unreferenced Person fork on the SAME live branch: the + // manifest's `feature` snapshot still places Person on main (Person was + // never written on feature), so this ref is an origin-2 orphan. + let person_uri = node_table_uri(&uri, "Person"); + { + let mut ds = Dataset::open(&person_uri).await.unwrap(); + let base = ds.version().version; + ds.create_branch("feature", base, None).await.unwrap(); + assert!( + ds.list_branches().await.unwrap().contains_key("feature"), + "precondition: forged orphan Person fork present on the live branch" + ); + } + + let company_uri = node_table_uri(&uri, "Company"); + let main_people = count_rows(&db, "node:Person").await; + let main_companies = count_rows(&db, "node:Company").await; + + db.cleanup(CleanupPolicyOptions { + keep_versions: Some(1), + older_than: None, + }) + .await + .unwrap(); + + // Origin-2 orphan reclaimed... + { + let ds = Dataset::open(&person_uri).await.unwrap(); + assert!( + !ds.list_branches().await.unwrap().contains_key("feature"), + "cleanup must reclaim the manifest-unreferenced Person fork on the live branch" + ); + } + // ...but the legitimate Company fork on the same live branch is kept. + { + let ds = Dataset::open(&company_uri).await.unwrap(); + assert!( + ds.list_branches().await.unwrap().contains_key("feature"), + "cleanup must NOT reclaim a legitimately-forked table on a live branch" + ); + } + // main is untouched. + assert_eq!(count_rows(&db, "node:Person").await, main_people); + assert_eq!(count_rows(&db, "node:Company").await, main_companies); +} + +// Regression (iss-848): a table with rows but NULL vectors (the load-before- +// embed window) must not abort index building. The vector (IVF) index cannot +// train on 0 vectors, so `create_vector_index` errors with "KMeans cannot +// train 1 centroids with 0 vectors". `build_indices_on_dataset_for_catalog` +// is the chokepoint every caller funnels through (load/mutate via +// prepare_updates_for_commit, ensure_indices, optimize, schema apply, merge), +// so per-index fault isolation there must defer that one column (pending) and +// still build the sibling scalar indexes, instead of propagating the error. +// This exercises both the load path (which builds indices inline) and the +// ensure_indices reconciler. Pre-fix this fails at the load step. +#[tokio::test] +async fn index_build_tolerates_null_vector_rows() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let schema = "node Doc {\n \ + slug: String @key\n \ + n: I64 @index\n \ + embedding: Vector(8)? @index\n\ + }\n"; + let mut db = Omnigraph::init(uri, schema).await.unwrap(); + // Rows present, embeddings null (loaded but not yet embedded). + load_jsonl( + &mut db, + "{\"type\":\"Doc\",\"data\":{\"slug\":\"d1\",\"n\":1}}\n\ + {\"type\":\"Doc\",\"data\":{\"slug\":\"d2\",\"n\":2}}", + LoadMode::Merge, + ) + .await + .expect("load rows with null embeddings"); + + // Must not abort: the untrainable vector column is deferred, the sibling + // BTREE on `n` still builds. + db.ensure_indices() + .await + .expect("ensure_indices must not abort when a vector column has no trainable vectors yet"); +} + +// iss-848: `optimize` converges declared-but-unbuilt indexes. After an @index is +// added post-data (a metadata-only apply that defers the physical build), the +// column is unindexed and reads scan. `optimize` — the operator's reconciler, +// run on a cron — must materialize it, by composing the ensure_indices +// reconciler after the compaction sweep. Pre-iss-848 optimize only maintained +// coverage of EXISTING indexes (optimize_indices) and never created missing ones. +#[tokio::test] +async fn optimize_materializes_index_declared_but_unbuilt() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let v1 = "node Doc {\n slug: String @key\n rank: I32\n}\n"; + let mut db = Omnigraph::init(uri, v1).await.unwrap(); + load_jsonl( + &mut db, + "{\"type\":\"Doc\",\"data\":{\"slug\":\"d1\",\"rank\":1}}\n\ + {\"type\":\"Doc\",\"data\":{\"slug\":\"d2\",\"rank\":2}}", + LoadMode::Merge, + ) + .await + .unwrap(); + + // Add @index on `rank` after data exists: a metadata-only apply that defers + // the physical build (iss-848), so the column is declared-indexed but unbuilt. + let v2 = "node Doc {\n slug: String @key\n rank: I32 @index\n}\n"; + db.apply_schema(v2).await.expect("index-only apply"); + + // Precondition: `rank` is declared @index but unbuilt -> reads degrade. + { + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Doc").await.unwrap(); + assert!( + matches!( + TableStore::key_column_index_coverage(&ds, "rank") + .await + .unwrap(), + IndexCoverage::Degraded { .. } + ), + "rank must be unindexed after the deferred apply" + ); + } + + db.optimize().await.unwrap(); + + // Postcondition: optimize's reconciler materialized the declared index. + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Doc").await.unwrap(); + assert_eq!( + TableStore::key_column_index_coverage(&ds, "rank") + .await + .unwrap(), + IndexCoverage::Indexed, + "optimize must build the declared-but-unbuilt rank index" + ); +} + +// iss-848 (PR review): the rename path also defers index building. A RenameType +// migration writes the renamed table as a new dataset with the existing rows +// but no indexes (its inline build was removed). optimize must then materialize +// the declared index on the renamed table. +#[tokio::test] +async fn optimize_materializes_index_after_type_rename() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let v1 = "node Doc {\n slug: String @key\n rank: I32 @index\n}\n"; + let mut db = Omnigraph::init(uri, v1).await.unwrap(); + load_jsonl( + &mut db, + "{\"type\":\"Doc\",\"data\":{\"slug\":\"d1\",\"rank\":1}}\n\ + {\"type\":\"Doc\",\"data\":{\"slug\":\"d2\",\"rank\":2}}", + LoadMode::Merge, + ) + .await + .unwrap(); + + // Rename Doc -> Item; rows are preserved on the new table key. + let v2 = "node Item @rename_from(\"Doc\") {\n slug: String @key\n rank: I32 @index\n}\n"; + let result = db.apply_schema(v2).await.expect("rename apply"); + assert!(result.applied); + assert_eq!( + count_rows(&db, "node:Item").await, + 2, + "rename must preserve rows" + ); + + // Post-rename the renamed table's declared rank index is unbuilt (deferred). + { + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Item").await.unwrap(); + assert!( + matches!( + TableStore::key_column_index_coverage(&ds, "rank") + .await + .unwrap(), + IndexCoverage::Degraded { .. } + ), + "rank must be unindexed immediately after the rename" + ); + } + + db.optimize().await.unwrap(); + + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Item").await.unwrap(); + assert_eq!( + TableStore::key_column_index_coverage(&ds, "rank") + .await + .unwrap(), + IndexCoverage::Indexed, + "optimize must build the renamed table's deferred rank index" + ); +} diff --git a/crates/omnigraph/tests/ordering.rs b/crates/omnigraph/tests/ordering.rs index 4e9296b..2684b1c 100644 --- a/crates/omnigraph/tests/ordering.rs +++ b/crates/omnigraph/tests/ordering.rs @@ -7,7 +7,7 @@ //! keys yield a TOTAL, deterministic order (and `ORDER … LIMIT` is //! deterministic). NULL placement is `nulls_first = !descending` (NULLs first //! under ASC, last under DESC). Both are documented in -//! `docs/user/query-language.md`. +//! `docs/user/queries/index.md`. mod helpers; diff --git a/crates/omnigraph/tests/scalar_indexes.rs b/crates/omnigraph/tests/scalar_indexes.rs new file mode 100644 index 0000000..8d8a3f0 --- /dev/null +++ b/crates/omnigraph/tests/scalar_indexes.rs @@ -0,0 +1,74 @@ +//! Coverage for `build_indices_on_dataset_for_catalog`'s per-property index +//! dispatch: which scalar/vector index each `@index`/`@key` column gets. +//! +//! The observable signal is `TableStore::key_column_index_coverage`, which +//! reports `Indexed` only when a BTREE covers the column (the same helper the +//! traversal chooser uses). Enums and orderable scalars must get a BTREE so +//! `=`/range/IN/IS NULL are index-accelerated; free-text Strings keep FTS +//! (which `key_column_index_coverage` does not count as a BTREE, by design). + +mod helpers; + +use omnigraph::db::Omnigraph; +use omnigraph::loader::{LoadMode, load_jsonl}; +use omnigraph::table_store::{IndexCoverage, TableStore}; + +use helpers::*; + +const SCHEMA: &str = r#" +node Item { + slug: String @key + status: enum(active, archived) @index + published: DateTime @index + rank: I32 @index + title: String @index + note: String? +} +"#; + +const DATA: &str = r#"{"type":"Item","data":{"slug":"a","status":"active","published":"2024-06-01T00:00:00Z","rank":1,"title":"alpha","note":"n1"}} +{"type":"Item","data":{"slug":"b","status":"archived","published":"2023-01-01T00:00:00Z","rank":2,"title":"beta","note":"n2"}} +{"type":"Item","data":{"slug":"c","status":"active","published":"2025-02-02T00:00:00Z","rank":3,"title":"gamma","note":"n3"}}"#; + +// Enums and orderable scalars (DateTime, numeric) get a BTREE from load's +// build-indices pass, so a `=`/range filter on them uses the index. Free-text +// String `@index` keeps FTS (no BTREE), and an un-annotated column has no +// scalar index — both report `Degraded`, which is the negative control that +// keeps this test from being vacuously green. +#[tokio::test] +async fn node_scalar_and_enum_index_columns_get_btree() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, SCHEMA).await.unwrap(); + load_jsonl(&mut db, DATA, LoadMode::Overwrite).await.unwrap(); + + let snap = snapshot_main(&db).await.unwrap(); + let ds = snap.open("node:Item").await.unwrap(); + + for col in ["status", "published", "rank"] { + let cov = TableStore::key_column_index_coverage(&ds, col).await.unwrap(); + assert_eq!( + cov, + IndexCoverage::Indexed, + "column '{col}' (enum/DateTime/numeric @index) must get a BTREE, got {cov:?}" + ); + } + + // Free-text String @index -> FTS, which is not a BTREE -> Degraded. + let title_cov = TableStore::key_column_index_coverage(&ds, "title") + .await + .unwrap(); + assert!( + matches!(title_cov, IndexCoverage::Degraded { .. }), + "free-text String @index should keep FTS (no BTREE), got {title_cov:?}" + ); + + // No @index annotation -> no scalar index at all -> Degraded. + let note_cov = TableStore::key_column_index_coverage(&ds, "note") + .await + .unwrap(); + assert!( + matches!(note_cov, IndexCoverage::Degraded { .. }), + "un-annotated column should have no scalar index, got {note_cov:?}" + ); +} diff --git a/crates/omnigraph/tests/schema_apply.rs b/crates/omnigraph/tests/schema_apply.rs index cc0cae2..508451a 100644 --- a/crates/omnigraph/tests/schema_apply.rs +++ b/crates/omnigraph/tests/schema_apply.rs @@ -736,3 +736,108 @@ edge Knows: Person -> Person { // current contract, the data is *unreachable* via omnigraph // (no manifest entry), which is the user-facing guarantee. } + +// Regression (bug 3 / dev-graph iss-848): a `Vector @index` on a 0-row table +// must not abort an otherwise-valid schema apply. A vector (IVF) index trains +// k-means centroids over the column's vectors, so Lance cannot build it on 0 +// vectors — it errors with "Creating empty vector indices with train=False is +// not yet implemented". When a *later* migration touches that table (here, an +// unrelated scalar `@index` on `body`), schema apply reconciles the table's +// whole index set, which previously tried to materialize the dormant vector +// index and aborted the entire migration (all-or-nothing). The build is now +// deferred (pending) when the column is untrainable, instead of failing the +// migration. The dormant index is materialized by a later `ensure_indices` / +// `optimize` once the table has rows. Full decoupling — intent recorded at +// apply, an async reconciler converges physical coverage — is iss-848. +#[tokio::test] +async fn apply_schema_defers_vector_index_on_empty_table() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + + // init does not build indices, so the declared-but-unbuilt vector index + // sits harmless on the empty table (this is how it survived earlier + // applies that never touched the table). + // `slug` is the user @key; omnigraph injects its own internal `id` column, + // so the key field must not be named `id`. + let v1 = "node Doc {\n \ + slug: String @key\n \ + body: String?\n \ + embedding: Vector(8) @index\n\ + }\n"; + let mut db = Omnigraph::init(uri, v1).await.unwrap(); + + // Add an *unrelated* scalar @index on `body`. This routes Doc through + // schema apply's index reconcile, which must NOT abort on the untrainable + // empty vector index. + let v2 = "node Doc {\n \ + slug: String @key\n \ + body: String? @index\n \ + embedding: Vector(8) @index\n\ + }\n"; + let result = db.apply_schema(v2).await.expect( + "schema apply must succeed: an empty-table vector @index is deferred, not fatal", + ); + assert!(result.applied, "the scalar @index change must apply"); + + // The deferred vector index is not dropped — once the table has a + // trainable vector, `ensure_indices` materializes it without error. (If + // the guard wrongly skipped a non-empty column, this would still be + // unindexed; if it wrongly tried to build on empty, the apply above would + // have failed.) + load_jsonl( + &mut db, + r#"{"type":"Doc","data":{"slug":"d1","body":"hello","embedding":[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]}}"#, + LoadMode::Merge, + ) + .await + .expect("loading a Doc with an embedding must succeed"); + db.ensure_indices() + .await + .expect("the deferred vector index must build once the table has a trainable vector"); +} + +// iss-848: adding an `@index` to an existing column is a pure metadata change. +// Schema apply records the intent (the catalog/IR now declares the index) but +// must NOT build the index inline, so the table's data and manifest version are +// untouched. The physical index is materialized later by ensure_indices / +// optimize. Pre-iss-848 the indexed_tables block built the index inline and +// bumped the table version. +#[tokio::test] +async fn index_only_constraint_apply_touches_no_table_data() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let v1 = "node Doc {\n slug: String @key\n n: I64\n}\n"; + let mut db = Omnigraph::init(uri, v1).await.unwrap(); + load_jsonl( + &mut db, + r#"{"type":"Doc","data":{"slug":"d1","n":1}}"#, + LoadMode::Merge, + ) + .await + .expect("load a Doc"); + + let before = db + .snapshot_of(ReadTarget::branch("main")) + .await + .unwrap() + .entry("node:Doc") + .unwrap() + .table_version; + + // Add an @index on the existing `n` column. + let v2 = "node Doc {\n slug: String @key\n n: I64 @index\n}\n"; + let result = db.apply_schema(v2).await.expect("index-only apply must succeed"); + assert!(result.applied, "the @index addition must apply"); + + let after = db + .snapshot_of(ReadTarget::branch("main")) + .await + .unwrap() + .entry("node:Doc") + .unwrap() + .table_version; + assert_eq!( + before, after, + "adding an @index must not bump the table version (no inline index build)" + ); +} diff --git a/crates/omnigraph/tests/search.rs b/crates/omnigraph/tests/search.rs index 480ec3c..425c51b 100644 --- a/crates/omnigraph/tests/search.rs +++ b/crates/omnigraph/tests/search.rs @@ -60,6 +60,15 @@ query hybrid_search_string($vq: String, $tq: String) { limit 3 } "#; +// Same shape as MOCK_SEARCH_SCHEMA but the vector records the model that +// produced its stored vectors, opting into the query-time same-space check. +const MODEL_RECORDED_SCHEMA: &str = r#" +node Doc { + slug: String @key + title: String @index + embedding: Vector(4) @embed("title", model="test-model-a") @index +} +"#; const SEARCH_MUTATIONS: &str = r#" query insert_doc($slug: String, $title: String, $body: String, $embedding: Vector(4)) { insert Doc { @@ -89,6 +98,15 @@ async fn init_mock_embedding_search_db(dir: &tempfile::TempDir) -> Omnigraph { db } +async fn init_model_recorded_search_db(dir: &tempfile::TempDir) -> Omnigraph { + let uri = dir.path().to_str().unwrap(); + let mut db = Omnigraph::init(uri, MODEL_RECORDED_SCHEMA).await.unwrap(); + load_jsonl(&mut db, &mock_embedding_seed_data(), LoadMode::Overwrite) + .await + .unwrap(); + db +} + fn mock_embedding_seed_data() -> String { [ ("alpha-doc", "alpha guide", mock_embedding("alpha", 4)), @@ -510,9 +528,14 @@ async fn explicit_vector_nearest_does_not_require_gemini_credentials() { #[tokio::test] #[serial] -async fn string_nearest_requires_gemini_credentials_when_mock_is_disabled() { +async fn string_nearest_requires_provider_credentials_when_mock_is_disabled() { + // With mock off and no provider key, the default (openai-compatible) + // provider fails loudly rather than silently producing garbage vectors. let _guard = EnvGuard::set(&[ ("OMNIGRAPH_EMBEDDINGS_MOCK", None), + ("OMNIGRAPH_EMBED_PROVIDER", None), + ("OPENROUTER_API_KEY", None), + ("OPENAI_API_KEY", None), ("GEMINI_API_KEY", None), ]); @@ -528,7 +551,105 @@ async fn string_nearest_requires_gemini_credentials_when_mock_is_disabled() { .await .unwrap_err(); - assert!(err.to_string().contains("GEMINI_API_KEY")); + assert!( + err.to_string() + .contains("OPENROUTER_API_KEY or OPENAI_API_KEY"), + "unexpected error: {err}" + ); +} + +#[tokio::test] +#[serial] +async fn nearest_string_passes_when_query_model_matches_recorded_model() { + let _guard = EnvGuard::set(&[ + ("OMNIGRAPH_EMBEDDINGS_MOCK", Some("1")), + ("OMNIGRAPH_EMBED_MODEL", Some("test-model-a")), + ("OMNIGRAPH_EMBED_PROVIDER", None), + ("OPENROUTER_API_KEY", None), + ("OPENAI_API_KEY", None), + ("GEMINI_API_KEY", None), + ]); + + let dir = tempfile::tempdir().unwrap(); + let mut db = init_model_recorded_search_db(&dir).await; + + let result = query_main( + &mut db, + MOCK_SEARCH_QUERIES, + "vector_search_string", + ¶ms(&[("$q", "alpha")]), + ) + .await + .unwrap(); + + assert_eq!(result_slugs(&result)[0], "alpha-doc"); +} + +#[tokio::test] +#[serial] +async fn nearest_string_errors_when_query_model_differs_from_recorded_model() { + let _guard = EnvGuard::set(&[ + ("OMNIGRAPH_EMBEDDINGS_MOCK", Some("1")), + ("OMNIGRAPH_EMBED_MODEL", Some("test-model-b")), + ("OMNIGRAPH_EMBED_PROVIDER", None), + ("OPENROUTER_API_KEY", None), + ("OPENAI_API_KEY", None), + ("GEMINI_API_KEY", None), + ]); + + let dir = tempfile::tempdir().unwrap(); + let mut db = init_model_recorded_search_db(&dir).await; + + let err = query_main( + &mut db, + MOCK_SEARCH_QUERIES, + "vector_search_string", + ¶ms(&[("$q", "alpha")]), + ) + .await + .unwrap_err(); + + // The error must name both the recorded model and the resolved one. + let msg = err.to_string(); + assert!(msg.contains("test-model-a"), "got: {msg}"); + assert!(msg.contains("test-model-b"), "got: {msg}"); +} + +#[tokio::test] +#[serial] +async fn injected_embedding_config_is_used_instead_of_env() { + // No mock flag and no provider keys in env, so `from_env()` would error. + // Injecting a Mock config proves the resolver uses the injected config + // (RFC-012 Phase 5), and its model satisfies the recorded same-space check. + let _guard = EnvGuard::set(&[ + ("OMNIGRAPH_EMBEDDINGS_MOCK", None), + ("OMNIGRAPH_EMBED_PROVIDER", None), + ("OMNIGRAPH_EMBED_MODEL", None), + ("OPENROUTER_API_KEY", None), + ("OPENAI_API_KEY", None), + ("GEMINI_API_KEY", None), + ]); + + let dir = tempfile::tempdir().unwrap(); + let mut db = init_model_recorded_search_db(&dir) + .await + .with_embedding_config(std::sync::Arc::new(omnigraph::embedding::EmbeddingConfig { + provider: omnigraph::embedding::Provider::Mock, + model: "test-model-a".to_string(), + base_url: String::new(), + api_key: String::new(), + })); + + let result = query_main( + &mut db, + MOCK_SEARCH_QUERIES, + "vector_search_string", + ¶ms(&[("$q", "alpha")]), + ) + .await + .unwrap(); + + assert_eq!(result_slugs(&result)[0], "alpha-doc"); } // ─── BM25 search ──────────────────────────────────────────────────────────── diff --git a/crates/omnigraph/tests/staged_writes.rs b/crates/omnigraph/tests/staged_writes.rs index 3771ad4..cf0e04c 100644 --- a/crates/omnigraph/tests/staged_writes.rs +++ b/crates/omnigraph/tests/staged_writes.rs @@ -1046,3 +1046,54 @@ async fn lance_restore_loses_to_concurrent_append_via_orphaning() { let v2_ids = collect_ids(&v2_batches); assert_eq!(v2_ids, vec!["alice".to_string(), "bob".to_string()]); } + +/// Regression for PR #229: `commit_staged` must skip Lance's per-commit +/// auto-cleanup hook. A graph created BEFORE the v7 bump (6.0.1 defaulted +/// `WriteParams::auto_cleanup` ON) carries `lance.auto_cleanup.*` config on its +/// datasets that `auto_cleanup = None` on new writes cannot retroactively clear; +/// Lance's hook fires off that *stored* config at commit time. Without the skip, +/// the engine's own writes would GC the versions `__manifest` pins for +/// snapshots/time-travel. (The substrate negative control — that the config +/// really does GC without the skip — lives in +/// `lance_surface_guards.rs::skip_auto_cleanup_suppresses_version_gc`.) +#[tokio::test] +async fn commit_staged_skips_auto_cleanup_so_pinned_versions_survive() { + use std::collections::HashMap; + + let dir = tempfile::tempdir().unwrap(); + let uri = format!("{}/people.lance", dir.path().to_str().unwrap()); + let store = TableStore::new(dir.path().to_str().unwrap()); + + let mut ds = TableStore::write_dataset(&uri, person_batch(&[("seed", Some(0))])) + .await + .unwrap(); + let v1 = ds.version().version; + + // Simulate a pre-bump dataset: aggressive legacy auto_cleanup config (fire on + // every commit, delete anything older than now). + let mut cfg = HashMap::new(); + cfg.insert("lance.auto_cleanup.interval".to_string(), "1".to_string()); + cfg.insert("lance.auto_cleanup.older_than".to_string(), "0ms".to_string()); + ds.update_config(cfg).await.unwrap(); + + // Several writes through the engine's staged commit path. + for i in 0..5i32 { + let name = format!("p{i}"); + let staged = store + .stage_append(&ds, person_batch(&[(name.as_str(), Some(i))]), &[]) + .await + .unwrap(); + ds = store + .commit_staged(Arc::new(ds.clone()), staged.transaction) + .await + .unwrap(); + } + + // `commit_staged` sets `with_skip_auto_cleanup(true)`, so the legacy config + // must NOT have GC'd the `__manifest`-pinned create version. + assert!( + ds.checkout_version(v1).await.is_ok(), + "commit_staged must skip Lance auto-cleanup so a pre-bump graph's pinned \ + v{v1} survives; it was GC'd" + ); +} diff --git a/crates/omnigraph/tests/writes.rs b/crates/omnigraph/tests/writes.rs index b006f4c..8120940 100644 --- a/crates/omnigraph/tests/writes.rs +++ b/crates/omnigraph/tests/writes.rs @@ -1540,3 +1540,109 @@ async fn second_sequential_update_on_same_row_succeeds() { "Alice's age must reflect the second update" ); } + +// An interrupted first-write fork (create_branch succeeded, the manifest +// publish did not) leaves a fully-formed Lance branch ref on the table that +// the manifest never references — a "manifest-unreferenced fork". The branch +// itself stays a valid manifest branch, so `cleanup`'s reconciler (keyed on +// the manifest branch list) never reclaims it. Today the next write to that +// table on that branch re-enters the fork path, `create_branch` collides, and +// the engine wedges with "incomplete prior delete; run `omnigraph cleanup`". +// +// We forge that exact residue (a live `feature` branch + a directly-created +// `feature` ref on the Person table the manifest doesn't reference) and assert +// the next write — via both `load` and `mutate` — self-heals by reclaiming the +// orphan fork and re-forking, rather than wedging. No process death / timing +// needed: the forge is the post-crash state. +#[tokio::test] +async fn first_write_self_heals_manifest_unreferenced_fork_on_live_branch() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let mut db = init_and_load(&dir).await; + db.branch_create("feature").await.unwrap(); + + // Forge the manifest-unreferenced fork directly at the Lance layer. + let person_uri = node_table_uri(&uri, "Person"); + { + let mut ds = lance::Dataset::open(&person_uri).await.unwrap(); + let base = ds.version().version; + ds.create_branch("feature", base, None).await.unwrap(); + assert!( + ds.list_branches().await.unwrap().contains_key("feature"), + "precondition: forged orphan fork present on Person" + ); + } + + // load → must self-heal, not wedge with "incomplete prior delete". + let row = r#"{"type":"Person","data":{"name":"Zoe","age":30}}"#; + db.load_as("feature", None, row, LoadMode::Merge, None) + .await + .expect("load onto a manifest-unreferenced fork must self-heal, not wedge"); + + // mutate → same path, must also self-heal. + mutate_branch( + &mut db, + "feature", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "Yan")], &[("$age", 41)]), + ) + .await + .expect("mutate onto a manifest-unreferenced fork must self-heal"); + + // The healed branch holds the new rows; main is untouched (still no Zoe/Yan). + let feature_people = count_rows_branch(&db, "feature", "node:Person").await; + let main_people = count_rows(&db, "node:Person").await; + assert!( + feature_people >= main_people + 2, + "feature must contain the two new rows on top of the inherited set \ + (feature={feature_people}, main={main_people})" + ); +} + +// A node delete cascades to every edge table touching that node, forking those +// edge tables during execution. The up-front fork-queue acquisition must cover +// those cascade-forked edges, not just the node table named in the IR — else +// commit_all's held-guard coverage check fails the write (and, before the +// coverage check was promoted out of debug-only, edge commits would slip +// through unserialized). This drives the new code via a DELETE (the only +// cascading op), on a branch, as the FIRST write (so it actually forks). +#[tokio::test] +async fn branch_cascade_delete_forks_node_and_edges_under_held_queues() { + let dir = tempfile::tempdir().unwrap(); + let mut db = init_and_load(&dir).await; + db.branch_create("feature").await.unwrap(); + + // Baseline inherited from main (Alice has 2 Knows + 1 WorksAt edge). + let main_people = count_rows(&db, "node:Person").await; + let main_knows = count_rows(&db, "edge:Knows").await; + + // First write to `feature` is `delete Person Alice`, whose cascade forks + // node:Person AND edge:Knows + edge:WorksAt. Pre-fix the up-front set held + // only node:Person, so commit_all's coverage check rejected the write. + mutate_branch( + &mut db, + "feature", + MUTATION_QUERIES, + "remove_person", + &mixed_params(&[("$name", "Alice")], &[]), + ) + .await + .expect("branch cascade-delete must hold queues for cascade-forked edge tables"); + + // Alice and her edges are gone on feature; main is untouched. + assert_eq!( + count_rows_branch(&db, "feature", "node:Person").await, + main_people - 1, + "feature should have Alice removed from the inherited set" + ); + assert!( + count_rows_branch(&db, "feature", "edge:Knows").await < main_knows, + "feature should have Alice's cascade-deleted Knows edges removed" + ); + assert_eq!( + count_rows(&db, "node:Person").await, + main_people, + "main must be untouched by the branch delete" + ); +} diff --git a/docs/dev/architecture.md b/docs/dev/architecture.md index 9d31545..004a98a 100644 --- a/docs/dev/architecture.md +++ b/docs/dev/architecture.md @@ -1,6 +1,6 @@ # Architecture -OmniGraph is a typed property-graph engine built as a coordination layer over many Lance datasets, with Git-style branches and commits across the whole graph, multi-modal querying (vector + FTS + BM25 + RRF + graph traversal) in one runtime, an HTTP server with Cedar policy, and a CLI driven by a single `omnigraph.yaml`. +OmniGraph is a typed property-graph engine built as a coordination layer over many Lance datasets, with Git-style branches and commits across the whole graph, multi-modal querying (vector + FTS + BM25 + RRF + graph traversal) in one runtime, an HTTP server with Cedar policy, and a CLI driven by a per-operator `~/.omnigraph/config.yaml` plus team-owned cluster directories. ## Reading guide @@ -10,7 +10,7 @@ Three views, increasing zoom: 2. **Layer view** — the eight-layer stack inside one OmniGraph process. 3. **Component zoom-ins** — what's inside each layer. -For runtime flows (read query, mutation), see [`docs/dev/execution.md`](execution.md). For the on-disk layout of a graph, see [`docs/user/storage.md`](../user/storage.md). +For runtime flows (read query, mutation), see [`docs/dev/execution.md`](execution.md). For the on-disk layout of a graph, see [`docs/user/storage.md`](../user/concepts/storage.md). L1 (orange in the diagrams) is what we inherit from Lance; L2 (blue) is what OmniGraph adds. The L1/L2 framing is also called out in prose at the bottom of this doc. @@ -280,7 +280,7 @@ flowchart LR eng --> wq ``` -The server applies Cedar policy at the HTTP boundary today. The roadmap, called out in [docs/dev/invariants.md](invariants.md) as a known gap, is to push policy into the planner as predicates. After Cedar, mutating handlers go through `WorkloadController` (per-actor admission cap + byte budget; PR 2 / MR-686) before reaching the engine. The engine itself holds an `Arc` so concurrent mutations on the same `(table, branch)` serialize at the queue, while disjoint keys run in parallel — see [docs/user/server.md](../user/server.md) "Per-actor admission control" and [docs/dev/writes.md](writes.md). The CLI bypasses the HTTP layer (and admission) and calls the engine API directly. +The server applies Cedar policy at the HTTP boundary today. The roadmap, called out in [docs/dev/invariants.md](invariants.md) as a known gap, is to push policy into the planner as predicates. After Cedar, mutating handlers go through `WorkloadController` (per-actor admission cap + byte budget; PR 2 / MR-686) before reaching the engine. The engine itself holds an `Arc` so concurrent mutations on the same `(table, branch)` serialize at the queue, while disjoint keys run in parallel — see [docs/user/server.md](../user/operations/server.md) "Per-actor admission control" and [docs/dev/writes.md](writes.md). The CLI bypasses the HTTP layer (and admission) and calls the engine API directly. Code paths: diff --git a/docs/dev/branch-protection.md b/docs/dev/branch-protection.md index 2b6cc37..1d1c094 100644 --- a/docs/dev/branch-protection.md +++ b/docs/dev/branch-protection.md @@ -8,7 +8,7 @@ This page explains what the policy says and how to change it. | Setting | Value | Why | |---|---|---| -| **Required status checks (strict)** | `Classify Changes`, `Check AGENTS.md Links`, `Test Workspace`, `Test omnigraph-server --features aws`, `CODEOWNERS matches source`, `CODEOWNERS not hand-edited` | Every PR must pass workspace tests, AGENTS.md link integrity, and the CODEOWNERS hygiene checks. The two CODEOWNERS contexts must equal the job `name:` values in `.github/workflows/codeowners.yml` **verbatim** — a context naming a job that never reports (the old `CODEOWNERS / drift` used the job *id*, and the job was path-filtered) leaves every PR permanently pending and forces admin overrides. `strict: true` requires the branch to be up-to-date with `main` before merge. | +| **Required status checks (strict)** | `Classify Changes`, `Check AGENTS.md Links`, `Test omnigraph-server --features aws`, `CODEOWNERS matches source`, `CODEOWNERS not hand-edited` | Every PR must pass the AWS-feature build/test, AGENTS.md link integrity, and the CODEOWNERS hygiene checks. **`Test Workspace` is deliberately NOT required** — it runs only on push to `main` (post-merge), tags, and manual `workflow_dispatch`, to keep PR turnaround fast (it was the ~15min+ slow gate). It is therefore *not* listed here: a required check that never reports on PRs (the `test` job is `if: github.event_name != 'pull_request'`) would leave every PR permanently pending — the same job-never-reports trap the CODEOWNERS contexts call out below. The trade-off (a regression lands on `main` and is caught by the post-merge run, so `main` can briefly go red) and its mitigations are documented in [ci.md](ci.md). The two CODEOWNERS contexts must equal the job `name:` values in `.github/workflows/codeowners.yml` **verbatim** — a context naming a job that never reports (the old `CODEOWNERS / drift` used the job *id*, and the job was path-filtered) leaves every PR permanently pending and forces admin overrides. `strict: true` requires the branch to be up-to-date with `main` before merge. | | **Required approving reviews** | `1` | At least one reviewer. With a 2-person team, going higher would block all merges when one person is unavailable. | | **Require code-owner reviews** | `true` | The reviewer must be a code owner per `.github/CODEOWNERS`. This is what makes the codeowners chassis enforced. | | **Dismiss stale reviews on new commits** | `true` | A push after approval invalidates the prior review. Prevents the "approve, then sneak in unreviewed changes" pattern. | diff --git a/docs/dev/ci.md b/docs/dev/ci.md index 1124cb4..2e80f40 100644 --- a/docs/dev/ci.md +++ b/docs/dev/ci.md @@ -3,6 +3,9 @@ `.github/workflows/`: - **ci.yml**: text-only changes skip; otherwise `cargo test --workspace --locked` on ubuntu-latest with protobuf compiler. OpenAPI-drift check that auto-commits the regenerated `openapi.json` for same-repository PRs. Also runs the AGENTS.md cross-link integrity check (`scripts/check-agents-md.sh`). + - **`Test Workspace` does not run on pull requests.** The job is gated `if: github.event_name != 'pull_request'`, so the full workspace + failpoints suite runs only on push to `main` (post-merge), on `v*` tags, and on manual `workflow_dispatch`. This was a deliberate PR-latency trade-off — it was the slowest gate (~15min warm, up to the 75min cold ceiling). `RustFS S3 Integration` `needs: test`, so it is push-/dispatch-only for the same reason. The fast PR gates remain: `Classify Changes`, `Check AGENTS.md Links`, `Test omnigraph-server --features aws`, and the two CODEOWNERS checks. `Test Workspace` is correspondingly **not** in the required-check list (`.github/branch-protection.json`); see [branch-protection.md](branch-protection.md). + - **Consequences to internalize:** (1) a regression that the suite would catch now lands on `main` and turns the post-merge run red, rather than being blocked pre-merge — `main` can briefly break, so run `cargo test --workspace --locked` locally before merging anything non-trivial, or trigger this workflow on your branch via the Actions "Run workflow" button. (2) `openapi.json` is no longer auto-regenerated on PRs (that step is inside the `test` job); for server/API changes, regenerate it locally with `OMNIGRAPH_UPDATE_OPENAPI=1 cargo test -p omnigraph-server --test openapi` and commit it, or the strict drift check fails the post-merge `main` run. + - **Applying this policy:** removing `Test Workspace` from the JSON is inert until an admin runs `./scripts/apply-branch-protection.sh`. **Run it immediately after this change merges** — until then GitHub still requires a `Test Workspace` context that no longer reports on PRs, which leaves every open PR permanently pending (the job-never-reports trap). - **AWS feature build job**: `cargo build/test -p omnigraph-server --features aws` on ubuntu-latest. - **Windows binary build job**: `cargo build --release --locked -p omnigraph-cli -p omnigraph-server` on windows-latest with smoke checks for `omnigraph.exe version`, `omnigraph-server.exe --help`, and PowerShell installer syntax. - **RustFS S3 integration**: spins up RustFS in Docker, runs `s3_storage`, `server_opens_s3_graph_directly_and_serves_snapshot_and_read`, and `local_cli_s3_end_to_end_init_load_read_flow`. diff --git a/docs/dev/cluster-config-specs.md b/docs/dev/cluster-config-specs.md index d248be2..b9dfde8 100644 --- a/docs/dev/cluster-config-specs.md +++ b/docs/dev/cluster-config-specs.md @@ -3,11 +3,11 @@ **Status:** Draft / thinking-in-progress **Type:** Architecture direction **Date:** 2026-06-07 -**Relationship:** generalizes today's `omnigraph.yaml` graph/query/policy configuration surface ([CLI reference](../user/cli-reference.md), [server docs](../user/server.md)) into a future cluster control plane. The distilled rules are in [cluster-axioms.md](cluster-axioms.md); detailed downstream implementation spec and blast-radius assessment in [cluster-config-implementation-spec.md](cluster-config-implementation-spec.md). This is a proposed architecture, not an implemented RFC. +**Relationship:** generalizes today's `omnigraph.yaml` graph/query/policy configuration surface ([CLI reference](../user/cli/reference.md), [server docs](../user/operations/server.md)) into a future cluster control plane. The distilled rules are in [cluster-axioms.md](cluster-axioms.md); detailed downstream implementation spec and blast-radius assessment in [cluster-config-implementation-spec.md](cluster-config-implementation-spec.md). This is a proposed architecture, not an implemented RFC. > **Implementation status.** The examples below describe the full target schema. > Stage 2B only accepts the read-only subset documented in -> [cluster-config.md](../user/cluster-config.md). Future-phase fields such as +> [cluster-config.md](../user/clusters/config.md). Future-phase fields such as > `env_file`, `apply`, `providers`, `pipelines`, `embeddings`, `ui`, `aliases`, > and `bindings` are intentionally rejected with typed diagnostics until their > reconciler semantics are implemented. diff --git a/docs/dev/execution.md b/docs/dev/execution.md index 0e8e3fc..e9ac9eb 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -177,4 +177,4 @@ For all three modes, a mid-load failure (RI / cardinality violation, validation ## Embeddings during load -If a node type has `@embed` properties, the loader calls the engine embedding client (Gemini, RETRIEVAL_DOCUMENT) per row to populate the vector column. See [embeddings.md](../user/embeddings.md). +The loader does **not** embed `@embed` properties at load time. `@embed` is a catalog annotation consumed by query typecheck/lint; vectors are supplied directly in the load data, or pre-filled by the offline `omnigraph embed` pipeline. Query-time `nearest($v, "string")` auto-embeds the query string via the provider-independent embedding client. See [embeddings.md](../user/search/embeddings.md). (Ingest-time `@embed` execution is a planned RFC-012 phase.) diff --git a/docs/dev/index.md b/docs/dev/index.md index b1dc4fb..a0a6afb 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -20,13 +20,13 @@ constraints. User-facing behavior should still be documented through | Area | Read | |---|---| | System structure, L1/L2 framing, component diagrams | [architecture.md](architecture.md) | -| On-disk layout, manifest schema, URI behavior | [storage.md](../user/storage.md) | +| On-disk layout, manifest schema, URI behavior | [storage.md](../user/concepts/storage.md) | | Direct-publish writes, D2, staged writes, recovery sidecars | [writes.md](writes.md) | | Query execution, mutation execution, loader flow | [execution.md](execution.md) | -| Index lifecycle and graph topology indexes | [indexes.md](../user/indexes.md) | -| Branch and commit internals | [branches-commits.md](../user/branches-commits.md) | +| Index lifecycle and graph topology indexes | [indexes.md](../user/search/indexes.md) | +| Branch and commit internals | [branches-commits.md](../user/branching/index.md) | | Three-way merge implementation and conflicts | [merge.md](merge.md) | -| Diff/change-feed implementation | [changes.md](../user/changes.md) | +| Diff/change-feed implementation | [changes.md](../user/branching/changes.md) | | Branch protection policy | [branch-protection.md](branch-protection.md) | | CODEOWNERS source of truth | [codeowners.md](codeowners.md) | @@ -34,14 +34,14 @@ constraints. User-facing behavior should still be documented through | Area | Read | |---|---| -| Schema grammar, catalog, migration planner | [schema-language.md](../user/schema-language.md) | -| Query grammar, IR, lints, mutation restrictions | [query-language.md](../user/query-language.md) | -| Embedding client and `@embed` integration | [embeddings.md](../user/embeddings.md) | -| Cedar policy surface and server gating | [policy.md](../user/policy.md) | -| Server auth, OpenAPI, endpoint handlers | [server.md](../user/server.md) | -| Error taxonomy and serialization | [errors.md](../user/errors.md) | -| Constants and tunables | [constants.md](../user/constants.md) | -| Transaction model public contract | [transactions.md](../user/transactions.md) | +| Schema grammar, catalog, migration planner | [schema-language.md](../user/schema/index.md) | +| Query grammar, IR, lints, mutation restrictions | [query-language.md](../user/queries/index.md) | +| Embedding client and `@embed` integration | [embeddings.md](../user/search/embeddings.md) | +| Cedar policy surface and server gating | [policy.md](../user/operations/policy.md) | +| Server auth, OpenAPI, endpoint handlers | [server.md](../user/operations/server.md) | +| Error taxonomy and serialization | [errors.md](../user/operations/errors.md) | +| Constants and tunables | [constants.md](../user/reference/constants.md) | +| Transaction model public contract | [transactions.md](../user/branching/transactions.md) | ## Project Operations @@ -79,6 +79,9 @@ Working documents for in-flight feature work. Removed when the work lands. | Per-operator config — `~/.omnigraph/` identity, keyed credentials, named servers (the operator slice of RFC-002) | [rfc-007-operator-config.md](rfc-007-operator-config.md) | | Deprecate `omnigraph.yaml` — one concern per config surface; key-by-key migration map and staged retirement | [rfc-008-deprecate-omnigraph-yaml.md](rfc-008-deprecate-omnigraph-yaml.md) | | Unify CLI embedded/remote access paths — parity referee, shared wire-DTO crate, `GraphClient` trait, declared plane capabilities | [rfc-009-unify-access-paths.md](rfc-009-unify-access-paths.md) | +| Restructure the CLI around explicit planes — one graph-addressing model, declared capability surface, plane-grouped help (expands RFC-009 Phase 4) | [rfc-010-cli-planes-restructure.md](rfc-010-cli-planes-restructure.md) | +| CLI refactoring — one addressing & config model post-`omnigraph.yaml`: scope + `--graph` + derived access path, served-default / privileged-direct, profiles, named queries, capability classifier (completes RFC-008) | [rfc-011-cli-refactoring.md](rfc-011-cli-refactoring.md) | +| Provider-independent embedding configuration — one resolved `EmbeddingConfig` + sealed provider enum (Gemini/OpenAI/Mock), identity recorded in the schema IR, query-time same-space validation, NFR floor | [rfc-012-embedding-provider-config.md](rfc-012-embedding-provider-config.md) | ## Boundary diff --git a/docs/dev/invariants.md b/docs/dev/invariants.md index b3bcfaf..2fa87d1 100644 --- a/docs/dev/invariants.md +++ b/docs/dev/invariants.md @@ -15,6 +15,38 @@ Use it this way: - Keep implementation ledgers, roadmap detail, and historical MR notes in the per-area docs. This file is the filter, not the encyclopedia. +## Governing principle: logical contract over physical state + +The hard invariants below are instances of one rule. Keep it in view whenever +a change touches the boundary between what the graph *means* and how it is +physically stored. + +> **Logical state is the contract. Physical state — index coverage, fragment +> layout, compaction versions, staged writes — is derived, rebuildable, and may +> be produced asynchronously. A physical operation must never fail a logical +> one. Preconditions are checked against logical state; physical reconciliation +> is idempotent and may lag or retry. Genuine logical conflicts still fail +> loudly: the licence to lag covers physical convergence, not correctness.** + +Invariants that instantiate it: **2** (manifest-atomic visibility) and **5** +(recovery is part of the commit protocol) — a partially-written physical layer +never changes what a graph commit means; **7** (indexes are derived state) — a +query is correct under partial index coverage, and expensive index work +converges from manifest state instead of gating the write path; **13** (failures +bounded and observable) — the licence to lag is not a licence to drop, so a +physical step that cannot make progress is surfaced, not swallowed. Deny-list +items that enforce it: synchronous inline vector/FTS index rebuilds on the +commit path; state that drifts from Lance or the manifest when it can be +derived; job queues for manifest-derivable state where a reconciler fits. + +The failure shape it rules out: a legitimate background operation on the +physical layer (compaction, an index build, an interrupted staged write) is +allowed to break a logical operation (a query's correctness, a migration's +success, a branch's writability). The smell to watch for is a logical operation +whose precondition is a *physical* fact — a cached file version, an index's +existence, a fragment count. Make the precondition logical and let a reconciler +converge the physical state. + ## Hard Invariants 1. **Respect the substrate.** Lance owns columnar storage, per-dataset @@ -58,7 +90,7 @@ Use it this way: branch they read even when index coverage is partial. Expensive index work should converge from manifest state instead of extending the critical write path. Scalar staged index builds and vector inline residuals are documented - in [writes.md](writes.md) and [indexes.md](../user/indexes.md). + in [writes.md](writes.md) and [indexes.md](../user/search/indexes.md). 8. **Schema identity survives renames.** Accepted schema identity must remain stable across type and property renames. Rename support belongs in migration @@ -100,14 +132,14 @@ Use it this way: |---|---|---| | Multi-table commit | Manifest CAS plus recovery sidecars; not a single Lance primitive | [writes.md](writes.md), [architecture.md](architecture.md) | | Constructive mutations | In-memory `MutationStaging`, one end-of-query table commit per touched table, then one manifest publish | [writes.md](writes.md), [execution.md](execution.md) | -| Deletes | Inline-commit residual; delete-only queries allowed, mixed insert/update/delete rejected by D2 | [query-language.md](../user/query-language.md), [writes.md](writes.md) | -| Branch delete | Manifest is the single authority, flipped atomically first; per-table forks + commit-graph branch are derived state, reclaimed best-effort (`force_delete_branch`) with the `cleanup` reconciler as the guaranteed backstop. Reusing a name whose reclaim failed before `cleanup` surfaces an actionable error | [branches-commits.md](../user/branches-commits.md), [maintenance.md](../user/maintenance.md) | -| Schema validation | Type checks, required fields, defaults, edge endpoint checks, and edge cardinality are enforced on write paths | [schema-language.md](../user/schema-language.md), [execution.md](execution.md) | -| Unique constraints | Intra-batch and write-path checks exist; intake and branch-merge derive the composite key through one shared function (`loader::composite_unique_key`, a separator-free `Vec` tuple) and fail loudly on an un-keyable column type rather than silently exempting it; full cross-version uniqueness against already-committed rows is still a gap | [schema-language.md](../user/schema-language.md) | +| Deletes | Inline-commit residual; delete-only queries allowed, mixed insert/update/delete rejected by D2 | [query-language.md](../user/queries/index.md), [writes.md](writes.md) | +| Branch delete | Manifest is the single authority, flipped atomically first; per-table forks + commit-graph branch are derived state, reclaimed best-effort (`force_delete_branch`) with the `cleanup` reconciler as the guaranteed backstop. Reusing a name whose reclaim failed before `cleanup` surfaces an actionable error | [branches-commits.md](../user/branching/index.md), [maintenance.md](../user/operations/maintenance.md) | +| Schema validation | Type checks, required fields, defaults, edge endpoint checks, and edge cardinality are enforced on write paths | [schema-language.md](../user/schema/index.md), [execution.md](execution.md) | +| Unique constraints | Intra-batch and write-path checks exist; intake and branch-merge derive the composite key through one shared function (`loader::composite_unique_key`, a separator-free `Vec` tuple) and fail loudly on an un-keyable column type rather than silently exempting it; full cross-version uniqueness against already-committed rows is still a gap | [schema-language.md](../user/schema/index.md) | | Storage trait | `TableStorage` (via `db.storage()`) is staged-only; the inline-commit residuals (`delete_where`, `create_vector_index`) are split onto a separate sealed `InlineCommitResidual` trait reached via `db.storage_inline_residual()` (MR-854), so §1 holds by construction; capability/stat surfaces are roadmap | [writes.md](writes.md), [architecture.md](architecture.md) | -| Index lifecycle | `ensure_indices` is explicit today; reconciler-based convergence is roadmap | [indexes.md](../user/indexes.md), [maintenance.md](../user/maintenance.md) | -| Traversal IDs | Runtime still builds `TypeIndex`; Lance stable row-id based graph IDs are roadmap | [architecture.md](architecture.md), [query-language.md](../user/query-language.md) | -| Auth | Bearer token hashing and server-side actor resolution are implemented at the HTTP boundary | [server.md](../user/server.md), [policy.md](../user/policy.md) | +| Index lifecycle | `@index`/`@key` declares *intent*; the physical index is derived state and never fails a logical op. `schema apply` builds no indexes (records intent only; index-only changes touch no table data). `load`/`mutate` build inline through one chokepoint (`build_indices_on_dataset_for_catalog`, type-dispatched by `node_prop_index_kind`: enum + orderable scalar → BTREE, free-text String → FTS, Vector → vector) that fault-isolates an untrainable Vector column into a *pending* index instead of aborting. `optimize`/`ensure_indices` is the reconciler: it creates declared-but-missing indexes and folds appended/rewritten fragments into existing ones (`optimize_indices`), reporting still-pending columns. Explicit maintenance call, not yet a background loop | [indexes.md](../user/search/indexes.md), [maintenance.md](../user/operations/maintenance.md) | +| Traversal IDs | Runtime still builds `TypeIndex`; Lance stable row-id based graph IDs are roadmap | [architecture.md](architecture.md), [query-language.md](../user/queries/index.md) | +| Auth | Bearer token hashing and server-side actor resolution are implemented at the HTTP boundary | [server.md](../user/operations/server.md), [policy.md](../user/operations/policy.md) | | Tests | Tempdir-backed Lance tests are the current substrate; the storage adapter has an in-memory backend for adapter-level contract tests, but Lance datasets bypass it | [testing.md](testing.md) | The branch-delete reconciler is authority-derived: it reclaims orphaned forks @@ -132,13 +164,18 @@ them explicit. new writer cannot couple a write with a HEAD advance through the default surface. The dead legacy methods (`append_batch` on the trait, `merge_insert_batch{,es}`, `create_{btree,inverted}_index`) were removed. The - remaining residuals are `delete_where` (gated on MR-A — Lance v7.x bump) - and `create_vector_index` (gated on Lance #6666); see - [lance.md](lance.md) and [writes.md](writes.md). New write paths should use - the staged shape unless a documented Lance blocker applies. + remaining residuals are `delete_where` and `create_vector_index`. The Lance + 6.0.1 → 7.0.0 bump landed, so the staged two-phase delete API + (`DeleteBuilder::execute_uncommitted`, Lance #6658) is now available and MR-A + is unblocked — but the migration itself is still pending, so `delete_where` + stays inline for now. `create_vector_index` remains gated on Lance #6666 + (still open). See [lance.md](lance.md) and [writes.md](writes.md). New write + paths should use the staged shape unless a documented Lance blocker applies. - **Deletes and vector indexes:** `delete_where` and vector index creation still - advance Lance HEAD inline because the required public Lance APIs are missing. - Keep D2 and recovery coverage in place until those residuals are removed. + advance Lance HEAD inline. The public delete two-phase API now exists (Lance + #6658 shipped in 7.0.0), so the delete residual is unblocked pending the MR-A + migration; vector index creation is still blocked (Lance #6666 open). Keep D2 + and recovery coverage in place until those residuals are removed. - **Blob-column compaction:** Lance `compact_files` mis-decodes blob-v2 columns under its forced `BlobHandling::AllBinary` read ("more fields in the schema than provided column indices"), so `optimize` skips any table with a `Blob` @@ -160,6 +197,22 @@ them explicit. one-winner-CAS territory; closing this fully needs a cross-process serialization primitive (e.g. lease-based use of the schema-apply lock branch) — design it before promoting multi-process write topologies. +- **Fork reclaim is in-process-safe only:** the first write to a table on a + branch forks it (a Lance `create_branch` that advances state before the + manifest publish). An interrupted fork (crash, or a cancelled request + future) leaves a manifest-unreferenced branch ref. The next write self-heals + it — `reclaim_orphaned_fork_and_refork` (`force_delete_branch` + re-fork) + — but reclaim is only safe because the writer holds the per-`(table, + branch)` write queue from before the fork through the publish AND re-checks + the live manifest under it, so no *in-process* writer can be mid-fork. A + reclaim cannot serialize against a foreign-*process* in-flight fork: it may + force-delete a peer's just-created ref, which makes that peer's commit fail + and retry — the same one-winner-CAS exposure as above, not corruption. The + reclaim never fires unless in-process-queue + manifest authority both prove + the ref is manifest-unreferenced. `cleanup`'s per-table reconciler + (`reconcile_orphaned_branches`) is the guaranteed backstop for any fork the + write path never revisits. Both degrade to a no-op if Lance ships an atomic + multi-dataset branch op. - **Local `write_text_if_match` is not a cross-process CAS:** object-store backends use a true conditional put (ETag If-Match; the in-memory test backend too), but upstream `object_store` leaves `PutMode::Update` diff --git a/docs/dev/lance.md b/docs/dev/lance.md index a4e311f..4c624b3 100644 --- a/docs/dev/lance.md +++ b/docs/dev/lance.md @@ -156,7 +156,24 @@ If a future need pulls one of these into scope, add a row to the matching domain When Lance ships a major release that changes any of the above (file format bump, new index type, transaction semantics change, new branching primitive), refresh this index in the same change as the omnigraph upgrade. Stale Lance pointers are worse than no pointers. -### Last alignment audit: 2026-05-22 (Lance 6.0.1 upstream; omnigraph pinned at 6.0.1) +### Last alignment audit: 2026-06-15 (Lance 7.0.0 upstream; omnigraph pinned at 7.0.0) + +Migration from Lance 6.0.1 → 7.0.0 landed in this cycle. **Arrow stayed 58, DataFusion stayed 53** (no change) — the only transitive bump is `object_store` 0.12.5 → 0.13.2. 141 upstream commits reviewed (6.0.1 → 7.0.0); no fixes lost (the 6.0.x release-branch backports are all forward-ported into 7.0.0). Behavior-affecting findings: + +- **object_store 0.13 moved convenience methods behind a new `ObjectStoreExt` trait** (`get`/`put`/`head`/`rename`/`delete`; `list`/`list_with_delimiter`/`put_opts` stay on the core `ObjectStore` trait). Fix = add `use object_store::ObjectStoreExt;` to `storage.rs` and `db/manifest/namespace.rs`; no call-site changes. Mirrors Lance's own migration in PR #6672. The local-FS `PutMode::Update` gap is unchanged (still unimplemented upstream), so `storage.rs::write_text_if_match`'s local content-token emulation stays. +- **`roaring` must be pinned to 0.11.4** (`cargo update -p roaring --precise 0.11.4`). Lance 7.0.0's `UpdatedFragmentOffsets` newtype (PR #6650) derives `Eq` over `HashMap`, which needs `RoaringBitmap: Eq` — added only in roaring 0.11.4 (roaring-rs PR #341). Lance's loose `roaring = "0.11"` constraint otherwise resolves the broken 0.11.3 and **lance itself fails to compile** (`RoaringBitmap: Eq is not satisfied`). roaring is transitive (no direct workspace dep); the pin lives only in `Cargo.lock`. +- **`_row_created_at_version` for merge-insert INSERT rows now = the commit version** (PR #6774; was a fallback of 1 / dataset-creation version). Flipped `lance_version_columns.rs::lance_merge_insert_new_row_stamps_created_at_version` to assert `== v2`. Production change-detection keys on `_row_last_updated_at_version` + ID-set membership, so classification logic is unaffected (the `changes/mod.rs` rationale comment was corrected). +- **BTREE range-query bound inclusiveness fixed** (PR #6796, issue #6792): `x <= hi AND x > lo` returned the wrong boundary row on 6.0.1. omnigraph today builds BTREE only on string `@key` columns (`id`/`src`/`dst`) and queries them by equality/IN, not range, so its *current* query patterns almost certainly never hit this bug — but the corrected boundary semantics are a contract we rely on the moment a BTREE-range path appears (BTREE-on-properties via the index-type tickets, or a range-on-key query). Pinned by `lance_surface_guards.rs::btree_range_query_boundary_is_correct` (reproduces #6792's 5-row + BTREE shape). +- **`WriteParams::auto_cleanup` default flipped from on (every-20-commits) to `None`** (PR #6755). On 6.0.1 the on-by-default hook could GC versions the `__manifest` pins for snapshots/time-travel. omnigraph owns cleanup explicitly (`optimize.rs::cleanup_all_tables`). Two parts to the fix, because `auto_cleanup` is **create-time config only and has no effect on existing datasets** (Lance `write.rs` docs): (1) `auto_cleanup: None` at all 11 `WriteParams` sites so *new* datasets store no cleanup config; (2) — the load-bearing half — `skip_auto_cleanup: true` on every commit path, because graphs created **before** the bump still carry the on-config in their datasets, and Lance's hook fires off the *dataset's stored* config at commit time (`io/commit.rs`: `if !commit_config.skip_auto_cleanup`). So the staged commit path (`commit_staged` → `CommitBuilder::with_skip_auto_cleanup(true)`), the `__manifest` publisher (`MergeInsertBuilder::skip_auto_cleanup(true)`), and the direct `WriteParams` paths all skip the hook. Without this, an upgraded graph would still auto-cleanup and delete `__manifest`-pinned versions. Pinned by `lance_surface_guards.rs::skip_auto_cleanup_suppresses_version_gc` (negative control + with-skip survival). +- **Lance #6658 SHIPPED in 7.0.0** (`DeleteBuilder::execute_uncommitted`, exposed via PR #6781) → MR-A (migrate `delete_where` to the staged two-phase API, retire the parse-time D2 rule) is now **unblocked**, tracked separately (dev-graph `iss-950`). The bump itself keeps `delete_where` inline; the `_compile_delete_result_field_shape` guard is left untouched until MR-A. +- **The unenforced primary key is now immutable once set** (`lance::dataset::transaction`, ~L2472–2480: `if !primary_key_before.is_empty() && (writes_primary_key || primary_key_after != primary_key_before) → "the unenforced primary key is a reserved key and cannot be changed once set"`). omnigraph marks `__manifest.object_id` as the unenforced PK (`lance-schema:unenforced-primary-key`) for merge-insert row-level CAS — baked into `manifest_schema()` at init, and added by the `migrate_v1_to_v2` internal-schema migration for pre-v0.4.0 graphs. The migration relied on Lance 6's idempotent re-apply for crash-recovery (a crash after the field-set but before the stamp bump re-enters the migration with the PK already present); under v7 that re-apply errors, so a real v1 graph could never finish migrating. Fixed by guarding the set on the manifest's unenforced-PK field (`db/manifest/migrations.rs::migrate_v1_to_v2`): `["object_id"]` → no-op, `[]` → set, any other PK field → loud refusal (the wrong CAS key, unchangeable under v7). Pinned by `lance_surface_guards.rs::unenforced_primary_key_is_immutable_once_set` (red if Lance relaxes immutability); regression: `db::manifest::tests::test_publish_migrates_pre_stamp_manifest_to_current_version` (was red under v7). +- **Native `DirectoryNamespace` no longer recognizes omnigraph's manifest-tracked tables** (`lance-namespace-impls` dir.rs ~L1310): `list/describe/create_table_version` route through `check_table_status`, which reports an omnigraph table absent → `TableNotFound`. The decoupling is *contingent on omnigraph's legacy boolean PK key*, not an unconditional v7 property: v7's namespace eagerly adds the new `lance-schema:unenforced-primary-key:position` key to any `__manifest` lacking it; that write hits the immutable-PK rule above (the boolean key already set the PK), so `ensure_manifest_table_up_to_date` errors and the namespace silently falls back to directory listing. omnigraph keeps the boolean key deliberately — Lance honors it permanently (maps to PK position 0), and one uniform on-disk format beats a new-vs-old split (existing graphs can't be re-keyed to the position key under that same immutability rule). omnigraph production never uses Lance's native namespace (its publisher writes `__manifest` directly via merge_insert; its own `namespace.rs` impls are custom), so this is test-only — the `test_directory_namespace_direct_publish_cannot_replace_native_omnigraph_write_path` surface guard was realigned to the v7 behavior (it now asserts the native namespace is fully decoupled, which only strengthens the guard's thesis). +- **Still NOT fixed in 7.0.0:** vector-index two-phase (Lance #6666 open) — `create_vector_index` inline residual retained; blob-column compaction — `compact_files_still_fails_on_blob_columns` guard still red on a fix, `optimize` still skips blob tables behind `LANCE_SUPPORTS_BLOB_COMPACTION`. +- **No Lance API surface omnigraph uses changed at *compile* time** (the only compile break was object_store) — but **two runtime behaviors did** (the unenforced-PK immutability and the native-namespace `TableNotFound`, above), each caught by the full engine test suite rather than the build. `CleanupPolicy`, `WriteParams` (apart from the `auto_cleanup` default), `CompactionOptions`, the namespace models (resolved via `lance-namespace-reqwest-client` 0.7.7, unchanged across the bump), `Operation`, `ManifestLocation`, and `MergeInsertBuilder` shapes are all stable. Lesson: a clean build is not a clean alignment — run `cargo test --workspace` before declaring a Lance bump done. + +Bump this date stanza on the next alignment pass. + +### Prior alignment audit: 2026-05-22 (Lance 6.0.1 upstream; omnigraph pinned at 6.0.1) Migration from Lance 4.0.0 → 6.0.1 landed in this cycle (DataFusion 52 → 53, Arrow 57 → 58, lance-tokenizer 6.0.1 added, tantivy* removed). Direct 4 → 6 jump; v5.x was not used as an intermediate (rationale in `~/.claude/plans/shimmering-percolating-duckling.md`). Behavior-affecting findings: @@ -169,6 +186,7 @@ Migration from Lance 4.0.0 → 6.0.1 landed in this cycle (DataFusion 52 → 53, - **`Dataset::checkout_version(N).await?.restore().await?`**: `restore()` takes `&mut self` and returns `Result<()>` (mutates in place, does not consume + return a new dataset). The recovery rollback hammer at `db/manifest/recovery.rs:505-522` continues to work. Pinned by `lance_surface_guards.rs::_compile_checkout_version_then_restore_signature`. - **`DatasetBuilder::from_namespace(...).with_branch(...).with_version(...).load()`** surface preserved (the namespace builder chain at `db/manifest/namespace.rs:162-174`). Pinned by `lance_surface_guards.rs::_compile_dataset_builder_from_namespace_signature`. - **`compact_files(&mut ds, CompactionOptions::default(), None)`** signature stable. `CompactionOptions` still does not expose `data_storage_version`; `compact_files` builds its own `WriteParams { ..Default::default() }`. Note: `LanceFileVersion::default()` is now V2_1 in v6, so optimize-rewritten fragments come out at V2_1 by default (was V2_0 in v4). Existing explicit V2_2 pins on creates/appends still apply. +- **`Dataset::optimize_indices(&mut self, &lance_index::optimize::OptimizeOptions)`** (via `DatasetIndexExt`) is a depended-on surface as of the index-coverage work: `db/omnigraph/optimize.rs` calls it after `compact_files` to fold appended/rewritten fragments into existing indexes (incremental merge, not retrain). It is a **committing** call (mutates in place, advances HEAD; no uncommitted variant in v6.0.1), so optimize treats it as an inline-commit residual under the `SidecarKind::Optimize` recovery sidecar. Signature pinned by `lance_surface_guards.rs::_compile_optimize_indices_signature`; the incremental-coverage behavior pinned by `optimize_indices_extends_fragment_coverage` (appended fragment uncovered before, covered after). - **`Dataset::delete(predicate)` returns `DeleteResult { new_dataset: Arc, num_deleted_rows: u64 }`** — unchanged shape. Pinned by `lance_surface_guards.rs::_compile_delete_result_field_shape`. MR-A will repurpose this guard to the staged two-phase variant once `DeleteBuilder::execute_uncommitted` migration lands. - **File reader read methods now async** (Lance PR #6710, v6.0). No effect — omnigraph reaches Lance exclusively through `Dataset::scan` and the staged-write API. - **Tokenizer vendored as `lance-tokenizer`** (Lance PR #6512, v6.0). No effect — no direct tokenizer imports. @@ -178,6 +196,4 @@ Migration from Lance 4.0.0 → 6.0.1 landed in this cycle (DataFusion 52 → 53, - **`Dataset::force_delete_branch`** (`branches().delete(name, force=true)`, dataset.rs:524) tolerates a missing branch-*contents* ref (vs plain `delete_branch`'s `RefNotFound`), but on the local store still errors `NotFound` if the branch `tree/` directory is fully absent (`remove_dir_all`'s NotFound is not caught for Lance's native error variant, refs.rs:526-549). Both variants still refuse a branch with referencing descendants (`RefConflict`). `TableStore::force_delete_branch` wraps this to be fully idempotent (tolerates already-absent). The single-authority branch-delete redesign uses it for orphan reclamation (eager best-effort reclaim + cleanup reconciler). Pinned by `lance_surface_guards.rs::force_delete_branch_semantics`. Branch delete is "flip the ref atomically, then `remove_dir_all(tree/{branch})`"; branch-exclusive data lives under `tree/{branch}/` so a drop reclaims it immediately without touching `main`. - **Lance blob-v2 `compact_files` bug** (no public issue found as of 2026-06): `compact_files` disables binary-copy for blob datasets and forces `BlobHandling::AllBinary` on the read side; the v2.1+ structural decoder then mis-counts column infos for the blob-v2 struct and fails with `Invalid user input: there were more fields in the schema than provided column indices / infos` (`lance-encoding/src/decoder.rs::ColumnInfoIter::expect_next`). This fails even a pristine uniform-V2_2 multi-fragment blob table; vector/list/scalar/ragged columns and mixed file versions all compact fine. Reads/queries use descriptor handling (`BlobHandling::default()`) and are unaffected. `optimize` skips blob-bearing tables behind `LANCE_SUPPORTS_BLOB_COMPACTION = false` (`db/omnigraph/optimize.rs`), reporting `SkipReason::BlobColumnsUnsupportedByLance`. Pinned by `lance_surface_guards.rs::compact_files_still_fails_on_blob_columns`, which turns red when the bug is fixed → flip the gate, remove the skip branch + the `maintenance.rs::optimize_skips_blob_table_and_reports_skip` skip assertions. -Surface guards added: `crates/omnigraph/tests/lance_surface_guards.rs` (10 named guards; 5 runtime + 5 compile-only). Future Lance bumps re-run this file first as the smoke check. Two additional guards from the original plan deferred to follow-up (`manifest_cas_returns_row_level_contention_variant` needs full publisher-race harness; `table_version_metadata_byte_compatible_with_v4` needs `pub(crate)` reach extension). - -Bump this date stanza on the next alignment pass. +Surface guards added: `crates/omnigraph/tests/lance_surface_guards.rs` (10 named guards; 5 runtime + 5 compile-only; plus the index-coverage work's `_compile_optimize_indices_signature` and `optimize_indices_extends_fragment_coverage`). Future Lance bumps re-run this file first as the smoke check. Two additional guards from the original plan deferred to follow-up (`manifest_cas_returns_row_level_contention_variant` needs full publisher-race harness; `table_version_metadata_byte_compatible_with_v4` needs `pub(crate)` reach extension). diff --git a/docs/dev/rfc-001-queries-envelope-mcp.md b/docs/dev/rfc-001-queries-envelope-mcp.md index b5d62d4..94d15e8 100644 --- a/docs/dev/rfc-001-queries-envelope-mcp.md +++ b/docs/dev/rfc-001-queries-envelope-mcp.md @@ -348,4 +348,4 @@ Callers move at their own pace. The envelope upgrades + URL rename ship in v0.6. - RFC 8288 (`Link` relations, `successor-version`) - MCP spec: [modelcontextprotocol.io](https://modelcontextprotocol.io) - [invariants.md](./invariants.md) — substrate boundaries this work respects -- [../user/server.md](../user/server.md) — current HTTP surface (post-MR-656 picks up the `/query`+`/mutate` rename and deprecation) +- [../user/server.md](../user/operations/server.md) — current HTTP surface (post-MR-656 picks up the `/query`+`/mutate` rename and deprecation) diff --git a/docs/dev/rfc-009-unify-access-paths.md b/docs/dev/rfc-009-unify-access-paths.md index 43dcdf2..9b2d842 100644 --- a/docs/dev/rfc-009-unify-access-paths.md +++ b/docs/dev/rfc-009-unify-access-paths.md @@ -68,7 +68,7 @@ anything moves — mirroring the storage collapse, where the pinned contract tests gated the swap, and the test-monolith modularization (#192/#193), which makes Phase 3 tractable: the CLI dispatch is 1,184 lines today, not 4,200. -### Phase 1 — Parity matrix (the referee; do first, no refactor) +### Phase 1 — Parity matrix (the referee; do first, no refactor) *(landed)* A CLI integration test (extend the `system_local.rs` harness, which already spawns both binaries): one fixture graph; for every forked verb, run the @@ -81,7 +81,16 @@ This pins today's behavior so Phase 3 can't silently change it, and catches every future fork drift. It also incidentally covers utoipa annotation↔route mismatches (a lying `#[utoipa::path]` makes the remote leg 404). -### Phase 2 — One wire-DTO crate +**Phase 1 outcome (landed):** `crates/omnigraph-cli/tests/parity_matrix.rs` +— 11 rows green with an **empty divergence ledger**: with matched Cedar +policy on both arms, embedded and remote agree on every forked verb's +scrubbed JSON and exit codes. Two findings along the way: like-for-like +requires the same policy bundle on both arms (a tokens-only server is +default-deny by design — the harness encodes this), and inline execution's +unbound-param matches-all vs the invoke path's hard error is a cross-path +asymmetry, filed as #207 and pinned (not repaired) by the matrix. + +### Phase 2 — One wire-DTO crate *(landed)* Move the HTTP request/response types and the single `engine result → DTO` mapping per verb into a shared crate (working name `omnigraph-api-types`), @@ -113,6 +122,15 @@ neither axum nor the engine's internals. The engine crate does not depend on it — the `engine result → DTO` mapping lives in the shared crate (or the CLI/ server side), taking engine result types as input. +**Phase 2 outcome (landed):** `crates/omnigraph-api-types` holds the wire +DTOs + their `engine-result → DTO` mappings; `omnigraph-server::api` is a +`pub use` re-export (so `openapi.json` is byte-identical — the referee +passed with zero diff), and the CLI consumes the crate directly. One +deliberate refinement of the original sketch: `LoadOutput` is a rendered +CLI output type, not a wire DTO, so it stayed CLI-side — both its mappings +(local `LoadResult`, remote `IngestOutput`) now sit together in +`output.rs`. The parity matrix passed textually unchanged. + ### Phase 3 — `GraphClient` trait, two implementations ```text @@ -143,15 +161,20 @@ and cluster commands must work with the server down) explicit in code. "Server" targets include operator-config named servers (RFC-007), not only literal `http(s)://` URIs. -### Phase 5 — Route alignment +### Phase 5 — Route alignment (landed) -Add a canonical `/load` endpoint (the handler already exists behind the -`/ingest` shim); point `RemoteClient` at it; keep `/ingest` on its existing -deprecation path. While here, check whether the server uses `utoipa-axum`'s -router-coupled registration (`OpenApiRouter`/`routes!`); if it hand-mounts -routes beside `#[utoipa::path]` annotations, prefer migrating registration so -path annotations and mount points are the same declaration (the modularization -already hit one orphaned-attribute incident of exactly this class). +Added a canonical `POST /load` (shared `run_ingest` body; the deprecated +`/ingest` is now a thin alias carrying `#[deprecated]` + RFC 9745/8288 +`Deprecation`/`Link: ` headers, exactly mirroring `/mutate`↔`/change`) +and pointed the CLI's remote `load` arm at it; `/ingest` stays on its +deprecation path. `/load` reuses `IngestRequest`/`IngestOutput` (as canonical +`/mutate` reuses `Change*`); a DTO rename is a separate change. + +Registration finding: the server **hand-mounts** routes (`.route(...)`) beside a +manual `#[openapi(paths(...))]` list, not `utoipa-axum`'s `OpenApiRouter`/ +`routes!`. This PR followed the existing manual pattern (one `.route` + one +`paths(...)` entry + the `#[utoipa::path]` annotation) rather than migrating +registration — the migration is a worthwhile but orthogonal cleanup, deferred. ## Non-goals diff --git a/docs/dev/rfc-010-cli-planes-restructure.md b/docs/dev/rfc-010-cli-planes-restructure.md new file mode 100644 index 0000000..ad7f7b9 --- /dev/null +++ b/docs/dev/rfc-010-cli-planes-restructure.md @@ -0,0 +1,449 @@ +# RFC: Restructure the CLI Around Explicit Planes + +**Status:** Proposed +**Date:** 2026-06-13 +**Audience:** CLI/server/cluster maintainers +**Builds on:** [rfc-009-unify-access-paths.md](rfc-009-unify-access-paths.md) +(Phases 3a–3c landed — the embedded/remote data-plane fork is now one +`GraphClient` enum; this RFC **expands RFC-009 Phase 4** from a narrow +embedded-vs-remote capability table into the full plane model, and leaves +Phase 5 route alignment where it is), +[rfc-007-operator-config.md](rfc-007-operator-config.md) (operator +`--server`/`--graph`/`--target` addressing — the surfaces this RFC makes +uniform across planes), +[rfc-008-deprecate-omnigraph-yaml.md](rfc-008-deprecate-omnigraph-yaml.md). +**Sequencing:** post-v0.7.0, after RFC-009 Phase 3c (done). + +## Summary + +The CLI silently spans **three planes** — data, storage/maintenance, and +control — and forces the operator to know which plane each verb lives on *and* +address a graph differently per plane. The same graph you query as +`--server prod --graph knowledge` you must maintain as +`s3://bucket/knowledge.omni`. Plane restrictions (`graphs list` is server-only, +`optimize` is storage-only) are *accidental* — discovered by hitting a cryptic +error, not *declared*. + +This RFC makes the plane model **explicit and coherent** with three moves: + +1. **One graph-addressing model** across every verb (`--target`/`--graph`/ + positional URI/`--server`), resolving to a storage URI for maintenance and a + remote client for data — instead of two different ways to name one graph. +2. **A declared, per-subcommand capability surface** (RFC-009 Phase 4): each + verb declares its plane(s); wrong-plane invocations get an honest "this is + storage-plane, `--server` doesn't apply" error from one table, not scattered + `bail!`s. +3. **Plane-grouped `--help`** so the model is legible at a glance. + +No new server feature. Storage maintenance stays off the wire — deliberately. + +## Current state of affairs + +The CLI has 23 top-level commands. They divide into three planes, addressed +three different ways: + +| Plane | Verbs | Reaches the graph by | Addressing surface | +|---|---|---|---| +| **Data** | `query`, `mutate`, `load`, `ingest`, `branch *`, `snapshot`, `export`, `commit *`, `schema show/apply` (and `graphs list`, **remote-only today** — see note) | embedded engine **or** HTTP server (one `GraphClient`) | positional URI **or** `--target` / `--graph` / `--server` (config aliases) | +| **Storage / maintenance** | `init`, `optimize`, `repair`, `cleanup`, `schema plan`, `queries validate` | embedded engine **only**, directly on storage (`file://` or `s3://`) | positional URI **or** `--target` — **no `--server` / `--graph`** (except `init`, which today takes **only a required positional URI** — no `--target`) | +| **Control** | `cluster validate/plan/apply/approve/status/refresh/import/force-unlock` | a cluster **directory** (`file://` or `s3://`), not a graph URI | `--config ` | + +### What's confusing (validated facts) + +1. **Two names for one graph.** Data verbs resolve `--server prod --graph + knowledge` through `GraphClient::resolve*` (the embedded/remote fork collapsed + in RFC-009 Phases 3a–3c; only the two `GraphClient` factories call + `apply_server_flag`). Maintenance verbs instead use + `resolve_uri`/`resolve_local_uri` and accept only a positional URI or + `--target` — so to compact the graph you *query* as `--server prod --graph + knowledge` you must *type* `s3://bucket/knowledge.omni`. One graph, two + addressing vocabularies. + + > **Note (`graphs list`).** It is routed through `GraphClient` only to share + > the addressing/token resolver; its embedded arm fails loudly, so it is + > **remote-only today** (the later capability table and *Relationship to + > RFC-009* record it as remote-now / embedded-cluster-later). + +2. **Plane restrictions are accidental, not declared.** `graphs list` is + server-only and `optimize`/`repair`/`cleanup`/`init` are storage-only purely + by code shape. Point `optimize` at an `https://` URL and you get whatever + `Omnigraph::open` says about an https URI — accidental error text that, per + Hyrum's Law, is already someone's dependency. The capability is real but + unstated. + +3. **The split is per-subcommand, and the family names hide it.** `schema plan` + is storage-only (`resolve_local_uri`) while `schema show`/`schema apply` are + data-plane (the graph client). `queries validate` opens the graph to + typecheck while `queries list` only reads the registry config. The plane is + a property of the *subcommand*, not the family. + +4. **Maintenance has no server/cluster counterpart at all.** There is no HTTP + route and no `cluster` subcommand for `optimize`/`cleanup`/`repair` (verified: + nothing in the server route table, nothing in `omnigraph-cluster/src`). For a + server-backed deployment you run the *same CLI* against the storage URI, + out-of-band from the serving process. This is correct (maintenance is + heavyweight, destructive, single-operator — it should not be a multi-tenant + HTTP surface), but it is **undocumented in the CLI's own shape**, so it reads + as an omission rather than a decision. + +5. **`init` has a hidden control-plane twin.** Bare `init` creates a single + graph from storage; in cluster mode the equivalent is `cluster apply` + (graph-creation stage, with ledger/recovery/approval semantics). Same intent, + two entry points, no signpost between them. + +6. **Flat `--help`.** All 23 commands list as one undifferentiated block, so the + plane a verb belongs to is tribal knowledge. + +The net effect: a new operator must already know OmniGraph's plane architecture +to predict which flags work on which verb and how to name a graph. The CLI does +not teach its own model. + +## Target CLI ergonomics + +The throughline: **you name a graph one way, and the CLI tells you what works +where.** Simple examples of the end state: + +### One name for a graph, everywhere + +A config target `knowledge` works on every verb that touches that graph: + +```bash +omnigraph query --target knowledge --query q.gq # data (embedded or remote, auto) +omnigraph load --target knowledge --data rows.jsonl # data +omnigraph optimize --target knowledge # maintenance (resolves to its storage URI) +omnigraph cleanup --target knowledge --keep 10 --confirm +omnigraph repair --target knowledge --confirm +``` + +The positional URI form still works everywhere, unchanged: + +```bash +omnigraph optimize s3://bucket/knowledge.omni +``` + +### Data plane: same command, embedded or remote + +You don't pick "local vs server" syntax — resolution decides: + +```bash +omnigraph query ./local.omni --query q.gq # opens engine directly +omnigraph query --server prod --graph knowledge --query q.gq # over HTTP +omnigraph query --target knowledge --query q.gq # whichever the config says +``` + +### Maintenance: `--target` must resolve to direct storage (loud if not) + +```bash +$ omnigraph optimize --target prod +error: `--target prod` resolves to a remote server (https://prod…). + `optimize` is a storage-plane command and needs direct storage access. + Pass the graph's s3://… URI, or use --cluster --graph . +``` + +Cluster-managed graphs get an explicit, intentional path (no implicit +`cluster.yaml` peeking): + +```bash +omnigraph optimize --cluster ./cluster --graph knowledge +``` + +### Wrong-plane = one honest, stable error + +```bash +$ omnigraph optimize --server prod +error: `optimize` is a storage-plane command; `--server` addresses the data + plane and does not apply here. Use --target or a storage URI. + +$ omnigraph graphs list ./local.omni +error: `graphs list` needs a remote multi-graph server (http/https) today. + (Embedded cluster-catalog enumeration is planned — RFC-009.) +``` + +### `--help` teaches the model + +``` +DATA PLANE run against a graph (embedded or --server) + query mutate load branch snapshot export commit schema show schema apply + +STORAGE / MAINTENANCE direct storage access; no server + init optimize repair cleanup schema plan queries validate + +CONTROL PLANE manage a cluster directory + cluster + +INSPECT / SESSION + graphs list queries list lint policy embed login logout config +``` + +### Exceptions, signposted (not silent) + +```bash +omnigraph init --schema s.pg ./new.omni # plain path: fine + +$ omnigraph init --target knowledge --schema s.pg # cluster-managed target: redirected +error: `knowledge` is a cluster-managed graph. Create it via `cluster apply` + (which records ledger + recovery + approvals), not `init`. +``` + +**In one line:** one way to name a graph, the right flags accepted per verb, and +a CLI that tells you its planes instead of making you memorize them. + +## Proposed shape (mechanism) + +### One addressing model for every graph-addressing verb + +Route **all** graph-addressing verbs — data *and* maintenance — through one +resolver that turns `(positional URI | --target | --graph | --server)` into +either a **storage URI** (`file://`/`s3://`) → embedded execution, or a **remote +`GraphClient`** → HTTP execution, per the verb's declared plane. + +**Authority rule (the precedence must not be silent).** `--target` is an +operator/legacy target lookup; `cluster.yaml` is a *different* authority surface +(read only by `cluster` commands and `--cluster` boot). A maintenance verb must +not quietly consult both and invent a precedence. The rule: + +- A maintenance verb's `--target` resolves through the **operator/legacy** + config and its URI must already be **direct storage**; a target that resolves + to a remote (`http(s)://`) URL **fails loudly** (see the example above). +- **Cluster-managed graphs are addressed explicitly** via a cluster-root + + graph-id pair (spelled `--cluster --graph ` for illustration), so + reading cluster state is an intentional mode — never an implicit fallback + between operator config and `cluster.yaml`. + + > **Flag-shape caveat (deferred).** `--graph` is *already* a global flag that + > `requires = "server"` and appends `/graphs/` to a **remote** URL — a + > different meaning, and clap won't permit `--graph` without `--server`. So the + > cluster-maintenance addressing needs either a distinct flag (e.g. + > `--cluster-graph `) or an explicit global-flag migration. This is why + > the cluster-managed resolver is **deferred to a later slice** (it also rides + > the applied-state-vs-declared-config open question below); the + > operator/legacy `--target` path lands first. + +### A declared, per-subcommand capability surface (RFC-009 Phase 4, expanded) + +One table, **per subcommand** (family-level rows hide exactly the cases the +table exists to make non-accidental): + +| Command | Data (embedded) | Data (remote) | Storage (direct) | Config / session | Notes | +|---|---|---|---|---|---| +| `query`, `mutate`, `load`, `ingest` | ✅ | ✅ | — | — | `ingest` is the deprecated alias of `load` | +| `branch create/list/delete/merge` | ✅ | ✅ | — | — | | +| `snapshot`, `export`, `commit list/show` | ✅ | ✅ | — | — | | +| `schema show` | ✅ | ✅ | — | — | | +| `schema apply` | ✅ | ✅ | — | — | declarative alternative: `cluster apply` | +| `schema plan` | — | — | ✅ | — | local resolver today | +| `queries validate` | — | — | ✅ | — | opens the graph to typecheck | +| `init` | — | — | ✅ | — | cluster-managed graphs → `cluster apply` | +| `optimize`, `repair`, `cleanup` | — | — | ✅ | — | | +| `graphs list` | (later) | ✅ | — | — | remote today; embedded-cluster later (RFC-009) | +| `queries list` | — | — | — | ✅ | reads the registry config; no graph | +| `lint` | — | — | ✅ | ✅ | `--schema` file, or opens a local graph | +| `policy validate/test/explain` | — | — | — | ✅ | reads policy files + config | +| `embed` | — | — | — | ✅ | local tooling (files + embedding API) | +| `login`, `logout`, `config`, `version` | — | — | — | ✅ | session / config; no graph | + +The resolver consults this table. A wrong-plane invocation produces one honest, +stable message instead of N ad-hoc `bail!`s and accidental `open` errors. + +### Plane-grouped `--help` + +Group the command list by plane (the `--help` block shown under Target CLI +ergonomics). Cosmetic, zero behavior change, highest legibility-per-line. + +### Maintenance stays off the wire (decision, not omission) + +This RFC **does not** add server routes for `optimize`/`cleanup`/`repair`: + +- **Serving = the server.** Multi-tenant, safe-for-many-callers data plane. +- **Storage maintenance = the CLI against storage**, addressed uniformly, + run by an operator or a scheduled job with storage access. + +Adding maintenance-over-HTTP would re-introduce a heavyweight, destructive +multi-tenant surface and *add* a plane rather than clarify the three we have. +A future cluster-driven maintenance reconciler (scheduled compaction/GC as a +control-plane policy) is explicitly **out of scope** — net-new design (who runs +it, with what resource bounds), not a CLI restructure. + +### `init` is an explicit exception (decision) + +Direct-storage `init` against a plain URI/target stays. But if a target resolves +to a **cluster-managed** graph root, `init` **refuses and signposts** `cluster +apply` (which records ledger, recovery, and approval artifacts) rather than +initializing that root out of band. This closes the "hidden twin" of the current +state. + +## Compatibility + +Additive and low-risk: + +- **`--target`/`--graph` on maintenance verbs** is new capability; the positional + URI form keeps working unchanged. +- **Grouped `--help`** is cosmetic. +- **Capability-surface error text** changes the message you get on a wrong-plane + or misaddressed invocation. Per Hyrum's Law that text is observable; the change + is deliberate, release-noted, and replaces an *accidental* `Omnigraph::open` + string with a *stable, declared* one — a net improvement, but flagged. + +No engine, server, or wire-protocol change. The work is CLI-internal: the shared +resolver, the capability table, and help grouping. + +## Test plan + +Extend the existing CLI suites rather than adding a duplicate harness: + +- **`parity_matrix.rs`** — capability exclusions (the per-subcommand plane table + becomes the source of truth for which verbs are remote-only / storage-only). +- **`cli_data.rs`** — maintenance wrong-plane errors (`optimize --server`, + `optimize --target `), and `--target` resolving to direct storage. +- **`cli_schema_config.rs`** — `graphs list` plane behavior, `schema plan` + vs `schema show/apply` plane split, and plane-grouped `--help` output. +- **`system_local.rs`** — `--server` / operator-targeting edge cases end-to-end. + +Pin the new wrong-plane error strings deliberately: this RFC is intentionally +replacing accidental `Omnigraph::open` strings with stable capability errors, and +those strings become observable behavior (Hyrum). + +## Relationship to RFC-009 + +RFC-009 Phase 4 was scoped as "declared plane capabilities" for the +embedded-vs-remote axis only. This RFC **subsumes and broadens** that phase into +the full three-plane, per-subcommand model (adds uniform maintenance addressing, +the authority rule, and help grouping). RFC-009 Phase 5 (remote `load` → +`/load` route alignment) is unaffected and remains in RFC-009. + +**`graphs list` reconciliation:** RFC-009's answered open question (pinned in +`parity_matrix.rs`'s exclusions comment) targets `graphs list` becoming +Both-capability once the embedded arm enumerates the cluster catalog. This RFC +**aligns** with that rather than superseding it: the capability table shows +`graphs list` as remote today, embedded-cluster later. + +## Open questions + +1. **Capability-table location** — a CLI-internal const, or surfaced (e.g. in + `--help` and a machine-readable `omnigraph capabilities` for tooling)? +2. **`--cluster --graph ` for maintenance** — does the maintenance + command resolve the storage URI from the applied cluster state, or from the + declared `cluster.yaml`? (Applied state is the truth the server serves; + declared config may be ahead of it.) + +## Review comments (Codex, 2026-06-13) + +Overall take: the direction is right. The planes already exist; making them +declared in code, help text, and error messages should reduce operator surprise. +Keeping storage maintenance off HTTP is also the right boundary: `optimize`, +`repair`, and `cleanup` are direct-storage operator actions, not a multi-tenant +serving surface. + +Before implementation, tighten these points: + +1. **Resolver authority needs a sharper rule.** The proposal says maintenance + resolves storage URIs "from `cluster.yaml` / operator config", but those are + different authority surfaces. Today `--target` is an operator/legacy + graph-target lookup; cluster config is read by `cluster` commands and by + `--cluster` server boot. Do not make a maintenance command silently consult + both and pick a precedence. Either: + - `--target` on maintenance means an operator/legacy target whose URI is + already direct storage, with remote targets failing loudly; or + - add an explicit cluster-root/config resolver for this case, so reading + cluster state is an intentional mode. + + **Resolution (accepted):** both — `--target` resolves through operator/legacy + config and must be direct storage (remote → loud fail); cluster-managed graphs + use the explicit `--cluster --graph ` resolver. See *Authority + rule* under Proposed shape. + +2. **`graphs list` conflicts with RFC-009's target shape.** This RFC classifies + `graphs list` as remote-only, while RFC-009's answered open question says it + becomes Both-capability once the embedded arm enumerates the cluster catalog. + Pick one direction here: either this RFC explicitly supersedes that target, + or the capability table should show `graphs list` as remote today and + embedded-cluster later. + + **Resolution (accepted):** align, don't supersede. The table shows `graphs + list` remote-today / embedded-cluster-later. See *Relationship to RFC-009*. + +3. **The capability table should be per subcommand, not per family.** The + family-level rows hide the exact cases the table is supposed to make + non-accidental. At minimum, call out: + - `schema plan` as local/storage-backed today, while `schema show` and + `schema apply` route through the graph client; + - `queries validate` versus `queries list`, which do not have the same + plane shape; + - `lint`, `policy`, `embed`, `login`, `logout`, `config`, and `version`, so + enumeration/session/tooling commands are intentionally classified instead + of falling outside the model. + + **Resolution (accepted):** the capability table is now per-subcommand and + classifies every command, including the session/tooling group. + +4. **`init` should be an explicit exception.** Direct-storage `init` is fine. + A cluster-managed graph should be created by `cluster apply`, with ledger, + recovery, and approval semantics. If a named target resolves to a + cluster-managed graph root, `init` should signpost `cluster apply` rather + than quietly initializing that root out of band. + + **Resolution (accepted):** promoted from open question to a decision. See + *`init` is an explicit exception*. + +Testing notes for the implementation slice: + +- Extend the existing CLI suites rather than adding a new duplicate harness: + `parity_matrix.rs` for capability exclusions, `cli_data.rs` for maintenance + wrong-plane errors, `cli_schema_config.rs` for `graphs list` / help behavior, + and `system_local.rs` for `--server` / operator-targeting edge cases. +- Pin the new wrong-plane error strings deliberately. This RFC is intentionally + replacing accidental `Omnigraph::open` strings with stable capability errors, + and those strings become observable behavior. + + **Resolution (accepted):** captured as the *Test plan* section. + +## Verification comments (Codex, 2026-06-13) + +Follow-up verification against the current CLI/server code found a few +remaining current-state nits. These are doc-shape issues, not objections to the +proposal: + +1. **Current-state table overstates `graphs list`.** The table under *Current + state of affairs* still lists `graphs list` with data verbs that reach the + graph by embedded engine or HTTP. Current code routes it through `GraphClient` + only to share the resolver, but the embedded arm fails loudly; the later + RFC text correctly says remote today / embedded-cluster later. Make the + current-state row match that. + + **Resolution (accepted):** the Data row now marks `graphs list` **remote-only + today**, with a note that it rides `GraphClient` only to share the resolver. + +2. **Current-state table overstates `init` addressing.** `init` is grouped with + maintenance verbs whose addressing surface is positional URI or `--target`. + Current `init` only accepts a required positional URI and has no `--target` + or config path. The proposal can add that capability, but the current-state + table should not describe it as already present. + + **Resolution (accepted):** the Storage row now calls out that `init` takes + **only a required positional URI** today (no `--target`); adding `--target` to + `init` is part of the proposal, entangled with the `init`→`cluster apply` + signpost, not current state. + +3. **`apply_server_flag` call-site count is stale.** The text says data verbs + resolve `--server prod --graph knowledge` through `apply_server_flag` at + 16 call sites. Current code has the fork collapsed: data verbs call + `GraphClient::resolve*`, and only the two `GraphClient` factories call + `apply_server_flag`. Rephrase the verified fact around `GraphClient`, not + the old pre-collapse call-site count. + + **Resolution (accepted):** validated-fact #1 now describes the post-collapse + reality (`GraphClient::resolve*`; the two factories call `apply_server_flag`), + dropping the stale count. + +4. **`--cluster --graph ` collides with today's global `--graph` + semantics.** The target ergonomics section proposes that flag shape for + maintenance, but current `--graph` is a global flag that requires + `--server` and appends `/graphs/` to a remote server URL. Either choose + a separate cluster-maintenance graph flag shape, or call out the clap/global + flag migration explicitly as part of the implementation. + + **Resolution (accepted):** the *Authority rule* now carries a flag-shape + caveat — the cluster-managed resolver (and its flag shape, e.g. + `--cluster-graph` vs a `--graph` migration) is **deferred to a later slice**; + the operator/legacy `--target` path lands first. The illustrative + `--cluster --graph ` spelling is marked as not-final. diff --git a/docs/dev/rfc-011-cli-refactoring.md b/docs/dev/rfc-011-cli-refactoring.md new file mode 100644 index 0000000..d26dd84 --- /dev/null +++ b/docs/dev/rfc-011-cli-refactoring.md @@ -0,0 +1,756 @@ +# RFC-011: CLI refactoring — one addressing & config model + +**Status:** Accepted — implemented (the `omnigraph.yaml` excision landed as +#250/#251/#252; D1–D4, D6, D7, D9, D10 shipped). Two items remain: **D11** +(server-side maintenance jobs) is gated on the bulk-data-plane RFC #219; **D5** +(combined admin scope) stays deferred by design. +**Date:** 2026-06-14 +**Audience:** CLI/server maintainers +**Builds on:** [rfc-007-operator-config.md](rfc-007-operator-config.md) +(per-operator config, keyed credentials, named servers), +[rfc-008-deprecate-omnigraph-yaml.md](rfc-008-deprecate-omnigraph-yaml.md) +(the legacy file this RFC finishes removing), +[rfc-009-unify-access-paths.md](rfc-009-unify-access-paths.md) +(`GraphClient` — embedded ≡ remote at the execution layer), +[rfc-010-cli-planes-restructure.md](rfc-010-cli-planes-restructure.md) +(declared planes + the wrong-plane guard this RFC subsumes). +**Sequencing:** lands as / after RFC-008 stage 5 (the `omnigraph.yaml` removal). + +## Summary + +Refactor the CLI around one coherent model once `omnigraph.yaml` is gone. The +shape: + +- **One ontology** (store, server, cluster; cluster config vs operator config; + catalog; profile; capability) where each term names exactly one concept. +- **Addressing = scope + `--graph`, with the access path *derived*.** A command + resolves a *scope* (operator defaults, an optional named *profile*, or one + explicit primitive address — `--store` / `--server` / `--cluster`), selects a + graph inside it with `--graph`, and the **served-vs-direct access path falls out + of the scope's bindings × the verb's capability** — it is never a per-command + toggle and never inferred from a URI scheme. +- **Served is the front door; direct storage is privileged.** The everyday scope + is a *server* (a bearer token, no bucket credentials). Reading or writing a + remote store/cluster directly is an explicit, credentialed, admin/break-glass + act — never the default, never baked into everyday operator config. +- **The CLI is stateless per command.** No `current_profile` pointer, no + `USE`-style mode; every command is fully determined by its flags + static + config. You *select* a graph, you do not *switch into* one. +- **Definitions are named; payloads are passed.** Queries (`.gq`) and schema + (`.pg`) live in the catalog and are invoked by name; params and bulk data are + the only per-call inputs. + +This removes `--target`, `--cluster-graph`, `--uri` scheme-dispatch, and the +plane guard's "a `--target` that resolves to a remote URL" special case — and it +collapses the four-plane vocabulary, for users, into a single capability rule. + +## Motivation: the legacy file pollutes the taxonomy + +Today the CLI exposes four overlapping addressing forms but the system has only +three real entities; the mismatch is the whole problem, and `omnigraph.yaml` is +the carrier: + +1. **`--target` straddles kinds.** It resolves through the legacy + `omnigraph.yaml` `graphs:` map (`config.rs::resolve_target_uri`), and that + `.uri` can be a **storage location** (`file`/`s3`) *or* a **remote server** + (`http`). One flag, two access paths with different capability and trust + models. The wrong-plane guard's storage-plane remote rejection + (`helpers.rs:467`) exists *only* to compensate for this overload. +2. **Scheme-inferred transport.** ``/`--uri` has the same disease a level + down: `is_remote_uri` (`helpers.rs:15`) silently picks embedded vs remote from + the scheme. Transport is guessed from a string, not declared. +3. **No single environment concept.** Defaults are smeared across the deprecated + `omnigraph.yaml` (`cli.graph`, `server.graph`) with no clean way to name or + switch environments. + +Removing `omnigraph.yaml` is the moment to fix all three at once. + +## Ontology + +Every term is one concept. The rest of this RFC uses them precisely. + +### Entities — the things that exist + +- **Graph** — a typed property graph (node/edge types over Lance); the thing you + query and mutate. *Example: the `knowledge` graph.* +- **Store** — the storage location of a **single** graph: its Lance datasets at a + `file://`/`s3://` URI. Addressed directly with `--store`. *Example: + `s3://acme/clusters/brain/graphs/knowledge.omni`.* +- **Cluster** — a storage root holding **many** graphs plus the catalog and + control-plane state (state ledger, approvals, recovery). Managed as-code by the + team. *Example: the `brain` cluster at `s3://acme/clusters/brain`.* +- **Server** — an `omnigraph-server` process serving graphs over HTTP with bearer + auth and Cedar policy; boots from a bare graph or a cluster. *Example: `prod` at + `https://graph.example.com`, serving the `brain` cluster.* + +### Config & catalog — the descriptions + +- **Cluster config** — `cluster.yaml` in the cluster root, declaring the **desired + state** (graphs, schemas, stored queries, policies, storage), applied with + `cluster apply`. Team-owned; the source of truth for *what the system is*. +- **Catalog** — the **applied** registry the cluster owns in storage: the graphs, + stored queries, and policies `cluster apply` materialized. What a server serves + and what `query ` resolves against. *(Cluster config is the spec; the + catalog is the applied result.)* +- **Operator config** — `~/.omnigraph/config.yaml`, your **personal** file: + identity (actor), default graph, named servers/clusters, output prefs, optional + profiles. Declares *who I am*, never what the system is. +- **Profile** — an optional named bundle of **defaults inside the operator + config** (one of {cluster, server, store} + a default graph). Config data, + **not state**: selecting one fills in omitted flags for a command; it does not + put you "in" a mode. Chosen per command (`--profile `) or per shell + (`OMNIGRAPH_PROFILE`). +- **Credential** — a bearer token keyed to a **server name**, resolved via + `OMNIGRAPH_TOKEN_` or `~/.omnigraph/credentials` (`0600`); sent only to + the server it is keyed to. (Per RFC-007 — the operator config holds endpoints, + never tokens.) + +### What you run — definitions vs payloads + +- **Schema** — the `.pg` type definitions for a graph; authored as a file, applied + via `schema apply` (or `cluster apply`). +- **Stored query** — a named query in the catalog, the team's reusable contract; + invoked by name. *Example: `find_people`.* +- **Query file (`.gq`)** — an authoring artifact holding `query ` + declarations; becomes a stored query when `cluster apply` adopts it. For + authoring/ad-hoc, not everyday invocation. +- **Payload** — the per-call inputs that vary each run: params (`--params`, + positional args) and bulk data (`--data`). Never part of config. + +### How a command resolves + +- **Scope** — the resolved environment a command addresses: operator defaults, a + named profile, or one explicit primitive address. +- **Access path** — **served** (through a server) or **direct** (open storage + in-process). Derived from scope × capability; see "Access path" below. +- **Capability** — what a verb requires: `any`, `served`, `direct`, `control`, + or `local`. +- **Target shape** — whether the verb is **graph-scoped** (selects one graph + inside the scope), **scope-scoped** (operates on the whole server/cluster + scope), or **local** (does not resolve scope or graph). +- **Actor** — the identity a write is attributed to: server-resolved from the + bearer token (served), or `--as` ?? `operator.actor` (direct). + +### The relationships that prevent confusion + +- **Exactly two config surfaces:** **cluster config** (team) and **operator + config** (personal). Nothing else is "a config." +- A **profile is not a third config** — it lives *inside* the operator config, and + it is **defaults, not state**. +- A **catalog is not config** — it is the *applied state* the cluster owns. +- A **store is one graph; a cluster is many graphs** + catalog + control state. +- A **graph is the logical thing**; store/server/cluster are ways to reach it. +- "State" elsewhere is not the profile: *graph state* is committed data in Lance; + *cluster state* is the applied control-plane ledger. Neither is operator config. + +## Design + +### First principles + +> Addressing should be 1:1 with the system's real entities; the access path +> (served vs direct) should be **derived**, never inferred from a string or +> toggled per command; the CLI should be **terse by config and stateless per +> command**; and **definitions are named while payloads are passed**. + +Every command answers four orthogonal questions — kept orthogonal here: + +| Axis | Question | Today | Target | +|---|---|---|---| +| Scope | which environment? | `omnigraph.yaml` defaults / `--target` | operator defaults · `--profile` · one primitive | +| Target shape | whole scope or one graph? | implicit in command family | declared per verb | +| Graph | which graph in it? | tangled into the address | `--graph` only for graph-scoped server/cluster verbs | +| Access path | served or direct? | inferred from scheme / target | **derived** from scope × capability | +| Actor | who am I? | `--as` > `cli.actor` (yaml) > `operator.actor` | `--as`/`operator.actor` (direct) · token (served) | + +### A scope binds one entity — and served is the default + +A scope (a profile, the flat defaults, or one primitive flag) binds **exactly one +of** {server, cluster, store}. Server and cluster scopes may contain many graphs +and can carry a `default_graph`; a store scope is already one graph and does not +accept `--graph`. They differ by privilege, and **the everyday default is a +server**: + +- **server** → served (the everyday scope). A bearer token, **no storage + credentials**. Data verbs run through it, policy-enforced; maintenance verbs are + unavailable from this scope — there is no server route for them, so you must + name storage explicitly. This is what a normal operator's config binds. +- **cluster** → direct storage to a managed cluster, for **control, + maintenance, and graph-backed validation only** (`cluster *`, + `optimize`/`repair`/`cleanup`/`schema plan`, graph-backed `lint`, and + `queries validate`). Data verbs are **not** run directly against a cluster — + they go served, or `--store` for ad-hoc. **Privileged:** requires bucket + credentials, so it appears only in a maintainer's config or as an explicit + `--cluster` flag — never in an everyday operator's defaults. +- **store** → one graph's storage, direct. A **local file** store is ordinary + local dev; a **remote `s3://`** store is break-glass. No catalog (named queries + do not resolve — the ad-hoc lane). + +A scope names **one** thing, so there is no independent `server`+`cluster` pair +that could disagree (the audit's coherence hazard is gone by construction — the +default is just a server). And the storage root lives only where it must: + +### Direct storage access is privileged (the storage-root rule) + +> The storage root (`s3://…`) is **server-and-admin knowledge, never +> everyday-operator knowledge.** Everyday operator config binds a server (a bearer +> token, no bucket credentials). Direct remote access — opening a cluster root or +> an `s3://` store — is always **explicit and privileged**: you name +> `--cluster`/`--store`, and only someone with bucket credentials can. The CLI +> never opens a remote store from a default scope. + +This is the least-privilege posture — revoke a bearer token, don't rotate bucket +keys; only the **server process** and an occasional **maintenance admin** ever +hold storage credentials. It makes "use the server, not raw storage" +**structural**, not advisory: direct access requires credentials a normal operator +does not have *and* a flag they must type. The only storage root in an everyday +setup is the one the **server** boots from; operators never see it. (Local *file* +stores for dev are unaffected — a local file is not the production bucket.) + +### Access path is derived, not chosen + +The two access paths are genuinely different — not two transports for one thing: + +- **Served** (through a server): the server resolves your actor from a token and + enforces Cedar policy at the HTTP boundary. In cluster mode the **catalog and + config** (graph set, stored queries, policy bundles) are pinned to the applied + serving revision and move only on restart; **graph data** is read through the + server's engine handle against the requested branch/snapshot (it is not frozen + at boot, though a long-running server will not observe *out-of-band direct + writes* to storage until its handle refreshes). No storage credentials needed. +- **Direct** (open the Lance storage in-process): a **privileged** path — it needs + your own storage credentials, so only an admin/maintainer (or a local-dev file + store) takes it. Actor self-declared (`--as` ?? `operator.actor`), reads **live + storage HEAD**. There is **no server-side identity/auth gate** — but engine-level + Cedar policy *is* still enforced when the graph selection provides a policy + (enforcement is engine-wide; embedded `_as` writers call the same `enforce`). + "Direct" means "no HTTP boundary," not "unpoliced." + +Because they differ in authority, freshness, and availability, a graph reached via +a server and that graph's raw storage are **different things you name +differently** — not one identity you flip. Making the access path a per-command +toggle (`--via`) is the `--target` mistake in new clothes; it is rejected. + +> **The access path follows from the scope and the verb.** A **server** scope → +> served (data/catalog). A **cluster** scope → direct control, maintenance, and +> validation. A **store** scope → direct ad-hoc data (no catalog). The verb's +> capability picks which applies and rejects the mismatches. + +State the bound plainly: the everyday data path +(`query`/`mutate`/`load`/`branch`/`export`/`commit`) against a served graph +**never needs direct storage access**, and direct access is legitimate only in +bounded places: **bootstrap** (`init`), **storage-native maintenance** +(`optimize`/`repair`/`cleanup`/`schema plan`), **graph-backed validation** +(`lint`), **catalog validation** (`queries validate`), the **control plane** +(`cluster *`), **local dev** with no server, and **break-glass** (recovery, or +checking whether a long-running server's handle lags live HEAD). Everything else +is served. This is what makes "discourage direct storage" enforceable rather +than aspirational. + +This list is expected to **shrink**: Decision 11 moves +`optimize`/`cleanup` (and healthy-path `repair`) to server-managed jobs, which +would leave direct access to just standalone/local dev, the control plane, and +break-glass — and remove the last routine reason an admin needs bucket +credentials. + +### Capability semantics + +The CLI validates through verb capability, not plane jargon: + +| Capability | Meaning | Examples | +|---|---|---| +| `any` | graph-scoped data; served via a server scope; direct only against a **store** scope (local dev / break-glass); **errors on a cluster scope** | `query`, `mutate`, `load`, `export`, branch reads, `schema show/apply` | +| `served` | requires an HTTP server; may be graph-scoped or scope-scoped | `graphs list`, `queries list` | +| `direct` | graph-scoped storage-native or graph-backed validation; no server form exists | `init`, `optimize`, `repair`, `cleanup`, `schema plan`, graph-backed `lint` | +| `control` | cluster-scoped catalog/control-plane work; addresses the cluster, not a single raw store | `cluster *`, `queries validate` | +| `local` | does not address a graph or scope | `config`, `profile`, `lint --query ... --schema ...` | + +`any` does **not** mean "the user picks": the resolver picks from the scope. +Internally the exhaustive `command_plane` match (`planes.rs`) stays as the drift +guard; user-facing errors speak in terms of what the command needs. + +### Definitions vs payloads + +Queries and schema are **definitions** — contracts that live in the catalog and +are invoked **by name**; params and data are **payloads** passed per call. So the +everyday form is `omnigraph query [params]`, not +`omnigraph query --file find.gq`. A `.gq` path on a routine query is a smell: the +query is not in the catalog yet. Lifecycle: **author a `.gq` → `cluster apply` +adopts it → invoke by name thereafter.** + +Named queries resolve through a **server** (which serves the cluster's catalog). +`queries list` is therefore a served catalog read. `queries validate` is a +control/catalog check against the cluster-owned query definitions. A bare +`--store` has **no catalog**, so it is the ad-hoc lane (`-e` / `--file`), and +`--cluster` does not invoke stored queries. So named-query invocation is a +**served** convenience; direct access (`--store`) is always ad-hoc. + +| Kind | Examples | How it enters a command | +|---|---|---| +| Definition | stored query, schema | named in the catalog; authored as a file, adopted by `cluster apply` | +| Payload | params, bulk data | passed per call (`--params`, positional args, `--data`) | +| Authoring / ad-hoc | a `.gq` you're writing | `-e '…'`, `--file new.gq`, `lint --query new.gq --schema schema.pg`, `schema apply --schema` | + +### Resolution rule + +1. If the verb is `local`, reject graph/scope flags and run without resolving a + scope. +2. If a primitive address is supplied (`--store`/`--server`/`--cluster`), use it + and ignore operator-config scope defaults. *(A **named** primitive — `--server + prod`, `--cluster brain` — still resolves through the operator-config registry; + a **literal** — `--server https://…`, `--store s3://…` — bypasses it. Per + Decision 2: a value containing `://` is a literal, otherwise a config-name + lookup.)* +3. Else if `--profile ` (or `OMNIGRAPH_PROFILE`) selects a profile, use it. +4. Else use the operator config's flat defaults. Error only if neither resolves. + *(No sticky "current" pointer — each command resolves scope fresh.)* +5. Resolve the graph only for **graph-scoped** verbs. Server/cluster scopes: + exactly one graph in scope → use it; else `default_graph`; else require + `--graph `. Store scopes are already one graph, so `--graph` is rejected. + **Scope-scoped** verbs (`graphs list`, `queries list`, `queries validate`, + and `cluster *`) do not select a graph unless their own resource argument says + otherwise. +6. Derive the access path from capability × scope: + - `direct` verb → the scope's cluster/store; if the scope is a server, error + (name storage explicitly — it is privileged). + - `served` verb → the scope's server; if the scope is a cluster/store, error. + - `control` verb → the scope's cluster; if the scope is a server/store, error + (name a cluster explicitly — it is privileged). + - `any` verb → **served** if the scope is a server; **direct** against a + **store** scope (ad-hoc); on a **cluster** scope, error — cluster is + maintenance-only, so use a server for data or `--store` for ad-hoc. +7. Reject mismatches with an error naming the missing axis. + +Good errors: + +```text +scope "prod" has 4 graphs; pass --graph or set default_graph +optimize needs direct storage access; scope "prod" is a server — name storage with --cluster s3://… or --store (requires storage credentials) +graphs list enumerates a server scope; do not pass --graph +--store opens raw storage directly, bypassing any server (no HTTP auth gate, live HEAD); for recovery/inspection +``` + +### Config shape (operator config) + +`~/.omnigraph/config.yaml` — your personal file; the cluster config +(`cluster.yaml` + catalog) is the separate, team-owned surface. The default-graph +key is `default_graph` everywhere (the per-command flag is `--graph`). + +**Everyday operator — binds a server, holds no storage root:** + +```yaml +defaults: + server: prod + default_graph: knowledge + output: table +servers: + prod: { url: https://graph.example.com } # token keyed by name (RFC-007); no creds here + staging: { url: https://staging.example.com } +profiles: # optional, only for multiple environments + staging: { server: staging, default_graph: knowledge } +``` + +A normal operator never has a storage root or bucket credentials. Their default +scope is served; `optimize`/`repair`/`cleanup` error with a pointer to name +storage explicitly. + +**Maintainer — opts into a cluster root (and has bucket credentials):** + +```yaml +profiles: + brain-admin: { cluster: brain, default_graph: knowledge } # direct; admin/control/maintenance +clusters: + brain: { root: s3://acme/clusters/brain } # the s3:// root lives ONLY here +``` + +The `clusters:` block — the only place a storage root appears in operator config — +is **admin-only and opt-in**, absent from a normal operator's file. Equivalently, +skip config and name it per command: +`omnigraph optimize --cluster s3://acme/clusters/brain --graph knowledge`. The +cluster stays the source of truth for the managed catalog; tokens live in the +keyed credential store, never in this file. + +### Command shape + +Assume the everyday flat defaults: server `prod`, default graph `knowledge`. + +| Intent | Command | Path | +|---|---|---| +| Run a catalog query | `omnigraph query find_people` | served | +| …with params | `omnigraph query find_people --params '{"title":"Eng"}'` | served | +| Another graph in scope | `omnigraph query find_people --graph archive` | served | +| Write | `omnigraph load --data batch.jsonl --mode append` | served | +| A different environment | `omnigraph --profile staging query find_people` | served | +| One-off server, no config | `omnigraph query find_people --server https://graph.example.com --graph knowledge` | served | +| Maintain (admin, explicit storage) | `omnigraph optimize --cluster s3://acme/clusters/brain --graph knowledge` | direct (privileged) | +| Maintain (admin, via admin profile) | `omnigraph --profile brain-admin optimize --graph knowledge` | direct (privileged) | +| List catalog queries | `omnigraph queries list` | served | +| Validate cluster query catalog | `omnigraph queries validate --cluster s3://acme/clusters/brain` | control (privileged) | +| Offline query lint | `omnigraph lint --query new.gq --schema schema.pg` | local | +| Graph-backed query lint | `omnigraph lint --query new.gq --cluster s3://acme/clusters/brain --graph knowledge` | direct (privileged) | +| Local dev, no server | `omnigraph query -e 'match { … } return { … }' --store graph.omni` | direct (local file) | +| Break-glass: raw storage of a served graph | `omnigraph query --file find.gq --store s3://acme/clusters/brain/graphs/knowledge.omni` | direct (privileged, rare) | + +Note what the everyday rows are: **all served.** `optimize` does *not* appear in +the default-scope rows — from a server scope it errors and points you to name +storage (see the resolution rule), so maintenance is always a deliberate, +credentialed act. There is no "force served/direct" row — you never toggle the +path on a configured graph; the only way to reach raw storage is to *name it* +(`--cluster`/`--store`), which makes the privileged bypass unmistakable. Everyday +rows invoke a query **by name**; a `.gq` file appears only where there is no +catalog (bare store, break-glass) via `-e`/`--file`. + +## Before / after + +**Before** = best available today (legacy `omnigraph.yaml` `--target`, `.gq` +files, `--cluster-graph`, scheme inference). **After** = this model. + +| Intent | Before | After | +|---|---|---| +| Run a query | `omnigraph query --target knowledge --query find.gq --name find_people` | `omnigraph query find_people` | +| Another graph | `omnigraph query --target archive --query find.gq --name find_people` | `omnigraph query find_people --graph archive` | +| Load | `omnigraph load --data b.jsonl --mode append --target knowledge` | `omnigraph load --data b.jsonl --mode append` | +| Maintain (admin) | `omnigraph optimize --cluster brain --cluster-graph knowledge` | `omnigraph optimize --cluster s3://acme/clusters/brain --graph knowledge` | +| Another environment | edit `omnigraph.yaml`, or re-address with full URIs | `--profile staging …` or `OMNIGRAPH_PROFILE=staging` | +| One-off remote | `omnigraph query --uri https://… --query find.gq` *(scheme→remote)* | `omnigraph query find_people --server https://… --graph knowledge` | +| Raw storage of a served graph | `omnigraph query s3://…/knowledge.omni --query find.gq` *(looks like a normal query)* | `omnigraph query --file find.gq --store s3://…/knowledge.omni` *(explicit bypass)* | + +**Removed:** `--target`; `--cluster-graph` (`--graph` is the graph selector only +for graph-scoped server/cluster verbs); `--uri` http-scheme dispatch; `--via` +(never ships); everyday `--query ` (definitions are named); +`omnigraph.yaml` and its `cli.graph`/`server.graph` defaults. + +## Server-side corollary + +The same ontology applies to `omnigraph-server` boot: with `omnigraph.yaml` gone, +a server boots from a single bare graph URI **or** a cluster (`--cluster `, +RFC-005), never a `graphs:` map. The store/server/cluster ontology is then +consistent across CLI and server. + +## Migration & compatibility + +Addressing flags and config keys are observable contract (Hyrum); every removal is +staged and release-noted. + +- **`config migrate`** (shipped) maps each legacy `graphs:` entry **by what it + actually is**: `http(s)` URIs → a `server:` (the recommended everyday shape); + `file` URIs → a local `store:`; an `s3://` **graph** URI → an **admin** `store:` + (it is a single graph, not a cluster); an `s3://` **cluster root** (one that + carries cluster state) → an **admin** `cluster:`. Everyday `s3://` graph usage + migrates with a **warning** — prefer serving it via a server rather than + re-establishing direct remote access. It reports dropped keys. +- **Operators move to a server-default scope.** Where a legacy setup pointed + `cli.graph` at an `s3://` graph for everyday use, migration flags it: the + recommended shape is a `server:` scope (bearer token, no bucket creds), with the + `s3://` root kept only in a maintainer's config — not every operator's. +- **`--target`** warns for one release, then errors; **`OMNIGRAPH_NO_LEGACY_CONFIG=1`** + (already the strict switch) becomes the default — loading `omnigraph.yaml` is a + hard error. +- **`--cluster-graph` → `--graph`**: `--cluster-graph` is accepted with a warning + for one release, then removed. +- **`--graph` meaning change**: today `--graph` is "graph id on a multi-graph + server" (paired with `--server`); it generalizes to "select the graph for + graph-scoped verbs in server/cluster scopes." Existing `--server --graph` + usage keeps working (it is a strict superset); release-note the broadened + meaning and the fact that store/scope-scoped verbs reject it. +- **`--uri http://…`** warns, then errors with a pointer to `--server`. +- **`--as` on served paths**: today global `--as` is accepted (a no-op on remote + writes — the server resolves the actor from the token); rejecting it on the + served path is staged — warn for one release, then error. +- **`--alias`** → the `alias` namespace (`omnigraph alias `, Decision 4); + the old `--alias` flag warns for one release, then is removed. + +## Non-goals + +- **No change to the direct/served capability split.** Maintenance stays + storage-direct by design (no server routes for `optimize`/`repair`/`cleanup`); + this RFC only makes the split explicit. +- **No new transport.** Addressing surface, not protocol. +- **No positional sigil grammar** (`@server/graph`, `%cluster/graph`). Considered + and rejected: explicit flags are more discoverable; profiles already give + brevity. Revisit only on demonstrated expert-terseness demand. + +## Decisions + +The questions this RFC opened are resolved as follows. Two are explicitly +deferred (see below); they do not block the model. + +1. **Local-dev path → embedded `--store` scope.** Local dev runs the engine + in-process against a `--store ` (or a store-scoped profile); `omnigraph + serve` stays available but is not required. Consistent with embedded ≡ remote + (RFC-009). +2. **Primitives are one flag, typed by content.** `--server` and `--cluster` + accept either a config name or a literal URI: a value containing `://` is a + literal (bypasses the registry); otherwise it is a config-name lookup (error if + unknown). `--store` is always a URI. (Replaces the earlier "literal-vs-named" + question — no `--server-url`/`--cluster-root` split.) +3. **Stored invocation: `query ` (read) / `mutate ` (write), one + catalog namespace.** A name maps to one definition; the verb asserts its kind + and the CLI errors on mismatch (`'apply_labels' is a mutation — use + omnigraph mutate apply_labels`). No `invoke` verb. +4. **Aliases live under an `alias` namespace** — `omnigraph alias [args]`, + never bare top-level. An alias can therefore neither shadow nor be shadowed by a + built-in (current or future) verb. +6. **Profile merge: scope wholesale, prefs layered.** The entity binding + + `default_graph` come *wholesale* from the active scope (a profile, or flat + defaults if none) — never per-key merged across the entity dimension (that would + yield "server *and* cluster"). Only non-scope preferences (`output`, table + layout) take flat defaults as a base. Precedence: explicit flag > profile > flat + defaults. +7. **No default graph → error + list candidates.** A graph-scoped verb with no + `--graph`, no `default_graph`, and >1 graph in scope errors and lists candidates + (served: `GET /graphs`; cluster-direct: catalog enumeration). If enumeration is + policy-gated/unavailable, it says so and asks for `--graph`. Never auto-pick. +9. **Diagnostics & safety.** Writes echo the resolved scope + access path to stderr + (suppress with `--quiet`). Destructive verbs (`cleanup`, overwrite `load`, + `branch delete`) require confirmation when the scope is not local; `--yes` skips + it; **no TTY without `--yes` errors** (never silently proceed). `--json`/CI never + prompt — destructive without `--yes` errors. +10. **Cluster graphs evolve only via `cluster apply`.** `schema apply` (an `any` + verb) targets standalone graphs; against a cluster-managed graph it errors and + points at `cluster apply` (which records ledger/recovery/approvals — RFC-004). + Mirrors `init`'s refusal of a cluster-managed path. +11. **Maintenance moves server-side (committed direction).** `optimize`/`cleanup` + (and healthy-path `repair`) become server/cluster-managed async jobs — + policy-gated, audited, single-coordinator — with `direct` retained only as + break-glass (`repair` when the server is down). Runs out-of-band (a worker + + async job routes, the `POST …` / `GET …/{id}` shape of the bulk-data-plane RFC + (`docs/rfcs/0001-bulk-data-plane.md`, PR #219, not yet merged)), never inline in + serving; `schema plan` is + excluded (≈ `cluster plan` in cluster mode). The **mechanism** (job routes, + worker, scheduling) is a follow-up RFC; until it lands the capability table above + stands, and maintenance is `direct`. When it lands, the maintenance verbs' + capability becomes "served-job + direct break-glass." + +## Deferred + +Non-blocking; settle when convenient. + +- **D5 — combined admin scope.** A scope binds one entity; admins read via a + server scope and maintain via `--cluster`. A `deployments: { … }` object + (server + cluster validated coherent, referenced by a profile) is revisited only + if admin ergonomics demand it — and Decision 11 largely removes the need. +- **D8 — the `profile` command surface.** *Shipped:* `profile list` / `profile + show []` (read-only inspection). The *no sticky `profile use`* constraint + holds — it is a design principle, not a command. + +## Safety + +Dropping the sticky `current_profile` pointer removes the main footgun — a +destructive command silently inheriting a "current" environment from an earlier +session. Because each command resolves scope fresh, what is on the command line is +what runs. Two guards remain (a flat default or `OMNIGRAPH_PROFILE` can still point +at prod): echo the resolved scope + access path on writes, and require +confirmation (or `--yes`) for destructive verbs when the resolved scope is not +local (Decision 9). The most dangerous direct writes (`cleanup`, overwrite +`load`) are *structurally* rare now — unavailable from the everyday server scope, +and gated behind bucket credentials plus an explicit `--cluster`/`--store` — so a +normal operator's setup mostly cannot issue them by accident at all. + +## Invariants & deny-list check + +- **§10 query semantics first-class / §11 transport at the boundary:** preserved — + addressing resolves CLI-side to a `GraphClient`; no transport concepts leak into + engine crates. +- **§12 no client-set actor:** strengthened — the served path's actor stays + token-resolved and `--as` is rejected there; direct self-declares. +- **Least privilege (security posture):** everyday operators hold a revocable + bearer token, not bucket credentials; only the server process and maintenance + admins hold storage creds. Direct remote access is structural opt-in, not a + default — narrowing the blast radius of a leaked operator config. +- **§6 strong consistency:** both paths are snapshot-isolated per query; this RFC + changes addressing, not isolation. +- **Deny-list (no state that drifts):** profiles and aliases are static config + sugar that resolve to canonical scopes; they declare nothing the cluster or + server doesn't already own. No sticky session state is introduced. +- No Hard Invariant is weakened; the change is CLI surface + config removal. + +## Relationship to prior work + +The completion of the config/CLI lineage: RFC-007 added the operator config and +keyed credentials; RFC-008 demoted `omnigraph.yaml`; RFC-009 unified execution +behind `GraphClient`; RFC-010 declared the planes. This RFC removes the last +legacy addressing surface so the plane model becomes a clean function of the three +real entities, and folds the planes into a single capability rule. It is adjacent +to the public-track bulk-data-plane RFC (`docs/rfcs/0001-bulk-data-plane.md`, +PR #219, not yet merged), which canonicalizes `load`/`export` verbs; this RFC +canonicalizes how every verb *addresses* a graph. + +## Appendix: target CLI taxonomy (end state) + +The full command set under this model, organized by **capability** (the new +classifying axis) instead of plane — the end-state counterpart to the +current-taxonomy appendix below. Every command, with its end-state addressing. + +``` +omnigraph +│ +├─ any — data verbs · served by default (server scope, or --server ); +│ --graph selects the graph in scope; --store forces ad-hoc direct (no catalog) +│ ├─ query (alias: read*) invoke a stored query by NAME; -e/--file for ad-hoc +│ ├─ mutate (alias: change*) invoke a stored mutation by name; -e/--file for ad-hoc +│ ├─ load bulk write — --data, --mode required; --from forks a missing branch +│ ├─ export dump graph data (NDJSON / Arrow) +│ ├─ snapshot current per-table versions +│ ├─ branch { create | list | delete | merge } merge takes --into +│ ├─ commit { list | show } inspect the commit graph +│ └─ schema { show (alias: get) | apply } cluster graphs evolve via cluster apply (Decision 10) +│ +├─ served — needs a server (errors on a store/cluster scope) +│ ├─ graphs list enumerate the graphs a server serves +│ └─ queries list list stored queries in the served catalog +│ +├─ direct — storage-native, PRIVILEGED · --cluster | --store + bucket creds; never a server +│ ├─ init bootstrap a graph (--store ); refuses a cluster-managed path +│ ├─ optimize compaction; --graph selects +│ ├─ repair publish uncovered drift; --confirm / --force +│ ├─ cleanup version GC; --keep / --older-than / --confirm +│ ├─ schema plan migration preview (reads storage directly) +│ └─ lint --query graph-backed query lint (with --graph on cluster scope) +│ +├─ control — cluster/catalog control, PRIVILEGED · --cluster +│ ├─ cluster { validate | plan | apply | approve | status | refresh | import | force-unlock } +│ apply/approve take --as ; force-unlock takes +│ └─ queries validate validate cluster-owned stored queries against graph schemas +│ +└─ local — no graph + ├─ policy { validate | test | explain } offline Cedar tooling + ├─ profile { list | show } read-only; NO mutating `use` (no sticky state) + ├─ alias [args] personal shortcut; expands to its bound stored-query call (D4) + ├─ config { migrate } finish the omnigraph.yaml split (RFC-008) + ├─ login / logout per-server bearer credentials + ├─ embed offline embedding pipeline + ├─ lint --query --schema file-only query lint + └─ version (-v) +``` + +`*` `read`/`change` remain as deprecated aliases (warn on use); `ingest` and the +`check`→`lint` argv-shim are **removed**. `get` aliases `schema show`. + +### Addressing forms (end state) + +Three scope forms — one per real entity — plus the graph selector. No `--target`, +no `--cluster-graph`, no `--uri` scheme-dispatch, no `--via`. + +| Form | Resolves to | Access | Privilege | +|---|---|---|---| +| **server scope** — operator default, a `--profile`, or `--server ` | a served endpoint + keyed token | served | everyday (bearer token) | +| **cluster scope** — an admin profile, or `--cluster ` | a managed cluster's storage + catalog | direct | privileged (bucket creds) | +| **store scope** — `--store ` | one graph's storage (no catalog) | direct | local-dev (file) / break-glass (s3) | +| **`--graph `** | selects the graph for graph-scoped verbs in server/cluster scopes; invalid for store scopes and scope-scoped verbs | — | — | + +Resolution: explicit primitive (`--server`/`--cluster`/`--store`) → `--profile` / +`OMNIGRAPH_PROFILE` → operator flat defaults. Access path is then derived from the +scope kind × the verb's capability (see the Resolution rule); it is never inferred +from a URI scheme and never toggled. + +### What moved vs today + +| Command(s) | Today (plane) | End state (capability) | +|---|---|---| +| `query`/`mutate`/`load`/`export`/`snapshot`/`branch`/`commit`/`schema show`/`schema apply` | Data | **`any`** (served-default; `--store` ad-hoc) | +| `graphs list` | Data (remote-only) | **`served`** | +| `queries list` | Session | **`served`** (catalog read) | +| `init`/`optimize`/`repair`/`cleanup`/`schema plan`/graph-backed `lint` | Storage | **`direct`** (privileged) | +| `queries validate` | Storage | **`control`** (catalog validation) | +| `cluster *` | Control | **control** (unchanged) | +| `policy *`/`embed`/`login`/`logout`/`config`/`version`/offline `lint --query --schema` | Session | **`local`** | +| `ingest`; `--target`; `--cluster-graph`; `--uri http` dispatch | present | **removed** | +| — | — | **added:** `profile { list | show }` (read-only) | + +Cross-capability families: `schema` (`plan` is `direct`, `show`/`apply` are +`any`), `queries` (`list` is `served`, `validate` is `control`), and `lint` +(offline with `--schema` is `local`, graph-backed is `direct`) split per +subcommand/mode, exactly where their authority and data dependencies differ. + +## Appendix: current CLI taxonomy (today) + +The **as-is** command surface this RFC transforms, kept so the RFC is +self-contained. The source of truth is the exhaustive `command_plane` match in +`crates/omnigraph-cli/src/planes.rs`. +Where it disagrees with the design above (four planes, `--target`, +`--cluster-graph`, scheme-inferred transport), the design is the *target* and this +is *today*. + +### The four planes (today) + +| Plane | What it touches | Addressing accepted | +|---|---|---| +| **Data** | a graph — embedded **or** via a server | `` · `--target` · `--server` (+`--graph`) | +| **Storage** | direct storage, no server | `` · `--target` (local/S3 only) · some also `--cluster`+`--cluster-graph` | +| **Control** | a cluster *directory* | `--config ` | +| **Session** | no graph | — | + +`--server`/`--graph` are gated strictly to the data plane; `guard_addressing` +(`planes.rs:128`) rejects them elsewhere (RFC-010 Slice 1). + +### Command tree by plane (today) + +``` +omnigraph +├─ DATA ────────── run against a graph; embedded or --server +│ ├─ query (alias: read) · mutate (alias: change) · load · ingest (hidden, deprecated) +│ ├─ branch { create | list | delete | merge } · snapshot · export · commit { list | show } +│ ├─ graphs { list } (remote-only) +│ └─ schema { show (alias: get) | apply } ← show/apply are DATA +├─ STORAGE ─────── direct file://|s3:// access; --server rejected +│ ├─ init · optimize · repair · cleanup (optimize/repair/cleanup also: --cluster --cluster-graph) +│ ├─ lint (check shim) · schema plan ← plan is STORAGE +│ └─ queries validate +├─ CONTROL ─────── cluster directory via --config +│ └─ cluster { validate | plan | apply | approve | status | refresh | import | force-unlock } +└─ SESSION ─────── no graph + ├─ policy { validate | test | explain } · embed · login / logout + ├─ config { migrate } · queries list ← list is SESSION + └─ version (-v) +``` + +`read`/`change` are visible clap aliases (deprecated names, warn); `check` is an +argv-shim → `lint`; `get` aliases `schema show`; `ingest` is hidden but runs. + +### Cross-plane families (today) + +- **`schema`**: `schema plan` is Storage; `schema show`/`apply` are Data. +- **`queries`**: `queries validate` is Storage; `queries list` is Session. + +### Addressing forms (today) + +| Form | Looks up in | Resolves to | Source | +|---|---|---|---| +| `` / `--uri` | nothing (explicit) | the literal URI | — | +| `--target ` | `omnigraph.yaml` `graphs:` | that graph's `uri` (local / S3 / **http**) | `config.rs::resolve_target_uri` | +| `--server ` (+`--graph`) | `~/.omnigraph/config.yaml` `servers:` | a remote server URL | `helpers.rs::resolve_server_flag` | +| `--cluster --cluster-graph ` | served cluster state | the graph's storage URI | `helpers.rs` (RFC-010 Slice 3) | + +Precedence (`resolve_target_uri`): explicit ``/`--uri` → `--target` → +`cli.graph` default → error. `is_remote_uri` (`helpers.rs:15`) then selects +`GraphClient::Remote` vs `Embedded` (`client.rs:86`). + +### Enforcement points (today) + +- **`guard_addressing`** (`planes.rs:128`): `--server`/`--graph` on a non-data verb + fails with a declared message. +- **Storage-plane remote rejection** (`helpers.rs:467`): a storage verb whose + `--target` resolves to `http(s)://` is rejected. +- **`init` into a cluster layout** is refused (use `cluster apply`). + +## Audit comments + +Reviewed against the current CLI taxonomy, `planes.rs`, `cli.rs`, `helpers.rs`, +`client.rs`, RFC-007/RFC-010, and the user-facing CLI/server docs. + +### Validated + +- The target taxonomy now has a stable classifier: `any`, `served`, `direct`, + `control`, and `local` are all declared capabilities. +- Cluster scope is coherent: it is privileged direct storage for control, + maintenance, and validation, not a direct data path. `any` data verbs served by + default and reject cluster scope. +- Graph selection is no longer universal. Graph-scoped verbs select a graph; + scope-scoped verbs such as `graphs list`, `queries list`, `queries validate`, + and `cluster *` address the whole server/cluster scope. +- The current-state appendix still matches the implemented CLI: four planes, + `--target`, `--cluster-graph`, scheme-inferred transport, `schema plan` as + Storage, and `schema show/apply` as Data. + +Decisions and deferrals are tracked in [Decisions](#decisions) above — not +duplicated here. diff --git a/docs/dev/rfc-012-embedding-provider-config.md b/docs/dev/rfc-012-embedding-provider-config.md new file mode 100644 index 0000000..45083a2 --- /dev/null +++ b/docs/dev/rfc-012-embedding-provider-config.md @@ -0,0 +1,295 @@ +# RFC: Provider-Independent Embedding Configuration + +**Status:** Accepted — Phases 1-5 implemented +**Date:** 2026-06-15 +**Builds on:** the engine embedding client (`crates/omnigraph/src/embedding.rs`), the `@embed` catalog +annotation (`omnigraph-compiler/src/catalog`), the cluster `providers.embedding` surface +([cluster-config-specs.md](cluster-config-specs.md), [rfc-007-operator-config.md](rfc-007-operator-config.md) +for the secret-resolution pattern). +**Target release:** staged — NFR floor first, then the provider-independent config core; ingest-time `@embed` +execution is a separate later phase. + +## Summary + +OmniGraph's embedding subsystem is **hardwired to a single provider (Google Gemini)** and has no recorded +link between the model that produced a stored vector and the model that embeds a query string. Today that +happens to be self-consistent (one live client embeds both sides), but it is consistent by accident, not by +construction: the provider is hardcoded, the model is a moving `-preview` target, nothing validates that a +query vector and a stored vector share a space, and the one configurable knob (key + base URL) cannot change +the provider or model. + +This RFC makes embedding **provider-independent**: one resolved `EmbeddingConfig { provider, model, base_url, +api_key, dim, normalize }` behind a sealed provider abstraction, resolved once and shared by every embedder. +The **primary variant is OpenAI-compatible** — a single request/response shape (`POST {base}/embeddings`, +`{model, input, dimensions}`) that covers **OpenRouter** (the recommended default gateway, one key for Gemini, +OpenAI, Mistral, BGE, Qwen, sentence-transformers, …), OpenAI direct, and any self-hosted OpenAI-compatible +endpoint (vLLM, Ollama, LM Studio, Together). A native **Gemini** (`generativelanguage`) variant is retained +for shops that want to hit Google directly with its `RETRIEVAL_QUERY`/`RETRIEVAL_DOCUMENT` task-type +asymmetry, plus a deterministic **Mock**. The embedding *identity* (provider + model + dim) is recorded in the +schema IR so it travels with the data, and a query whose resolved embedder cannot match the stored vectors' +recorded identity is **rejected with a typed error instead of silently ranking across vector spaces.** +Provider/endpoint wiring lands on the already-reserved cluster `providers.embedding` field; secrets follow the +existing operator-credential pattern; no secret ever enters the schema. + +This RFC supersedes the framing in `docs/user/search/embeddings.md` that described "two embedding clients +with different defaults" — one of those clients was dead code with zero callers and has been removed (see +Phase 1); the OpenAI request shape returns as a first-class *provider variant* of the one client, not as a +second parallel client. + +## Motivation + +This work originated in an external handoff that reported a live cross-provider bug: gemini-3072 stored +vectors compared against OpenAI-1536 query vectors, silently. Investigation against the current source showed +the reported mechanism is **inaccurate** — the OpenAI client it blamed (`omnigraph-compiler/src/embedding.rs`) +was `pub(crate)`, `#![allow(dead_code)]`, and had **zero callers**; the live `nearest("string")` path and the +offline `omnigraph embed` CLI both use the engine **Gemini** client; and `@embed` does no ingest-time +embedding at all. So the documented happy path is self-consistent. But the investigation surfaced four real +problems the handoff's instincts correctly smelled: + +- **P1 — Provider is hardwired.** The one live client builds Google `generativelanguage` requests; only key + + base URL are configurable, not the provider or model. A non-Gemini shop cannot use `nearest("string")` + without a Gemini key, and cannot make it produce non-Gemini vectors. If they store their own vectors and + query with `nearest("string")`, the query is embedded with Gemini → a silent cross-space ranking. This is + the handoff's failure, reached by a different cause. +- **P2 — A dead, divergent second client + stale docs** invited exactly the misdiagnosis the handoff made. +- **P3 — No same-space guarantee recorded with the data.** Nothing stamps which model/dim produced a stored + vector, so write-side and read-side embedders can drift with no validation. +- **P4 — `@embed` is declarative-in-name-only.** It records a source property for the typechecker but never + embeds at ingest; the docs claimed otherwise. + +Per the project's first principle, the lower-liability shape is **one provider-independent client with the +identity recorded next to the data**, not N independently-defaulted clients kept in lockstep by discipline. +Hardcoding one provider mortgages every future "we need OpenAI / a local model / Vertex" against a rewrite; +recording identity once closes the silent-wrong-results class by construction. + +## Current state — which API we actually use + +| | Live engine client (`crates/omnigraph/src/embedding.rs`) | Deleted dead client (was `omnigraph-compiler/src/embedding.rs`) | +|---|---|---| +| Provider | **Google Gemini Developer API** (`generativelanguage`, *not* Vertex AI) | OpenAI | +| Endpoint | `POST {base}/models/{model}:embedContent` | `POST {base}/embeddings` | +| Auth | header `x-goog-api-key`, env `GEMINI_API_KEY` | `Authorization: Bearer`, env `OPENAI_API_KEY` | +| Model | `gemini-embedding-2-preview` (hardcoded) | `text-embedding-3-small` (env `NANOGRAPH_EMBED_MODEL`) | +| Base default | `https://generativelanguage.googleapis.com/v1beta` | `https://api.openai.com/v1` | +| Request body | `{model, content:{parts:[{text}]}, taskType, outputDimensionality}` | `{model, input:[…], dimensions}` | +| Response | `{embedding:{values:[f32]}}` | `{data:[{index, embedding:[f32]}]}` | +| Task types | `RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT` | none | +| Status | **live** — used by `nearest("string")` and `omnigraph embed` | **removed in Phase 1** (zero callers) | + +Both shapes honour a requested output dimensionality (Gemini `outputDimensionality`, OpenAI `dimensions`) +driven by the target column width, so dimension is already schema-driven. The two known shapes are exactly the +two initial provider variants this RFC defines — the OpenAI shape returns from git history as a `Provider` +variant of the single client. + +## Guide-level explanation + +### Configuring a provider (operator view) + +Pick a provider for the graph in `cluster.yaml` (the team-owned surface), referencing a secret by name. The +recommended default routes through OpenRouter (OpenAI-compatible, one key for many models): + +```yaml +providers: + embedding: + default: + kind: openai-compatible # openai-compatible | gemini | mock + base_url: https://openrouter.ai/api/v1 + model: google/gemini-embedding-2 # or openai/text-embedding-3-large, mistralai/mistral-embed, … + api_key: ${OPENROUTER_API_KEY} +graphs: + knowledge: + schema: knowledge.pg + embedding_provider: default +``` + +The same `openai-compatible` kind points at OpenAI direct (`base_url: https://api.openai.com/v1`, +`model: text-embedding-3-large`) or a self-hosted endpoint (vLLM/Ollama/LM Studio) by changing `base_url`. Use +`kind: gemini` only to reach Google's `generativelanguage` API directly (it keeps the query/document +task-type asymmetry that the OpenAI-compatible shape does not expose). Dimensions are schema-driven by the +target `Vector(N)` column, not duplicated in the provider profile. + +The zero-config tier keeps working with env only (`OMNIGRAPH_EMBED_PROVIDER`, `OMNIGRAPH_EMBED_BASE_URL`, +`OMNIGRAPH_EMBED_MODEL`, and the provider api-key env — `OPENROUTER_API_KEY` / `OPENAI_API_KEY` / +`GEMINI_API_KEY`), so no cluster file is required for a single-graph setup. + +### Recording identity in the schema + +`@embed` grows optional arguments that pin the embedding identity to the vector column: + +```pg +node Doc { + slug: String @key + text: String + v: Vector(3072) @embed("text", model="gemini-embedding-2", dim=3072) @index +} +``` + +The single-argument form `@embed("text")` keeps working unchanged. The recorded identity persists in the +schema IR (`_schema.ir.json`) and so travels with `schema apply` and `schema show`. + +### What a mismatch looks like + +If the resolved read-side embedder cannot produce the recorded identity (wrong model, wrong dim, wrong +provider), `nearest($v, "string")` fails with a typed error naming both sides, instead of returning a +plausible-but-meaningless ranking. Changing the recorded identity on an existing column is a loud schema-apply +refusal (it is a re-embed, a deliberate migration step), reusing the migration planner's existing +annotation-change rejection. + +## Reference-level design + +### One client, sealed provider abstraction + +Replace the two-variant `EmbeddingTransport` with a resolved config plus a sealed provider enum: + +```text +EmbeddingConfig { provider: Provider, model, base_url, api_key, dim, normalize } +enum Provider { + OpenAiCompatible, // POST {base}/embeddings, Bearer auth, {model, input, dimensions} → {data:[{embedding,index}]} + // covers OpenRouter (default gateway), OpenAI direct, vLLM/Ollama/LM Studio/Together + Gemini, // POST {base}/models/{model}:embedContent, x-goog-api-key, with RETRIEVAL_QUERY/DOCUMENT task types + Mock, // deterministic, offline +} +struct EmbeddingClient { config, http, retry, deadline } +``` + +`Provider` owns the per-API differences (endpoint suffix, auth header, request JSON, response JSON, task-type +support); the client owns retry/backoff, the deadline, normalization, and tracing — all provider-independent. +**OpenRouter is not a distinct variant** — it is `OpenAiCompatible` with `base_url = +https://openrouter.ai/api/v1`, which is the point: one OpenAI-compatible shape gives provider-independence +across every model OpenRouter fronts, so the gateway does the multi-provider fan-out and OmniGraph carries one +request shape. The native `Gemini` variant exists only for direct-to-Google with task-type asymmetry. An enum +(not a trait) is the earned complexity for this small, first-party set; if third-party plug-in providers are +ever needed, the enum becomes a trait behind the same `EmbeddingConfig` surface without touching callers. + +The OpenAI-compatible `input` accepts an **array**, giving batch embedding for free — which the later +ingest phase needs for throughput, and which removes the open dependency on Gemini's native +`batchEmbedContents`. + +### Config resolution (resolved once, shared) + +Precedence, highest first for served cluster graphs: applied cluster `providers.embedding.` profile → +env (`OMNIGRAPH_EMBED_*`, provider api-key env) → built-in defaults. The cluster `api_key` value is a +`${NAME}` env reference resolved at server boot; plaintext never lives in the schema, state ledger, or any +checked-in file. Resolution happens once per graph handle; the resolved client is shared by +`nearest("string")`. Direct single-graph serving, embedded callers, and the offline CLI keep the env path +unless they inject an `EmbeddingConfig` directly. + +### Identity recorded in the schema IR (not a new store) + +The `@embed` args serialize into `PropertyIR.annotations` → `_schema.ir.json`, which `schema apply` already +persists atomically and which the catalog (the one thing `nearest()` reads at query time) is built from. No +new metadata store, no manifest column, no extra read on the query path. The migration planner already rejects +non-description annotation changes as `UnsupportedChange`, so "recorded identity is immutable without a +deliberate re-embed migration" is the default behaviour, not new code. (A second, optional copy in Lance +field metadata — co-located with the vectors — is available later by activating the currently no-op +`UpdatePropertyMetadata` migration step; out of scope here.) + +### Query-time validation + +`resolve_nearest_query_vec` compares the resolved read-side identity against the column's recorded identity +before embedding; on mismatch it returns a typed `OmniError` naming recorded vs resolved (model, dim, +provider). This is the only behaviour that closes P3 by construction. + +### NFR floor (independent of the provider work) + +- **Deadline:** wrap every embed call (query or document) in a total-operation deadline + (`OMNIGRAPH_EMBED_DEADLINE_MS`) so a degraded provider cannot hang the caller for the current ~121 s worst + case (4 × 30 s timeout + backoff). +- **Observability:** `tracing` span per embed call (provider, model, dim, attempts, outcome, elapsed; `warn!` + per retry; token usage when the provider returns it). The subsystem has zero instrumentation today. +- **Single normalization:** one `normalize_vector` (the dead client carried a divergent second copy; removed + in Phase 1). +- **Stable model:** make the model configurable and default to a stable (non-`-preview`) model once the GA + name is confirmed. + +### Ingest-time `@embed` (later phase, not this RFC's core) + +Making `@embed` embed at ingest is a separate phase with a hard constraint: embedding is a slow, external, +**non-idempotent** side effect, so it must run **entirely before staging** — in the pure in-memory phase, +before any `stage_*`/Lance HEAD move, alongside the existing constraint validation — so a mid-load provider +failure aborts with zero drift. It must never sit inside or after the commit protocol, because the recovery +sweep cannot re-run or undo an external embedding. It also needs a content-hash skip (so `load --mode +overwrite` does not re-bill every row), batching, and a bounded-concurrency stage. Specified here only to fix +the design constraint; deferred to its own RFC/phase. + +### Phasing (implementation order) + +| Phase | Scope | Demo | +|---|---|---| +| **1 — NFR floor + dead-client removal** | deadline, observability, single normalize, configurable model, delete dead client + `NANOGRAPH_*` | a hung provider fails at the deadline; embed calls traced; `rg NANOGRAPH_` empty | +| **2 — Provider-independent config** | `EmbeddingConfig` + `Provider` enum (OpenAiCompatible covering OpenRouter/OpenAI/local, Gemini, Mock); env-first resolution; client reuse | point `base_url` at OpenRouter, run `nearest("string")`, get correct neighbours vs OpenRouter-stored vectors; CLI shares the config | +| **3 — Record identity in schema IR** | `@embed` args grammar + catalog + IR persistence | `schema show` reflects recorded model/dim | +| **4 — Query-time validation** | compare resolved vs recorded; typed error; planner refusal on identity change | stored model A vs read model B → loud error, never silent garbage | +| **5 — Cluster provider wiring** | `providers.embedding` resources; `graphs..embedding_provider`; `${NAME}` resolution at server boot | provider profile resolved from applied cluster state; legacy `omnigraph.yaml` untouched | +| later | ingest-time `@embed` (Shape C) | separate RFC | + +**Status:** Phases 1–5 are implemented (`@embed("…", model="…")` is recorded in the schema IR and validated at +query time with a typed same-space error; an unrecorded `@embed` keeps working with no check; cluster-served +graphs can bind an applied `providers.embedding` profile). Ingest-time `@embed` remains. + +## Invariants & deny-list check + +- **Invariant 9 (integrity failures are loud):** strengthened — query-time identity mismatch becomes a typed + error instead of silent wrong results. +- **Invariant 10 (query semantics are first-class IR concepts):** embedding identity becomes IR/catalog data, + not an out-of-band env guess. +- **Invariant 11 (transport stays at the boundary):** strengthened — Phase 1 removes the HTTP client + async + runtime (`reqwest`, `tokio`) from `omnigraph-compiler`, whose own manifest advertises "Zero Lance + dependency"; the embedding HTTP client lives only in the engine. +- **Invariant 12 / secret handling:** api-keys resolve through the existing credential chain; never in schema + or checked-in config. +- **Invariant 13 (bounded & observable):** addressed — the deadline bounds latency; tracing makes the + subsystem observable. +- **Deny-list — "silent fallback / dropped rows":** the cross-space ranking is exactly a silent-wrong-result; + this RFC closes it. +- **Deny-list — "new write paths that advance Lance HEAD before manifest publish without a recovery + sidecar":** the ingest phase (deferred) explicitly keeps embedding *before* staging, so it does not create a + new HEAD-advancing write path. No invariant is weakened. + +## Drawbacks & alternatives + +- **Do nothing.** The happy path works today, so the live risk is narrow (P1 + P3). But the provider hardwiring + and missing validation are a latent silent-wrong-results class that bites the first non-Gemini user. +- **Interim env-only provider switch (no schema record).** Cheaper, but leaves the same-space guarantee to + operator discipline (fails P3). Folded in as Phase 2's env-first resolution, with Phases 3–4 adding the + record/validate guarantee. +- **Trait-based provider plug-ins now.** Rejected as unearned complexity for two first-party providers; the + enum upgrades to a trait behind the same surface if needed. +- **Stamp identity in the manifest or Lance field metadata instead of the IR.** The manifest is the wrong + granularity; field metadata needs net-new wiring and a query-path dataset open. The IR is where `@embed` + already lives and is already read at query time (see spike). + +## Reversibility + +Mostly reversible. Phases 1–2 and 5 are code/config (env, CLI, cluster keys) and cheap to undo. Phase 3 +(recording identity in the schema IR) is **near-permanent** — it changes the on-disk `_schema.ir.json` shape +and the schema hash — so it earns the most scrutiny: the single-arg `@embed` form stays byte-compatible, and +recorded identity is additive (absent identity = today's behaviour). Provider request/response shapes are +external API contracts, not our format, so adding providers is reversible. + +## Gateway tradeoff (OpenRouter) + +Routing through OpenRouter (the default) buys provider-independence with one key and one billing relationship, +batch input, and access to the GA `google/gemini-embedding-2`. Costs to accept, all controllable: + +- **Extra network hop** → more query-path latency. The Phase-1 deadline bounds it; the cache mitigates repeats. +- **Text transits a third party.** OpenRouter's `provider: { data_collection }` routing preference controls + retention; shops with strict residency requirements use `kind: gemini`/`openai-compatible` pointed at the + provider (or a self-hosted endpoint) directly instead of the gateway. Provider-independence means this is a + config change, not a code change. +- **Loses Gemini's task-type asymmetry** when Gemini is reached via the OpenAI-compatible gateway (both sides + embed symmetrically). This is a retrieval-quality cost, **not** a same-space correctness cost — both stored + and query vectors take the identical path, so they stay in one space by construction. Shops that want the + asymmetry use `kind: gemini`. + +## Unresolved questions + +- GA Gemini model name — **resolved:** `google/gemini-embedding-2` (via OpenRouter) / `gemini-embedding-2` + (direct), 128–3072 dims (recommended 768/1536/3072). Default flips off `-preview` in Phase 2. +- Gemini `batchEmbedContents` availability — **moot** when going through the OpenAI-compatible gateway (its + `input` array batches); still relevant only for the direct `kind: gemini` path. +- Identity granularity: per-vector-property args vs one graph-level default profile referenced by name. +- Whether to backfill recorded identity for existing graphs, or treat absent-identity as "unvalidated, legacy" + permanently. +- Default model for the zero-config tier: `google/gemini-embedding-2` vs `openai/text-embedding-3-large` + (both 3072-capable) — pick the project default. diff --git a/docs/dev/testing.md b/docs/dev/testing.md index d2d08f3..8d6a305 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -7,7 +7,7 @@ This file is the always-on map of the test surface. **Consult it before every ta | Crate | Path | Style | |---|---|---| | `omnigraph` (engine) | `crates/omnigraph/tests/` | Integration tests (28 files), fixture-driven, share `tests/helpers/mod.rs` | -| `omnigraph-cli` | `crates/omnigraph-cli/tests/` | Per-area suites (post-modularization): `cli_cluster.rs` (cluster command surface + operator-actor cascade), `cli_cluster_e2e.rs` (spawned-binary lifecycle compositions — lost-state re-import recovery, out-of-band drift, graph-root destruction, multi-graph mixed-disposition convergence), `cli_data.rs` (load/read/change/branch/commit/export/snapshot/policy/embed/maintenance + operator format cascade), `cli_schema_config.rs` (init/config, schema plan/apply, RFC-008 deprecation warnings + `config migrate` + strict mode), `cli_queries.rs`, `system_local.rs` (full-cycle cluster lifecycle with a spawned `--cluster` server, applied-policy enforcement over HTTP, keyed-credential auth, operator aliases), `system_remote.rs`; share `tests/support/mod.rs` (hermetic `OMNIGRAPH_HOME` by default) | +| `omnigraph-cli` | `crates/omnigraph-cli/tests/` | Per-area suites (post-modularization): `cli_cluster.rs` (cluster command surface + operator-actor cascade), `cli_cluster_e2e.rs` (spawned-binary lifecycle compositions — lost-state re-import recovery, out-of-band drift, graph-root destruction, multi-graph mixed-disposition convergence), `cli_data.rs` (load/read/change/branch/commit/export/snapshot/policy/embed/maintenance + operator format cascade), `cli_schema_config.rs` (init/config, schema plan/apply), `cli_queries.rs`, `parity_matrix.rs` (RFC-009 Phase 1: the embedded-vs-remote referee — every forked verb run against both arms with matched Cedar policy and the same actor, scrubbed-JSON + exit-code equality; divergences are pinned in its `KNOWN_DIVERGENCES` ledger, never silently repaired), `system_local.rs` (full-cycle cluster lifecycle with a spawned `--cluster` server, applied-policy enforcement over HTTP, keyed-credential auth, operator aliases), `system_remote.rs`; share `tests/support/mod.rs` (hermetic `OMNIGRAPH_HOME` by default) | | `omnigraph-cluster` | mostly in-source `#[cfg(test)] mod tests`; `tests/failpoints.rs` (feature-gated); `tests/s3_cluster.rs` (bucket-gated full lifecycle on object storage) | Cluster config parser, local JSON state diff, state CAS/lock handling/recovery, read-only validate/plan/status plus explicit refresh/import graph observations, config-only apply (content-addressed payload publish, disposition gating, composite-digest convergence, idempotent re-apply), catalog payload verification (status read-only, refresh drift + self-heal), failpoint crash-mid-apply / CAS-race coverage, Stage 4A graph creation (create executor, recovery sidecars + sweep rows, create crash windows), Stage 4B schema apply (migration previews in plan, schema executor, schema-apply sweep classification, schema crash windows), Stage 4C gated deletes (digest-bound approvals, delete executor + tombstones, delete sweep rows, delete crash windows), and 5A policy binding metadata (applies_to in the applied revision, binding-change diffing + convergence, pre-5A backfill), and the 5B serving-snapshot read API (converged read, refusal rows) | | `omnigraph-server` | `crates/omnigraph-server/tests/` | Per-area suites (post-modularization): `auth_policy.rs`, `data_routes.rs`, `schema_routes.rs`, `stored_queries.rs`, `multi_graph.rs` (cluster-mode boot — converged serving, policy binding wiring, boot refusals — + the concurrent branch-ops matrix), `boot_settings.rs` (mode inference, PolicySource), `s3.rs` (bucket-gated: single-graph serving + config-free `--cluster s3://` boot), `openapi.rs` (OpenAPI drift / regeneration); share `tests/support/mod.rs` | | `omnigraph-compiler` | mostly in-source `#[cfg(test)] mod tests` | Parser, type-checker, IR lowering, lint | @@ -29,7 +29,7 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `point_in_time.rs` | Snapshots, time travel (`snapshot_at_version`, `entity_at`) | | `changes.rs` | `diff_between` / `diff_commits` | | `consistency.rs` | Cross-table snapshot isolation, atomic publish | -| `schema_apply.rs` | Migration plan + apply, schema-apply lock | +| `schema_apply.rs` | Migration plan + apply, schema-apply lock; index materialization deferred to the reconciler (iss-848): `apply_schema_defers_vector_index_on_empty_table` (an empty-table Vector `@index` never aborts the apply) and `index_only_constraint_apply_touches_no_table_data` (adding an `@index` is metadata-only — no table-version bump) | | `search.rs` | FTS / vector / hybrid (`bm25`, `nearest`, `rrf`) | | `traversal.rs` | `Expand`, variable-length hops, anti-join (CSR path — `OMNIGRAPH_TRAVERSAL_MODE` unset) | | `traversal_indexed.rs` | BTREE-indexed Expand (`execute_expand_indexed`) forced via `OMNIGRAPH_TRAVERSAL_MODE`, asserted semantically equal to the CSR path; own binary, all `#[serial]` so env writes never race | @@ -42,7 +42,7 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `lance_version_columns.rs` | Per-row `_row_last_updated_at_version` behavior | | `validators.rs` | Schema constraint enforcement (enum, range, unique, cardinality) across JSONL, insert, update paths | | `policy_engine_chassis.rs` | Engine-layer Cedar enforcement (MR-722): allow + deny through every `_as` writer via the SDK directly — no HTTP — proving embedded and CLI callers hit the same gate as the server, with action × scope shapes matching `authorize_request` | -| `maintenance.rs` | `optimize` (compaction), `repair` (explicit uncovered-drift publish), and `cleanup` (version GC): empty/idempotent/no-op edges, policy validation, head preservation; `optimize` publishes its own compaction (`optimize_publishes_compaction_to_manifest_so_schema_apply_succeeds`), skips pre-existing uncovered drift (`optimize_skips_preexisting_manifest_head_drift`), and refuses to run while a `__recovery` sidecar is pending (`optimize_defers_when_recovery_sidecar_is_pending`); `repair` previews/heals verified maintenance drift, refuses raw semantic drift without `--force`, and forced repair publishes only by explicit operator choice | +| `maintenance.rs` | `optimize` (compaction), `repair` (explicit uncovered-drift publish), and `cleanup` (version GC): empty/idempotent/no-op edges, policy validation, head preservation; `optimize` publishes its own compaction (`optimize_publishes_compaction_to_manifest_so_schema_apply_succeeds`), skips pre-existing uncovered drift (`optimize_skips_preexisting_manifest_head_drift`), and refuses to run while a `__recovery` sidecar is pending (`optimize_defers_when_recovery_sidecar_is_pending`); `repair` previews/heals verified maintenance drift, refuses raw semantic drift without `--force`, and forced repair publishes only by explicit operator choice; the index reconciler (iss-848): `index_build_tolerates_null_vector_rows` (an untrainable Vector column defers instead of aborting the build, sibling indexes still build) and `optimize_materializes_index_declared_but_unbuilt` (optimize creates a declared-but-deferred index) | | `failpoints.rs` | Failure-injection coverage (gated on `failpoints` feature). Includes the five per-writer Phase B → recovery integration tests (`recovery_rolls_forward_after_finalize_publisher_failure`, `schema_apply_phase_b_failure_recovered_on_next_open`, `branch_merge_phase_b_failure_recovered_on_next_open`, `ensure_indices_phase_b_failure_recovered_on_next_open`, `optimize_phase_b_failure_recovered_on_next_open`) and the write-entry in-process heal contract (the four `*_after_finalize_publisher_failure_heals_without_reopen` tests — load, mutation, schema apply, branch merge: a follow-up write on the same handle rolls a sidecar-covered residual forward without reopen/refresh) and the storage-fault matrix for the sidecar lifecycle (`recovery.sidecar_{write,delete,list}` / `recovery.record_audit` failpoints: Phase A put failure aborts with zero drift, Phase D delete failure is swallowed and healed by the next write, list failures are loud at heal and open, audit-append failures are retried to exactly one audit row; plus the bucket-gated `s3_load_recovers_after_publisher_failure_without_reopen`). | | `recovery.rs` | Open-time recovery sweep — sidecar I/O, classifier dispatch (NoMovement / RolledPastExpected / UnexpectedAtP1 / UnexpectedMultistep / InvariantViolation), all-or-nothing decision, roll-forward via `ManifestBatchPublisher::publish`, roll-back via `Dataset::restore`, audit row in `_graph_commit_recoveries.lance`, `OpenMode::ReadOnly` skip path | | `composite_flow.rs` | Compositional/narrative end-to-end stories — multi-step flows that compose mechanics covered by other test files. Catches integration regressions where individual operations all pass their unit tests but their composition breaks (sequential merges, post-merge main writes, time-travel through merge DAG, reopen consistency over multi-merge histories, post-optimize and post-cleanup strict writes). | diff --git a/docs/dev/writes.md b/docs/dev/writes.md index 82d6ba8..01c166e 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -19,8 +19,14 @@ publisher's row-level CAS on `__manifest` is the single fence. `__run__*` branch on an upgraded graph is swept off `__manifest` by the v2→v3 internal-schema migration on first read-write open. (The inert `_graph_runs.lance` bytes remain until a `delete_prefix` primitive lands.) -- Cancelled mutation futures leave **no graph-level state** — only orphaned - Lance fragments, which the existing `omnigraph cleanup` pipe reclaims. +- Cancelled mutation futures leave **no graph-visible state** — the manifest + is never advanced. They can leave two kinds of unreferenced residue, both + self-healing: orphaned Lance fragments (reclaimed by `omnigraph cleanup`), + and — on the *first* write to a table on a branch, which forks it before the + publish — a manifest-unreferenced branch ref. The next write to that table + reclaims the stale fork and re-forks (`reclaim_orphaned_fork_and_refork`), + and `cleanup`'s per-table reconciler is the guaranteed backstop; see the + fork-reclaim note in [invariants.md](invariants.md). ## Read-your-writes within a multi-statement mutation @@ -80,10 +86,17 @@ deferred to a follow-up cycle — tracked). Three writers have been migrated onto staged primitives: * **`ensure_indices`** (`db/omnigraph/table_ops.rs::build_indices_on_dataset_for_catalog`) - — scalar indices (BTree, Inverted) now use `stage_create_*_index` + - `commit_staged`. Vector indices stay inline (residual — Lance - `build_index_metadata_from_segments` is `pub(crate)` in 6.0.1; - companion ticket to lance-format/lance#6658 needed). + — scalar indices (BTree, Inverted) use `stage_create_*_index` + + `commit_staged`. Which index a `@index`/`@key` property gets is dispatched by + type via `node_prop_index_kind` (enum + orderable scalar → BTree, free-text + String → Inverted/FTS, Vector → vector). Vector indices stay inline (residual + — Lance `build_index_metadata_from_segments` is `pub(crate)` in 6.0.1; + companion ticket to lance-format/lance#6658 needed). This build is + existence-gated (it creates a *missing* index over current fragments); folding + fragments appended afterward into an *existing* index is `optimize`'s + `optimize_indices` pass — an inline-commit residual, not a staged write (Lance + exposes no uncommitted index-optimize), covered by the optimize recovery + sidecar (see [maintenance.md](../user/operations/maintenance.md)). * **`branch_merge::publish_rewritten_merge_table`** (`exec/merge.rs`) — merge_insert now uses `stage_merge_insert` + `commit_staged`. Deletes stay inline (Lance #6658 residual). @@ -305,7 +318,7 @@ success and one failure. The losing writer's error is `ManifestConflictDetails::ExpectedVersionMismatch { table_key, expected, actual }`. The HTTP server maps this to **409 Conflict** with body `{"error": "...", "code": "conflict", "manifest_conflict": { "table_key": -"...", "expected": N, "actual": M }}` — see [docs/user/server.md](../user/server.md). +"...", "expected": N, "actual": M }}` — see [docs/user/server.md](../user/operations/server.md). ## Audit diff --git a/docs/releases/v0.7.0.md b/docs/releases/v0.7.0.md index 4048041..b4ad903 100644 --- a/docs/releases/v0.7.0.md +++ b/docs/releases/v0.7.0.md @@ -1,90 +1,294 @@ # Omnigraph v0.7.0 -v0.7.0 takes the cluster control plane to object storage and overhauls the -configuration architecture around two single-owner surfaces. A cluster — -state ledger, content-addressed catalog, and graph data — can now live -entirely on an S3-compatible bucket, and a server can boot from that bucket -with no local files at all. Operator identity, credentials, and personal -aliases move to a home-level config; the legacy combined `omnigraph.yaml` -enters a guided, staged deprecation. +v0.7.0 is three large arcs in one release. **Operations:** the cluster control +plane moves to object storage and the configuration architecture collapses to two +single-owner surfaces — a cluster can live entirely on an S3-compatible bucket, a +server boots from it with no local files, and the legacy combined `omnigraph.yaml` +is **removed**. **CLI:** the command-line surface is unified and made honest — +embedded and remote runs are one execution path, `load` becomes the single +bulk-write command, every command declares the **capability** it needs (and +rejects flags that don't apply), and the server boots only from a cluster. +**Engine & substrate:** Lance moves to 7.x, traversal/index/recovery internals +get faster and self-healing, and text embedding becomes provider-independent. ## Highlights -- **Clusters on object storage (`storage:`).** `cluster.yaml` gains an - optional `storage: s3://bucket/prefix` root. Every stored byte — the - state ledger, lock, recovery sidecars, approval artifacts, catalog blobs, - and the derived graph roots (`/graphs/.omni`) — flows - through one storage layer, so `file://` (the default, byte-compatible - with existing clusters) and `s3://` are a single code path. The ledger's - compare-and-swap uses S3 conditional writes (`If-Match` / - `If-None-Match`), verified against AWS semantics, RustFS, and - Tigris-backed stores; the state lock is genuinely cross-machine on - object storage. -- **Config-free serving: `--cluster s3://bucket/prefix`.** The server - accepts a bare storage-root URI and boots from the applied revision on - the bucket — the ledger and catalog are the whole deployment artifact. - Policy bundles serve as digest-verified *content* from the catalog - (never re-read from disk), closing the last gap for fully remote - clusters. The preferred container shape becomes **bucket, no volume** - (see `docs/user/deployment.md`). -- **Per-operator configuration (`~/.omnigraph/`).** A home-level config - carries operator identity (`operator.actor`, the new last hop of the - `--as` chain), output defaults, named servers, and personal aliases. - `$OMNIGRAPH_HOME` relocates it; `$OMNIGRAPH_CONFIG` now stands in for - `--config` in both binaries. -- **Credentials keyed by server name.** `omnigraph login ` stores - a bearer token in `~/.omnigraph/credentials` (created `0600`; over- - permissive files are refused). Token resolution for a request whose URL - matches an operator-defined server: `OMNIGRAPH_TOKEN_` env → the - credentials file → the legacy `bearer_token_env` chain unchanged. A - token is only ever sent to the server it is keyed to. -- **Operator targeting and aliases.** `--server ` (with `--graph - ` for multi-graph servers) addresses operator-defined endpoints on - every remote-capable command. Operator aliases are pure *bindings* — - personal name → (server, graph, stored-query name, default params) — - invoking catalog-owned stored queries; they carry no query content. -- **`omnigraph.yaml` deprecation begins.** Loading the legacy file prints - a per-key notice naming each present key's new home - (`OMNIGRAPH_SUPPRESS_YAML_DEPRECATION=1` to silence in CI). - `omnigraph config migrate` proposes — and with `--write`, applies — the - split: team half to a ready-to-review `cluster.yaml`, personal half - merged into the operator config (existing entries always win). - `omnigraph init` no longer scaffolds the file. Migrated teams can set - `OMNIGRAPH_NO_LEGACY_CONFIG=1` to turn any legacy-file load into a hard - error. The file itself keeps working until its removal at the next - major version. +### Clusters & storage on object storage -## Breaking / behavior changes +- **Clusters on object storage (`storage:`).** `cluster.yaml` gains an optional + `storage: s3://bucket/prefix` root. Every stored byte — state ledger, lock, + recovery sidecars, approval artifacts, catalog blobs, and the derived graph + roots (`/graphs/.omni`) — flows through one storage layer, so + `file://` (the default, byte-compatible with existing clusters) and `s3://` + are a single code path. The ledger's compare-and-swap uses S3 conditional + writes (`If-Match`/`If-None-Match`), verified against AWS, RustFS, and other + S3-compatible stores; the state lock is genuinely cross-machine on object + storage. +- **Config-free serving: `--cluster s3://bucket/prefix`.** The server accepts a + bare storage-root URI and boots from the applied revision on the bucket — the + ledger and catalog are the whole deployment artifact. Policy bundles serve as + digest-verified *content* from the catalog (never re-read from disk). The + preferred container shape becomes **bucket, no volume** (see + `docs/user/deployment.md`). +- **Cluster-only server.** `omnigraph-server` boots **only** from `--cluster + ` and serves N graphs (N ≥ 1) under cluster routes + (`/graphs/{id}/…`, plus a read-only `GET /graphs` enumeration). The old + single-graph flat-route mode, positional-`` boot, and `omnigraph.yaml` + `graphs:`-map boot are gone — add or remove graphs with `cluster apply` and + restart. +- **One storage substrate + recovery liveness.** The cluster storage backend and + the engine both go through one `StorageAdapter` (versioned read, conditional + replace/CAS, prefix delete), exercised by a storage fault-injection matrix. + A long-lived server now heals a recoverable write on its *next write* rather + than only at restart. -- `omnigraph init` no longer writes an `omnigraph.yaml` into the working - directory. Start cluster configs from the documentation templates, or - run `omnigraph config migrate` against an existing legacy file. -- Loading a legacy `omnigraph.yaml` now emits a deprecation block on - stderr (suppressible; see above). Output on stdout is unchanged. -- `ServingPolicy` (cluster crate API) carries verified policy *content* - instead of a blob path; `read_serving_snapshot` and several cluster - command entry points are now async. +### Configuration: two single-owner surfaces + +The legacy combined `omnigraph.yaml` is **removed**. Configuration now lives in +two surfaces with single owners, plus a zero-config tier: + +- **Cluster config (`cluster.yaml` + checkout, team-owned)** declares what the + system *is*: graphs, schemas, stored queries, policies, storage. A server boots + from it via `--cluster`. +- **Per-operator config (`~/.omnigraph/config.yaml`, person-owned)** declares who + *you* are: `operator.actor` (the last hop of the `--as` chain), output + defaults, named servers + clusters, profiles, aliases, and a default scope. + `$OMNIGRAPH_HOME` relocates it. +- **Credentials keyed by server name.** `omnigraph login ` stores a + bearer token in `~/.omnigraph/credentials` (created `0600`; over-permissive + files refused). Resolution for a request whose URL matches an operator-defined + server: `OMNIGRAPH_TOKEN_` env → the credentials file → the default + `OMNIGRAPH_BEARER_TOKEN`. A token is only ever sent to the server it is keyed to. +- **Operator targeting and aliases.** `--server ` (with `--graph ` for + multi-graph servers) addresses operator-defined endpoints. Operator aliases are + pure, **read-only** *bindings* — personal name → (server, graph, stored-query + name, default params) — invoking catalog-owned stored queries; they carry no + query content and a binding to a stored mutation is rejected. +- **Default scopes.** `defaults.server` (served) or `defaults.store` (a zero-flag + *local* default — mutually exclusive with `server`) supply the no-flag scope, + with an optional `default_graph`. `--profile ` / `$OMNIGRAPH_PROFILE` + selects a named scope bundle wholesale; `omnigraph profile list` / + `profile show []` inspect what's defined (read-only). + +### Unified, capability-aware CLI + +- **One bulk-write command: `omnigraph load`.** `load` is now the single data-write + command and works against remote graphs (over HTTP with the same bearer/actor + resolution as every other remote command) — previously the only data command + forced to open storage directly. `--mode overwrite|append|merge` is **required** + (overwrite is destructive, so there is no default); `--from ` opts into + fork-if-missing for `--branch`. `omnigraph ingest` becomes a **deprecated + alias** (`--from main --mode merge` defaults; one-line stderr warning). +- **No implicit branch forks.** Loading into a branch that does not exist is an + **error** unless `--from ` is given — a typo'd branch name no longer + silently forks `main` and lands your data there. Same rule on the server. +- **One execution path, embedded ≡ remote.** Every CLI verb runs through one + `GraphClient` with two implementations (embedded engine, HTTP) sharing a single + wire-DTO crate (`omnigraph-api-types`). An executable parity matrix runs every + verb against both and asserts identical results, so local and remote no longer + drift. +- **Declared capabilities + honest addressing.** Every command declares the + **capability** it needs — `any` (run against a graph, served or embedded), + `served` (needs a server), `direct` (direct storage access), `control` + (manage/inspect a cluster), or `local` (no graph) — and the CLI enforces it. + Wrong-capability addressing now fails loudly with a declared message (e.g. + `--server` on `optimize`) instead of being silently ignored, and a maintenance + verb pointed at a remote target is rejected. `omnigraph --help` groups commands + by capability with a legend. +- **Address cluster graphs for maintenance.** `optimize` / `repair` / `cleanup` + accept `--cluster --graph ` (`--cluster` is a cluster directory, + storage-root URI, or a `clusters:` name from `~/.omnigraph/config.yaml`), + resolving the graph's storage URI from the served cluster state (no need to + hand-type `/graphs/.omni`). `--graph` is the single graph selector + across server and cluster scopes. Conversely, `omnigraph init` **refuses** a + cluster-managed path and points at `cluster apply` — graphs in a cluster are + created with ledger/recovery/approvals, not by hand. `schema apply` refuses a + cluster-managed graph for the same reason (and the server rejects a cluster- + backed schema apply with `409`, pointing at `cluster apply`). +- **Write diagnostics + destructive-write safety (RFC-011 Decision 9).** Every + write (`load`, `mutate`, `branch create|delete|merge`, `schema apply`, + `optimize`, `repair`, `cleanup`) echoes its resolved target + access path to + stderr — e.g. `omnigraph load → s3://…/knowledge.omni (direct, remote)` — + suppressible with the global `--quiet`. Destructive writes against a + **non-local** scope (`cleanup`, overwrite `load`, `branch delete` against an + `http(s)://` server or `s3://` store/cluster) require explicit consent: the + global `--yes`, an interactive TTY prompt, or — for a non-interactive / + `--json` run — a hard refusal instead of silently proceeding. Local (`file://`) + writes are unaffected. +- **Route alignment: canonical `POST /load`.** The server gains a canonical + `POST /load`; `POST /ingest` is now a deprecated alias that emits RFC 9745 + `Deprecation: true` + RFC 8288 `Link: ; rel="successor-version"` + headers (a sibling-relative reference that resolves under `/graphs/{id}/…`). + The CLI's `load` targets `/load`. +- **Operator aliases get their own namespace (`omnigraph alias `).** A + personal binding to a stored query on a named server is invoked as + `omnigraph alias [args]` (RFC-011 Decision 4), so an alias can never + shadow — or be shadowed by — a built-in verb. `alias` rejects global scope + flags (`--server`/`--graph`/`--store`/`--cluster`/`--profile`/`--as`) its + binding already owns. +- **No-graph addressing lists candidates (RFC-011 Decision 7).** When a scope + has no `--graph` and no `default_graph`, the CLI never silently picks. A + **cluster** scope with exactly one applied graph uses it automatically and + otherwise **lists the candidates** (from the served catalog). A multi-graph + **server** lists the candidates (from `GET /graphs`) and requires `--graph `. +- **Invoke stored queries by name (RFC-011 Decision 3).** `omnigraph query + ` / `mutate ` invoke a stored query **by name** from the served + catalog — `omnigraph query find_people` instead of `--query find.gq --name + find_people`. The verb asserts the query's kind (an `expect_mutation` flag on + `POST /queries/{name}`: `query ` is rejected with `'' is a + mutation — use omnigraph mutate `, and vice-versa). `.gq` files become + the explicit ad-hoc lane (`-e` / `--query`), with the positional selecting + which query in the source. + +### Engine & substrate + +- **Lance 6.0.1 → 7.0.0.** The columnar substrate is bumped to Lance 7.x with + correct-by-design alignment: the unenforced primary key is immutable once set, + `WriteParams::auto_cleanup` is disabled so version GC stays operator-owned, and + the native namespace/`object_store` 0.13 surface is pinned by surface-guard + tests. No on-disk format change for existing graphs. +- **Indexed graph traversal.** `Expand` can run over a BTREE-indexed path, + asserted semantically equal to the CSR traversal it accelerates. +- **Scalar index coverage + filter literal coercion.** Closes index-coverage gaps + and coerces filter literals correctly, cutting query latency on indexed scans. +- **Index materialization is derived state.** `schema apply` records + `@index`/`@key` *intent* and builds nothing (index-only changes touch no table + data); `load`/`mutate` build inline through one chokepoint but **defer** an + untrainable Vector column as *pending* instead of aborting; `optimize` is the + reconciler that materializes declared-but-missing indexes and folds appended + fragments back into existing ones. +- **Recovery liveness + one storage substrate.** Writers heal a recoverable + write on the *next write* (not only at the next read-write open); a storage + fault-injection matrix exercises the sidecar lifecycle; the cluster and engine + share one `StorageAdapter` over `object_store`. +- **Branch-fork self-heal.** Manifest-unreferenced branch forks are reclaimed + (eager best-effort + a `cleanup` reconciler backstop), so a failed branch-delete + reclaim no longer wedges a reused branch name. +- **Composite `@unique(a, b)`.** Enforced as a true composite key, with one shared + keying function for intake and branch-merge that fails loudly on an un-keyable + column type rather than silently exempting it. + +### Embeddings: provider-independent (RFC-012) + +- **One client, any provider.** Text embedding moves to a single + provider-independent `EmbeddingConfig` behind a sealed `Provider` enum: + **OpenAI-compatible** (the **OpenRouter** default gateway — one key for many + models — plus OpenAI-direct and self-hosted endpoints), native **Gemini**, and + a deterministic **Mock**. One client serves both the query path and the offline + `omnigraph embed` CLI, with a per-query deadline and `tracing` observability. + The dead, uncallable compiler-crate OpenAI client (and its `reqwest`/`tokio` + deps) was removed. +- **Same-space guarantee.** `@embed("source", model="…")` records the embedding + identity (model) in the schema IR so it travels with the data; a string + `nearest()` whose resolved embedder model differs from the recorded one is + **rejected with a typed error** instead of silently ranking across vector + spaces. (`@embed` still does no ingest-time embedding — deferred to a later + phase.) + +## Breaking & behavior changes + +- **`omnigraph.yaml` is removed.** The CLI and server no longer read it at all; + the `OmnigraphConfig` type, `omnigraph config migrate`, and the deprecation + env vars (`OMNIGRAPH_NO_LEGACY_CONFIG`, `OMNIGRAPH_SUPPRESS_YAML_DEPRECATION`, + `OMNIGRAPH_CONFIG`) are gone. Configure via a team `cluster.yaml` and a + per-operator `~/.omnigraph/config.yaml` (see Upgrade notes). +- **`omnigraph-server` boots only from `--cluster`.** The positional-`` + single-graph boot and the `omnigraph.yaml` `graphs:`-map boot are removed; all + HTTP is under `/graphs/{id}/…` (with flat `/healthz` and the `/graphs` + enumeration). Upgrade deployments to `omnigraph-server --cluster `. +- **Default embedding provider flips to OpenRouter.** Embedding is no longer + hardwired to Gemini: the default provider is **OpenAI-compatible via + OpenRouter**, `OMNIGRAPH_GEMINI_BASE_URL` is dropped, and Gemini-direct users + must set `OMNIGRAPH_EMBED_PROVIDER=gemini`. A `nearest("string")` query whose + resolved model differs from a property's recorded `@embed(model=…)` is now a + typed error rather than silent cross-space ranking. +- **`query --alias ` is removed.** Invoke operator aliases via + `omnigraph alias [args]`. +- **`query`/`mutate` no longer take a positional graph URI, `--uri`, or + `--name`** (RFC-011 D3). The positional is now the query name; address the + graph with `--store` (local) / `--server` / `--profile`, and select a query + within an ad-hoc `--query`/`-e` source with the positional (replacing + `--name`). By-name catalog invocation is **served-only** (a bare `--store` has + no catalog — use `-e`/`--query` there). Scripts using + `query --query f.gq --name q` become + `query --store --query f.gq q`. +- **Legacy data-plane addressing removed** (#238): `--target`, the positional + `http(s)://`→remote dispatch, and `--as` on a served write (the actor is + resolved server-side from the bearer token) no longer exist. +- **`omnigraph load` replaces direct-storage-only loading; `--mode` is required.** + Scripts calling `load` without `--mode` must add one (`overwrite|append|merge`). +- **`omnigraph ingest` is deprecated** (still works; one-line stderr warning). + Use `load --from --mode `. +- **Loading into a missing branch is now an error without `--from`** (CLI and + `POST /load`/`POST /ingest`): a missing branch returns 404 / fails, never an + implicit fork. Pass `--from ` (CLI) or the request `from` field (HTTP) to + fork-if-missing. This affects any workflow that relied on auto-forking. +- **Scope flags that can't apply now error instead of being silently ignored.** + `--server` on any direct/control/session command, `--cluster` outside the + cluster-scoped verbs, and `--graph` where no multi-graph scope applies all fail + with a declared message. `--graph` is the single graph selector and is + **accepted** on `optimize` / `repair` / `cleanup` when paired with `--cluster` + (replacing the removed `--cluster-graph`). +- **`schema apply` is refused against a cluster-managed graph.** The CLI signposts + `omnigraph cluster apply`; a cluster-backed server returns `409 Conflict` + (after the Cedar gate, so an unauthorized actor still gets `403`). Cluster + graphs evolve through `cluster apply`, never a direct apply. +- **Storage-plane error text changed.** A maintenance verb pointed at a remote + target now fails with a declared direct-capability message (replacing the older + "only supported against local graph URIs" wording). Error strings are observable + contract (Hyrum); pin against the new text. +- **Non-local destructive writes now require `--yes` in automation.** A + `cleanup` / overwrite-`load` / `branch delete` against an `http(s)://` or + `s3://` target with `--json` (or any non-TTY context) previously executed; + it now **refuses** unless `--yes` is passed. CI scripts that destroy remote + data must add `--yes`. Local (`file://`) writes are unchanged. +- **`omnigraph init` no longer scaffolds a config file,** and **refuses a + cluster-managed storage path** (`/graphs/.omni` under a cluster) — + create those graphs with `cluster apply`. +- **`POST /ingest` is deprecated** (kept indefinitely as a shim) and returns + `Deprecation`/`Link` headers. **A v0.7 CLI talks to `POST /load`,** which a + pre-0.7 server does not expose — upgrade the server and CLI together, or keep + using `ingest`. +- **`ServingPolicy` (cluster crate API) carries verified policy content instead + of a blob path; `read_serving_snapshot` and several cluster command entry points + are now `async`.** +- **`omnigraph --help` reorders commands** (grouped by capability) and **hides + the deprecated `ingest`** from the listing — `ingest` still runs. Help text is + observable; this is a deliberate output change. ## Upgrade notes - Existing clusters need no migration: an absent `storage:` key keeps the config-directory layout byte-for-byte. -- Existing `omnigraph.yaml` setups keep working through the deprecation - window; `omnigraph config migrate` produces the recommended split. -- Operator setup is three lines: - `mkdir -p ~/.omnigraph`, write `operator.actor` (and `servers:`) into - `~/.omnigraph/config.yaml`, then `echo $TOKEN | omnigraph login `. +- **`omnigraph.yaml` is no longer read.** There is no automated migrate command + in 0.7.0; recreate configuration as a team `cluster.yaml` (graphs, schemas, + stored queries, policies — see `docs/user/clusters/`) plus a per-operator + `~/.omnigraph/config.yaml` (identity, servers, credentials, defaults — see + `docs/user/cli/reference.md`). +- **`omnigraph-server` now requires `--cluster `** — there is no + positional-URI boot. Run `cluster apply` first, then serve the applied revision. +- **Gemini-direct embedding users** set `OMNIGRAPH_EMBED_PROVIDER=gemini` (the + default is now OpenRouter); `OMNIGRAPH_GEMINI_BASE_URL` is removed. +- Audit scripts for two CLI changes: add `--mode` to every `load`, and add + `--from ` anywhere you relied on a missing branch being auto-created. +- Upgrade server and CLI together for the `/load` route (or keep `ingest`). +- Operator setup is three lines: `mkdir -p ~/.omnigraph`, write `operator.actor` + (and `servers:`) into `~/.omnigraph/config.yaml`, then + `echo $TOKEN | omnigraph login `. ## Internals -- The cluster, server, and CLI crates were modularized (the 7.9k-line - cluster `lib.rs` is now eight focused modules; the server and CLI test - monoliths split into per-area suites) — pure code movement, no behavior - change. -- New gated end-to-end suites run the full cluster lifecycle against a - real S3-compatible store in CI, including a lock-release regression and - a config-free server boot from a bare bucket URI. -- The deployment guide gains the bucket-no-volume container recipe for - AWS and Railway, validated against a live Railway deployment - (Railway buckets are S3-compatible and pass the conditional-write - contract test). +- The cluster, server, and CLI crates were modularized (the ~7.9k-line cluster + `lib.rs` into focused modules; the server and CLI test monoliths into per-area + suites) — pure code movement. +- The parity matrix (embedded vs remote) is the new referee for CLI behavior; the + OpenAPI drift test guards `openapi.json`; Lance-surface guard tests pin the + upstream APIs the engine depends on (the first smoke check on a Lance bump). +- Gated end-to-end suites run the full cluster lifecycle against a real + S3-compatible store in CI (lock-release regression, config-free boot from a + bare bucket URI). +- The deployment guide gains the bucket-no-volume container recipe for AWS / + S3-compatible object storage. +- `clap` updated to 4.6.1. CI runs the full workspace suite on `main` post-merge + rather than on every PR (faster PR turnaround; the local + `cargo test --workspace --locked` is the pre-merge gate). diff --git a/docs/user/audit.md b/docs/user/audit.md deleted file mode 100644 index 845c2e0..0000000 --- a/docs/user/audit.md +++ /dev/null @@ -1,7 +0,0 @@ -# Audit / Actor tracking - -- `Omnigraph::audit_actor_id: Option` is the actor in effect. -- `_as` variants of every write API let callers override the actor: `mutate_as`, `load_as`, `branch_merge_as`, `apply_schema_as`, etc. -- Actor IDs are persisted on `GraphCommit.actor_id` with split storage in `_graph_commit_actors.lance` (the commit graph is split into `_graph_commits.lance` for the linkage and `_graph_commit_actors.lance` for the actor map). -- HTTP server uses the bearer-token actor automatically. The CLI resolves one actor chain everywhere: `--as` > legacy `cli.actor` in `omnigraph.yaml` > `operator.actor` in `~/.omnigraph/config.yaml` > none (RFC-007). -- Pre-v0.4.0 graphs also stored actor IDs on `RunRecord.actor_id` in `_graph_runs.lance` / `_graph_run_actors.lance`. The Run state machine was removed in MR-771; those files are inert post-v0.4.0. The v2→v3 manifest migration sweeps any stale `__run__*` branches on first write-open (MR-770); the inert dataset bytes remain until a `delete_prefix` primitive lands. diff --git a/docs/user/branches-commits.md b/docs/user/branches-commits.md deleted file mode 100644 index a4044cb..0000000 --- a/docs/user/branches-commits.md +++ /dev/null @@ -1,63 +0,0 @@ -# Branches, Commits, Snapshots - -## L1 — Lance per-dataset branches - -Lance supports branching at the dataset level: a branch is a named lineage of versions, and `fork_branch_from_state(source_branch, target_branch, source_version)` creates a copy-on-write fork. - -## L2 — Graph-level branches - -OmniGraph builds *graph branches* on top by branching every sub-table coherently: - -- `branch_create(name)` / `branch_create_from(target, name)` — disallowed name `main`; fails if branch exists; ensures the schema-apply lock is idle. Atomic and authority-first like `branch_delete`: it flips the `__manifest` branch (authority), then creates the derived commit-graph branch, force-dropping any orphaned commit-graph ref left by an incomplete prior delete (the manifest branch is fresh, so a same-named commit-graph branch is provably a zombie). If commit-graph creation fails, the manifest branch is rolled back so the name never half-exists. -- `branch_list()` — returns public branches, **filters the internal** `__schema_apply_lock__` branch. -- `branch_delete(name)` — refuses if there are descendants on the branch, or if it is the current branch. The manifest is the single authority for branch existence: deletion flips the `__manifest` branch ref first (one atomic op), after which the branch is gone from every snapshot. The owned per-table forks and the commit-graph branch are derived state, reclaimed best-effort with `force_delete_branch` after the flip. A failure during that reclaim (transient object-store error) does not fail the call or block the authority flip; the leftover forks are unreachable orphans that the [`cleanup`](maintenance.md) reconciler converges. One consequence: if a delete's best-effort reclaim fails, reusing that branch name before the next `cleanup` surfaces a clear error pointing at `cleanup` (the stale fork would otherwise collide on first write). -- **Lazy forking**: a branch only forks a sub-table when that sub-table is first mutated on it. Pure-read branches share fragments with their source. A fork collision is classified by the manifest authority, not by Lance branch versions: if the live manifest already records the fork on the active branch, a concurrent first-write won and the caller gets a retryable "refresh and retry"; if the manifest does not, a physical branch there is an orphan and the caller is pointed at `cleanup`. -- `sync_branch(branch)` — re-binds the in-memory handle to the latest head of the branch. - -## L2 — Commit graph (`db/commit_graph.rs`) - -In-memory shape of a graph commit: - -``` -GraphCommit { - graph_commit_id: ULID, - manifest_branch: Option, - manifest_version: u64, - parent_commit_id: Option, - merged_parent_commit_id: Option, // populated for merge commits - actor_id: Option, // joined in-memory from _graph_commit_actors.lance, NOT a column on _graph_commits.lance - created_at: i64 (microseconds since epoch), -} -``` - -Storage is split across two Lance datasets (both with stable row IDs): - -- `_graph_commits.lance` — every column above *except* `actor_id`. -- `_graph_commit_actors.lance` — optional separate `(graph_commit_id, actor_id)` map, created on demand. The `actor_id` field above is populated by joining this dataset in-memory at load time. - -Notes: - -- Every successful publish (load / change / merge / schema_apply) appends one commit. -- Merge commits have two parents; linear commits have one. -- API: `list_commits(branch)`, `get_commit(id)`, `head_commit_id_for_branch(branch)`. - -## L2 — Snapshots & time travel - -- `snapshot()` — current snapshot for the bound branch; cached. -- `snapshot_of(target)` — snapshot at a `ReadTarget` (branch | snapshot id). -- `snapshot_at_version(v: u64)` — historical snapshot from any manifest version. -- `entity_at(table_key, id, version)` — single-entity time travel without building a full snapshot. -- A `Snapshot` is a `(version, HashMap)` — cheap to build, snapshot-isolated cross-table reads. - -## L2 — Internal system branches - -Internal or legacy branch refs: - -- `__schema_apply_lock__` — serializes schema migrations; filtered from `branch_list()` but visible to internals. -- `__run__` — legacy from the pre-v0.4.0 Run state machine (removed in MR-771). These are swept off `__manifest` on the first read-write open by the v2→v3 internal-schema migration (MR-770), and `__run__*` is no longer a reserved name. Known limitation: a pre-v0.4.0 graph opened **read-only** still surfaces any stale `__run__*` branch in `branch_list()` until its first read-write open (the migration is write-path-only, like all manifest migrations). - -## L2 — Recovery audit trail - -The five migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`, `optimize_all_tables`) protect their multi-table commits with a sidecar at `__recovery/{ulid}.json` written before Phase B and deleted after Phase C. The next `Omnigraph::open` (gated on `OpenMode::ReadWrite`) runs the recovery sweep in `crates/omnigraph/src/db/manifest/recovery.rs`: classify per-table state, decide all-or-nothing per sidecar, roll forward / back, record an audit row. - -Audit rows live in `_graph_commit_recoveries.lance` (sibling to `_graph_commits.lance`) and reference the commit graph by `graph_commit_id`. The linked recovery commit is identified by that same `graph_commit_id`, and `actor_id="omnigraph:recovery"` is stored in `_graph_commit_actors.lance` (joined by `graph_commit_id`) — `_graph_commits.lance` itself does not carry the `actor_id` column. To find recoveries for a specific original actor: `omnigraph commit list --filter actor=omnigraph:recovery`, then join to `_graph_commit_recoveries.lance` by `graph_commit_id` to read `recovery_for_actor`. Schema: see `crates/omnigraph/src/db/recovery_audit.rs`. diff --git a/docs/user/changes.md b/docs/user/branching/changes.md similarity index 93% rename from docs/user/changes.md rename to docs/user/branching/changes.md index 58739e2..a9bceec 100644 --- a/docs/user/changes.md +++ b/docs/user/branching/changes.md @@ -1,6 +1,6 @@ # Change Detection / Diff -`changes/mod.rs`. Three-level algorithm: +Diffing two read targets uses a three-level algorithm: 1. **Manifest diff**: skip sub-tables whose `(table_version, table_branch)` is unchanged. 2. **Lineage check**: diff --git a/docs/user/branching/index.md b/docs/user/branching/index.md new file mode 100644 index 0000000..20ea125 --- /dev/null +++ b/docs/user/branching/index.md @@ -0,0 +1,40 @@ +# Branches, Commits, Snapshots + +## L1 — Lance per-dataset branches + +Lance supports branching at the dataset level: a branch is a named lineage of versions, and a copy-on-write fork creates a new branch from a source branch at a given version. + +## L2 — Graph-level branches + +OmniGraph builds *graph branches* on top by branching every sub-table coherently: + +- **Create** (`branch create` / `branch create --from `) — the name `main` is disallowed; fails if the branch exists. Atomic: the new branch becomes visible all-or-nothing, so a name never half-exists. +- **List** (`branch list`) — returns public branches, **filtering the internal** `__schema_apply_lock__` branch. +- **Delete** (`branch delete`) — refuses if there are descendants on the branch, or if it is the current branch. Once deleted, the branch is gone from every snapshot. The owned per-table forks are reclaimed best-effort; if that reclaim hits a transient object-store error, the leftover storage is reclaimed later by the [`cleanup`](../operations/maintenance.md) command. One consequence: if a delete's reclaim fails, reusing that branch name before the next `cleanup` surfaces a clear error pointing at `cleanup`. +- **Lazy forking**: a branch only forks a sub-table when that sub-table is first mutated on it. Pure-read branches share storage with their source. If two writers race to first-write the same branch, the loser gets a retryable "refresh and retry". + +## L2 — Commit graph + +Each graph commit carries a ULID id, the manifest branch and version it published, its parent commit (two parents for a merge commit, one for a linear commit), the actor who made it, and a creation timestamp. + +- Every successful publish (load / change / merge / schema apply) appends one commit. +- Merge commits have two parents; linear commits have one. +- Inspect history with `commit list` and `commit show`. + +## L2 — Snapshots & time travel + +Reading a branch at a past version, or a single entity at a past version, is +covered on the [time travel](time-travel.md) page. Merging branches and the +conflict kinds are on the [merge](merge.md) page. + +## L2 — Internal system branches + +- `__schema_apply_lock__` — serializes schema migrations; filtered from `branch list` but used internally. + +## L2 — Recovery audit trail + +Interrupted multi-table writes are recovered automatically the next time the graph is opened read-write. Recovery commits are recorded in the audit trail under the actor `omnigraph:recovery`, so you can find them with: + +```bash +omnigraph commit list --filter actor=omnigraph:recovery +``` diff --git a/docs/user/branching/merge.md b/docs/user/branching/merge.md new file mode 100644 index 0000000..fde2fab --- /dev/null +++ b/docs/user/branching/merge.md @@ -0,0 +1,47 @@ +# Merging Branches + +Merging integrates the changes on one branch into another. OmniGraph merges are +**three-way and row-level**: it compares both branches against their common +ancestor and merges each node/edge table row by row, then publishes the result as +**one atomic commit** across the whole graph. + +```bash +omnigraph branch merge review/2026-04-25 --into main s3://bucket/graph.omni +``` + +`branch merge [--into ]` merges `` into `` +(default `main`). + +## Outcomes + +A merge resolves to one of three outcomes: + +- **Already up to date** — the target already contains every change on the source; + nothing to do. +- **Fast-forward** — the target has no changes the source lacks, so the target + simply advances to the source. +- **Merged** — both sides diverged; a new merge commit is created with two parents. + +## Conflicts + +When both branches changed the same data incompatibly, the merge fails with a +structured list of conflicts (the HTTP server returns `409` with a +`merge_conflicts[]` array). No partial result is published — the merge is +all-or-nothing. The conflict kinds are: + +| Kind | Meaning | +|---|---| +| `DivergentInsert` | The same id was inserted on both branches. | +| `DivergentUpdate` | The same row was updated differently on both branches. | +| `DeleteVsUpdate` | One side deleted a row the other side updated. | +| `OrphanEdge` | An edge references a node the other side deleted. | +| `UniqueViolation` | The merged result would violate a unique constraint. | +| `CardinalityViolation` | The merged result would violate an edge cardinality constraint. | +| `ValueConstraintViolation` | The merged result would violate a value constraint (enum/range). | + +Each conflict carries the table, the row id (when applicable), the kind, and a +message. Resolve conflicts by reconciling the two branches — typically by making +the conflicting change on one side and re-merging. + +See [branches & commits](index.md) for the branch and commit-DAG model, and +[changes](changes.md) for diffing two branches before you merge. diff --git a/docs/user/branching/time-travel.md b/docs/user/branching/time-travel.md new file mode 100644 index 0000000..e6bd52d --- /dev/null +++ b/docs/user/branching/time-travel.md @@ -0,0 +1,31 @@ +# Snapshots & Time Travel + +Every read in OmniGraph happens against a **snapshot** — a consistent, cross-table +view of the graph at one manifest version. A query holds one snapshot for its whole +lifetime, so it never sees a partial write from a concurrent commit (see +[transactions](transactions.md)). + +## Reading the past + +- **Current head** — by default a read targets the current head of the bound branch. +- **By snapshot id** — read a branch or a specific snapshot id (`--snapshot` on + `omnigraph read`). +- **By version** — reconstruct a historical snapshot from any past manifest version. +- **Single entity** — look up one entity at a past version without building a full + snapshot (cheaper when you only need one node or edge). + +Snapshots are cheap to build: a snapshot is just the set of visible sub-table +versions at a manifest version, so cross-table reads stay snapshot-isolated. + +## CLI + +```bash +# Read a query against a past snapshot +omnigraph read --query ./q.gq --name find --snapshot s3://bucket/graph.omni +``` + +Time travel composes with branches: every branch has its own version history, and +you can read any branch at any of its past versions. Commits and the commit DAG +that these versions correspond to are described in +[branches & commits](index.md); diffing two versions is on the +[changes](changes.md) page. diff --git a/docs/user/transactions.md b/docs/user/branching/transactions.md similarity index 89% rename from docs/user/transactions.md rename to docs/user/branching/transactions.md index 39a86c4..6e6b1c4 100644 --- a/docs/user/transactions.md +++ b/docs/user/branching/transactions.md @@ -2,7 +2,7 @@ OmniGraph does not have `BEGIN` / `COMMIT` / `ROLLBACK`. Branches do that job. This page explains the model, when to use which primitive, and shows worked examples for the patterns that come up most. -The architectural rule lives in [`docs/dev/invariants.md`](../dev/invariants.md): +The architectural rule lives in [`docs/dev/invariants.md`](../../dev/invariants.md): > **Mutations publish at one boundary.** A `mutate_as` or `load` operation > accumulates constructive writes, commits each touched table at the end, then @@ -107,7 +107,7 @@ Properties: - Each query on the branch is its own publisher commit — so they're individually atomic. Per-query CAS works on branches just like on main. - The branch lives on disk. Process crash mid-workflow? Re-open and resume. - Multiple agents can work on different branches in parallel without blocking each other. -- The merge is a three-way merge at the row level. Conflicts surface as `OmniError::MergeConflicts(Vec)`, with structured kinds (`DivergentInsert`, `DivergentUpdate`, `DeleteVsUpdate`, …) so callers can handle them programmatically. +- The merge is a three-way merge at the row level. Conflicts surface as structured merge-conflict kinds (`DivergentInsert`, `DivergentUpdate`, `DeleteVsUpdate`, …) so callers can handle them programmatically. ### 4. Coordinating multiple agents @@ -129,14 +129,14 @@ omnigraph branch merge agent-b/work --into main graph.omni Each agent sees a consistent snapshot of `main` at the time it forked. The first merge to `main` lands as a fast-forward (or a no-op if no concurrent change). The second merge runs three-way: rows touched by both branches surface as `MergeConflict`s for the caller to resolve. -This is the workflow MR-797 / agentic loops are designed around: **branches are the unit of "an agent's working set."** +This is the workflow agentic loops are designed around: **branches are the unit of "an agent's working set."** ## Failure modes | Scenario | What happens | Caller action | |---|---|---| | Single query fails mid-flight | Publisher never publishes; target unchanged | Read the error, decide whether to retry | -| Concurrent writers race the same `(table, branch)` | Publisher CAS rejects the loser with `ManifestConflictDetails::ExpectedVersionMismatch` | Refresh handle, retry the query | +| Concurrent writers race the same `(table, branch)` | Publisher CAS rejects the loser with a version-mismatch conflict | Refresh handle, retry the query | | Branch with N successful mutations, then merge fails (three-way conflict) | Each individual mutation already committed on the branch; merge surfaces `MergeConflicts` | Inspect, decide whether to keep working on the branch, abandon it (`branch_delete`), or resolve and re-merge | | Process crashes mid-branch-workflow | Each completed mutation on the branch is durable | Re-open the graph, continue where you left off | @@ -161,8 +161,8 @@ This is the workflow MR-797 / agentic loops are designed around: **branches are ## See also -- [`docs/user/branches-commits.md`](branches-commits.md) — branch and commit-graph mechanics. -- [`docs/dev/merge.md`](../dev/merge.md) — three-way merge details and conflict kinds. -- [`docs/user/query-language.md`](query-language.md) — `.gq` syntax for the multi-statement queries used above. -- [`docs/dev/writes.md`](../dev/writes.md) — the per-query commit pipeline that gives single-query atomicity. -- [`docs/dev/invariants.md`](../dev/invariants.md) — the architectural rule. +- [`docs/user/branches-commits.md`](index.md) — branch and commit-graph mechanics. +- [`docs/dev/merge.md`](../../dev/merge.md) — three-way merge details and conflict kinds. +- [`docs/user/query-language.md`](../queries/index.md) — `.gq` syntax for the multi-statement queries used above. +- [`docs/dev/writes.md`](../../dev/writes.md) — the per-query commit pipeline that gives single-query atomicity. +- [`docs/dev/invariants.md`](../../dev/invariants.md) — the architectural rule. diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md deleted file mode 100644 index b419adf..0000000 --- a/docs/user/cli-reference.md +++ /dev/null @@ -1,211 +0,0 @@ -# CLI Reference (`omnigraph`) - -A reference for the `omnigraph` binary's command surface and `omnigraph.yaml` schema. For a quick-start guide, see [cli.md](cli.md). - -Top-level command families and subcommands. Graph-targeting commands accept a positional `URI`, `--uri`, a `--target ` resolved against `omnigraph.yaml`, or `--server ` (an operator-defined server from `~/.omnigraph/config.yaml`, optionally with `--graph ` for multi-graph servers; exclusive with the other forms); `cluster` commands use `--config `. - -## Top-level commands - -| Command | Purpose | -|---|---| -| `init` | `--schema ` → initialize a graph (no longer scaffolds `omnigraph.yaml` — RFC-008; start cluster configs from the [cluster.md](cluster.md) quick-start or `config migrate`) | -| `load` | bulk load a branch, local or remote (`--mode overwrite\|append\|merge` is **required** — overwrite is destructive, so there is no default). Without `--from` the target branch must exist; `--from ` forks a missing `--branch` from `` first | -| `ingest` | deprecated alias of `load --from ` (defaults: `--from main --mode merge`); prints a one-line warning to stderr | -| `query` (alias: `read`) | run named read query; source via `--query `, `-e`/`--query-string `, or `--alias ` (exactly one). `read` is the deprecated previous name and prints a one-line warning to stderr | -| `mutate` (alias: `change`) | run mutation query; same `--query` / `-e` / `--alias` mutual-exclusion as `query`. `change` is the deprecated previous name and prints a one-line warning to stderr | -| `snapshot` | print current snapshot (per-table version + row count) | -| `export` | dump to JSONL on stdout (`--type T`, `--table K` filters) | -| `branch create \| list \| delete \| merge` | branching ops | -| `commit list \| show` | inspect commit graph | -| `schema plan \| apply \| show (alias: get)` | migrations | -| `lint` (alias: `check`) | offline / graph-backed query validation. Replaces `query lint` / `query check`, which are kept as deprecated argv-level shims that print a one-line warning and rewrite to `omnigraph lint` | -| `config migrate` | propose (or `--write`: apply) the RFC-008 split of a legacy `omnigraph.yaml` — team half → ready-to-review `cluster.yaml`, personal half → `~/.omnigraph/config.yaml` (key-level merge, existing entries win), plus dropped-key reasons and manual steps | -| `cluster validate \| plan \| apply \| approve \| status \| refresh \| import \| force-unlock` | declarative cluster control plane. `validate` checks a local `cluster.yaml` folder and referenced schema/query/policy files; `plan` diffs it against local JSON state at `__cluster/state.json`, annotates dispositions, and embeds real schema-migration previews; `apply` converges the cluster — stored-query/policy catalog writes (content-addressed under `__cluster/resources/`), graph creates, schema updates (soft drops only; `--as` records the actor), and graph deletes behind a digest-bound approval from `cluster approve --as ` (`apply`/`approve` default the actor from the per-operator `omnigraph.yaml`'s `cli.actor` when `--as` is omitted; nothing else in that file affects cluster commands); what apply converges is what an `omnigraph-server --cluster ` deployment serves on its next restart (omnigraph.yaml deployments are unaffected); `status` reads the state ledger; `refresh`/`import` explicitly update local JSON state from read-only graph observations; `force-unlock ` manually removes a held local state lock by exact id | -| `optimize` | non-destructive Lance compaction (skips tables with `Blob` columns or uncovered drift; `--json` reports `skipped`) | -| `repair [--confirm] [--force]` | preview or explicitly publish uncovered manifest/head drift. `--confirm` heals verified maintenance drift and exits non-zero if suspicious/unverifiable drift is refused; `--force --confirm` publishes suspicious/unverifiable drift after operator review | -| `cleanup --keep N --older-than 7d --confirm` | destructive version GC | -| `embed` | offline JSONL embedding pipeline | -| `policy validate \| test \| explain` | Cedar tooling. Selects `cli.graph`, else `server.graph`, else top-level `policy.file` | -| `version` / `-v` | print `omnigraph 0.3.x` | - -## Config surfaces - -Two config surfaces with single owners (RFC-007/RFC-008), plus a zero-config -tier: - -| Surface | Owner | Location | Declares | -|---|---|---|---| -| Cluster config | the team, in a repo | `cluster.yaml` + checkout ([cluster-config.md](cluster-config.md)) | what the system **is**: graphs, schemas, queries, policies, storage | -| Operator config | one person | `~/.omnigraph/config.yaml` (override dir with `$OMNIGRAPH_HOME`) | who **I** am: identity, ergonomics | -| Flags / env | per invocation | — | everything, explicitly | - -`omnigraph.yaml` (below) is the legacy combined file — fully supported -today, slated for staged deprecation (RFC-008); its keys' future homes are -listed there. - -### `~/.omnigraph/config.yaml` (operator) - -```yaml -operator: - actor: act-andrew # default identity for every --as cascade: - # --as > legacy cli.actor > operator.actor > none -servers: # operator-owned endpoints; names key the credentials - prod: - url: https://graph.example.com # no tokens in this file, ever -defaults: - output: table # read format default, below --json/--format/alias/legacy -``` - -Absent file = empty layer. Unknown keys warn and load (a file written for a -newer CLI works on an older one). `$OMNIGRAPH_CONFIG=` stands in for -`--config` (the flag wins) in both the CLI and the server. - -#### Credentials keyed by server name - -`omnigraph login ` stores a bearer token in -`~/.omnigraph/credentials` (created `0600`; group/world-readable files are -refused). Token from `--token`, or — preferred, keeps it out of shell -history — one line on stdin: `echo $TOKEN | omnigraph login prod`. -`omnigraph logout ` removes it (idempotent). - -#### Operator aliases — bindings, not content - -An operator alias is a personal name for *invoking a stored query on a -named server* — it carries no query content (the stored query in the -catalog is the team's contract; the alias, its defaults, and its name are -yours): - -```yaml -aliases: - triage: - server: intel-dev # names an entry under servers: - graph: spike # optional (multi-graph servers) - query: weekly_triage # the STORED query's name — never a file - args: [since] # positional args -> params, in order - params: { limit: 20 } # fixed defaults; positionals/--params win - format: table -``` - -`omnigraph query --alias triage 2026-06-01` invokes -`POST /graphs/spike/queries/weekly_triage` with the keyed -credential. A legacy `omnigraph.yaml` alias with the same name wins during -the deprecation window (with a warning). - -A remote command whose URL prefix-matches an operator server's `url` (the -`gh` host model — no flags needed) resolves its token through: - -| Order | Source | -|---|---| -| 1 | `OMNIGRAPH_TOKEN_` env (`prod` → `OMNIGRAPH_TOKEN_PROD`) | -| 2 | `[]` section in `~/.omnigraph/credentials` | -| 3 | the legacy chain unchanged (`bearer_token_env` → `OMNIGRAPH_BEARER_TOKEN` → `auth.env_file`) | - -A token is only ever sent to the server it is keyed to: URLs matching no -operator server use the legacy chain alone. - -## `omnigraph.yaml` schema (legacy combined file) - -> **Deprecated (RFC-008).** Loading this file prints a per-key notice -> naming each present key's new home (suppress in CI with -> `OMNIGRAPH_SUPPRESS_YAML_DEPRECATION=1`); `omnigraph config migrate` -> produces the split. The file keeps working through the deprecation -> window. Migrated teams can set `OMNIGRAPH_NO_LEGACY_CONFIG=1` to turn -> any legacy-file load into a hard error (regression guard; the file's -> absence is always fine). - -```yaml -project: { name } -graphs: - : - uri: - bearer_token_env: - queries: # per-graph stored-query registry (server-role; multi-graph mode) - : # key MUST equal the `query ` symbol inside the .gq - file: # relative to this config's directory - mcp: - expose: true # default true: listed in the MCP catalog (GET /queries); set false to hide (still HTTP-callable) - tool_name: # optional MCP tool-name override (defaults to ; - # must be unique across exposed queries) -server: - graph: - bind: -cli: - graph: - branch: - output_format: json|jsonl|csv|kv|table - table_max_column_width: 80 - table_cell_layout: truncate|wrap -query: - roots: [, …] # search path for .gq files -auth: - env_file: .env.omni -aliases: - : - # accepted values: `read` / `query` (read alias), `change` / `mutate` - # (write alias). `query` and `mutate` are recommended; `read` and - # `change` remain accepted forever for back-compat. - command: read|change|query|mutate - query: - name: - args: [, …] - graph: - branch: - format: -queries: # top-level registry — applies only to a bare-URI (anonymous) graph; a graph served by name uses its `graphs..queries`. Mirrors top-level `policy`. - : { file: } # mcp.expose defaults to true -policy: - file: policy.yaml -``` - -## Cluster config preview - -```bash -omnigraph cluster validate --config company-brain -omnigraph cluster plan --config company-brain --json -omnigraph cluster apply --config company-brain --json -omnigraph cluster approve graph. --config company-brain --as -omnigraph cluster status --config company-brain --json -omnigraph cluster refresh --config company-brain --json -omnigraph cluster import --config company-brain --json -omnigraph cluster force-unlock --config company-brain --json -``` - -`--config` is a directory containing `cluster.yaml`; it defaults to `.`. -Stage 3A accepts graphs, schemas, stored queries, and policy bundle file -references. `cluster plan` reads local JSON state from -`/__cluster/state.json`; a missing file means empty state. Plan, -apply, refresh, and import acquire `__cluster/lock.json` by default and release -it before returning. `cluster apply` executes only stored-query/policy catalog -writes (content-addressed under `__cluster/resources/`) and requires an -existing `state.json`; graph/schema changes are deferred with warnings, and -applied resources do not serve traffic — the server still boots from -`omnigraph.yaml`. `cluster status` reads state only and reports any existing -lock metadata. `force-unlock` removes a lock only when the supplied id exactly -matches the lock file. `refresh` requires an existing `state.json`; `import` -creates one only when it is missing. Both observe declared graphs read-only at -`/graphs/.omni`. External state backends, graph/schema -apply, automatic stale-lock breaking, `plan --refresh`, pipelines, UI specs, -embeddings, aliases, and bindings are reserved for later stages. See -[cluster-config.md](cluster-config.md). - -## Output formats (`query` command, alias: `read`) - -- `json` — pretty-printed object with metadata + rows -- `jsonl` — one metadata line then one JSON object per row -- `csv` — RFC 4180-ish quoting -- `table` — fitted text table, honors `table_max_column_width` + `table_cell_layout` -- `kv` — grouped per-row key/value blocks - -## Param resolution - -Precedence (high to low): explicit `--params` / `--params-file`, alias positional args, `omnigraph.yaml` defaults. JS-safe-integer handling is built in (`is_js_safe_integer_i64`, `JS_MAX_SAFE_INTEGER_U64`) so 64-bit ids round-trip safely through JSON clients. - -## Bearer token resolution (CLI) - -1. `graphs..bearer_token_env` -2. `OMNIGRAPH_BEARER_TOKEN` global env -3. `auth.env_file` referenced `.env` - -## Duration parsing (cleanup) - -`s | m | h | d | w` units, e.g. `--older-than 7d`. diff --git a/docs/user/cli.md b/docs/user/cli.md deleted file mode 100644 index a6ce442..0000000 --- a/docs/user/cli.md +++ /dev/null @@ -1,164 +0,0 @@ -# CLI Guide - -## Core Graph Flow - -```bash -omnigraph init --schema schema.pg graph.omni -omnigraph load --data data.jsonl --mode overwrite graph.omni -omnigraph snapshot graph.omni --branch main --json -omnigraph query --uri graph.omni --query queries.gq --name get_person --params '{"name":"Alice"}' -omnigraph mutate --uri graph.omni --query queries.gq --name insert_person --params '{"name":"Mina","age":28}' -``` - -`omnigraph query` is the canonical read command (pairs with `POST /query`); -`omnigraph mutate` is the canonical write command (pairs with `POST /mutate`). -The previous names `omnigraph read` and `omnigraph change` keep working as -visible aliases — invocations emit a one-line deprecation warning to stderr -and otherwise behave identically. See [Deprecated names](#deprecated-names) -for the migration table. - -For ad-hoc reads and mutations (REPLs, AI agents, one-off scripts), pass the -GQ source inline with `-e` / `--query-string` instead of a file path: - -```bash -omnigraph query --uri graph.omni \ - -e 'query find($name: String) { match { $p: Person { name: $name } } return { $p.name, $p.age } }' \ - --params '{"name":"Alice"}' - -omnigraph mutate --uri graph.omni \ - -e 'query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }' \ - --params '{"name":"Inline","age":42}' -``` - -`-e` is mutually exclusive with `--query ` and `--alias `; exactly -one of the three must be provided. The inline source travels through the same -parser, lint, params binding, and commit machinery as a file-based query — -only the source loader changes. - -## Branching And Reviewable Data Flows - -```bash -omnigraph branch create --uri graph.omni --from main feature-x -omnigraph branch list --uri graph.omni -omnigraph branch merge --uri graph.omni feature-x --into main - -omnigraph load --data batch.jsonl --branch review/import-2026-04-09 --from main --mode merge graph.omni -omnigraph export graph.omni --branch main --type Person > people.jsonl -omnigraph commit list graph.omni --branch main --json -omnigraph commit show --uri graph.omni --json -``` - -## Remote Server Mode - -Serve a graph: - -```bash -omnigraph-server graph.omni --bind 127.0.0.1:8080 -``` - -Read through the HTTP API: - -```bash -omnigraph query \ - --target http://127.0.0.1:8080 \ - --query queries.gq \ - --name get_person \ - --params '{"name":"Alice"}' -``` - -If the server requires auth, set `OMNIGRAPH_SERVER_BEARER_TOKEN` on the server -and configure the matching `bearer_token_env` in `omnigraph.yaml`. - -## Multi-graph servers (v0.6.0+) - -Against a multi-graph server (started with `--config omnigraph.yaml` referencing a non-empty `graphs:` map), use `omnigraph graphs list` to enumerate the registered graphs. The server must configure bearer tokens and `server.policy.file` with a rule that allows `graph_list`; `/graphs` is closed by default even when the server runs with `--unauthenticated`. - -```bash -OMNIGRAPH_BEARER_TOKEN=admin-token \ - omnigraph graphs list --uri http://server.example.com --json -``` - -For config-driven clients, set the remote graph's `bearer_token_env` to an environment variable containing a token whose actor is authorized by `server.policy.file`. - -`list` rejects local URI targets — it's for remote multi-graph servers only. - -Runtime add/remove is **not** in v0.6.0. To add a graph, stop the server, add a `graphs.` entry to `omnigraph.yaml`, then restart. To remove, stop the server, delete the entry, restart. - -Per-graph URLs: hit a graph's cluster route from any subcommand by pointing `--uri` at it: - -```bash -omnigraph read --uri http://server.example.com/graphs/beta --query q.gq ... -``` - -## Runs, Policy, And Diagnostics - -```bash -omnigraph lint --query queries.gq --schema schema.pg --json -omnigraph check --query queries.gq graph.omni --json - -omnigraph schema plan --schema next.pg graph.omni --json -omnigraph schema apply --schema next.pg graph.omni --json -omnigraph policy validate --config omnigraph.yaml -omnigraph policy test --config omnigraph.yaml -omnigraph policy explain --config omnigraph.yaml --actor act-alice --action read --branch main - -omnigraph commit list graph.omni --json -omnigraph commit show --uri graph.omni --json -``` - -(The legacy `omnigraph run list/show/publish/abort` subcommands were removed in MR-771; mutations and loads publish atomically and the commit graph (`omnigraph commit list`) is the audit surface.) - -`query lint` and `query check` are the same command surface. In v1, graph-backed -lint uses local or `s3://` graph URIs; HTTP targets are only supported when you -also pass `--schema`. - -## Config - -`omnigraph.yaml` lets the CLI and server share named graphs, defaults, and -query roots: - -```yaml -graphs: - local: - uri: demo.omni - dev: - uri: http://127.0.0.1:8080 - bearer_token_env: OMNIGRAPH_BEARER_TOKEN - -cli: - graph: local - branch: main - -query: - roots: - - queries - - . -``` - -The config file can also define: - -- server bind defaults -- auth env files -- query aliases for common read and change commands -- `policy.file` for Cedar authorization rules - -When policy is enabled, `schema apply` is authorized through the -`schema_apply` action and is typically limited to admins on protected `main`. - -## Deprecated names - -The CLI was renamed to align with the HTTP server's canonical endpoint -names (`POST /query`, `POST /mutate`) and the `query` keyword in the GQ -language. The previous spellings keep working forever; invocations emit a -one-line warning to stderr and otherwise behave identically. - -| Old (deprecated) | New (canonical) | Migration | -|--------------------------|---------------------|----------------------------------------------------------| -| `omnigraph read` | `omnigraph query` | Same flags and behavior. `read` is a visible clap alias. | -| `omnigraph change` | `omnigraph mutate` | Same flags and behavior. `change` is a visible clap alias. | -| `omnigraph query lint` | `omnigraph lint` | Same flags. The argv-level shim rewrites `query lint` to `lint`. | -| `omnigraph query check` | `omnigraph check` | `check` is a visible alias of `omnigraph lint`. | - -The `command:` field in `aliases.` in `omnigraph.yaml` accepts both -`read` / `change` (legacy) and `query` / `mutate` (canonical); the two -spellings are interchangeable on the wire via serde aliases. diff --git a/docs/user/cli/index.md b/docs/user/cli/index.md new file mode 100644 index 0000000..6f49c42 --- /dev/null +++ b/docs/user/cli/index.md @@ -0,0 +1,175 @@ +# CLI Guide + +## Core Graph Flow + +```bash +omnigraph init --schema schema.pg graph.omni +omnigraph load --data data.jsonl --mode overwrite graph.omni +omnigraph snapshot graph.omni --branch main --json +# Invoke a stored query BY NAME from the catalog (served — addressed by scope): +omnigraph query get_person --params '{"name":"Alice"}' +omnigraph mutate insert_person --params '{"name":"Mina","age":28}' +``` + +`omnigraph query` is the canonical read command (pairs with `POST /query`); +`omnigraph mutate` is the canonical write command (pairs with `POST /mutate`). +The positional argument is the **stored-query name**, invoked from the served +catalog (RFC-011 D3) — the graph is addressed by scope (`--server` / `--profile` +/ defaults), and the verb asserts the query's kind (`query` rejects a stored +mutation, and vice-versa). The previous names `omnigraph read` and +`omnigraph change` keep working as visible aliases — invocations emit a one-line +deprecation warning to stderr. See [Deprecated names](#deprecated-names). + +For **ad-hoc** reads and mutations (REPLs, AI agents, one-off scripts, local dev), +pass the GQ source with `-e` / `--query-string` (inline) or `--query ` (a +file), and address a graph's storage directly with `--store`. By-name catalog +invocation is served-only — a bare `--store` has no catalog, so it's the ad-hoc +lane: + +```bash +omnigraph query --store graph.omni \ + -e 'query find($name: String) { match { $p: Person { name: $name } } return { $p.name, $p.age } }' \ + --params '{"name":"Alice"}' + +omnigraph mutate --store graph.omni \ + -e 'query add($name: String, $age: I32) { insert Person { name: $name, age: $age } }' \ + --params '{"name":"Inline","age":42}' + +# A multi-query file: the positional selects which query to run. +omnigraph query --store graph.omni --query queries.gq get_person --params '{"name":"Alice"}' +``` + +`-e` is mutually exclusive with `--query `. With either, the positional +name (optional) selects which query in the source to run. The inline source +travels through the same parser, lint, params binding, and commit machinery as a +file-based query — only the source loader changes. + +## Branching And Reviewable Data Flows + +```bash +omnigraph branch create --uri graph.omni --from main feature-x +omnigraph branch list --uri graph.omni +omnigraph branch merge --uri graph.omni feature-x --into main + +omnigraph load --data batch.jsonl --branch review/import-2026-04-09 --from main --mode merge graph.omni +omnigraph export graph.omni --branch main --type Person > people.jsonl +omnigraph commit list graph.omni --branch main --json +omnigraph commit show --uri graph.omni --json +``` + +## Remote Server Mode + +Serve a cluster-applied graph: + +```bash +omnigraph cluster apply --config ./company-brain +omnigraph-server --cluster ./company-brain --bind 127.0.0.1:8080 +``` + +Read through the HTTP API — invoke a stored query by name from the catalog: + +```bash +omnigraph query get_person \ + --server http://127.0.0.1:8080 \ + --params '{"name":"Alice"}' +``` + +A server is addressed with `--server` (a name from `~/.omnigraph/config.yaml` or a +literal URL); a positional `http(s)://` URI is rejected. If the server requires +auth, set its bearer token and `omnigraph login ` (or +`OMNIGRAPH_BEARER_TOKEN`). + +## Multi-graph servers + +A server boots from a cluster directory (`omnigraph-server --cluster `) and +serves every graph the cluster declares. Use `omnigraph graphs list` to enumerate +them. The cluster's server-level policy must allow `graph_list`; `/graphs` is +closed by default even when the server runs with `--unauthenticated`. + +```bash +OMNIGRAPH_BEARER_TOKEN=admin-token \ + omnigraph graphs list --server http://server.example.com --json +``` + +For an operator-defined server, store its token with `omnigraph login ` (or +`OMNIGRAPH_TOKEN_`); the actor must be authorized by the cluster's +server-level policy. + +`list` rejects local (`--store`) targets — it's for remote multi-graph servers only. + +Runtime add/remove via API is not exposed. To add or remove a graph, edit the +cluster's `cluster.yaml`, run `omnigraph cluster apply`, then restart the server. + +Per-graph addressing: select a graph on a multi-graph server with `--graph`: + +```bash +omnigraph query get_person --server http://server.example.com --graph beta --params '{"name":"Ada"}' +``` + +## Runs, Policy, And Diagnostics + +```bash +omnigraph lint --query queries.gq --schema schema.pg --json +omnigraph check --query queries.gq graph.omni --json + +omnigraph schema plan --schema next.pg graph.omni --json +omnigraph schema apply --schema next.pg graph.omni --json +omnigraph policy validate --cluster ./company-brain --graph knowledge +omnigraph policy test --cluster ./company-brain --graph knowledge --tests policy.tests.yaml +omnigraph policy explain --cluster ./company-brain --graph knowledge --actor act-alice --action read --branch main + +omnigraph commit list graph.omni --json +omnigraph commit show --uri graph.omni --json +``` + +(Mutations and loads publish atomically; the commit graph (`omnigraph commit list`) is the audit surface.) + +`query lint` and `query check` are the same command surface. In v1, graph-backed +lint uses local or `s3://` graph URIs; HTTP targets are only supported when you +also pass `--schema`. + +## Config + +Configuration has two surfaces with single owners (see the +[CLI reference](reference.md#config-surfaces) for the full schema): + +- **`~/.omnigraph/config.yaml`** — your personal operator config: default actor + (`--as`), named servers + credentials, clusters, profiles, aliases, and + default scope (`defaults.server` / `defaults.store` / `default_graph`). It + decides *who you are* and *what you address by default*. +- **`cluster.yaml`** (a team-owned cluster directory) — declares *what the system + is*: graphs, schemas, stored queries, policies, and storage. A server boots + from it (`--cluster `); see the [cluster guide](../clusters/index.md). + +```yaml +# ~/.omnigraph/config.yaml +operator: + actor: act-andrew +servers: + dev: + url: http://127.0.0.1:8080 +defaults: + server: dev + default_graph: knowledge +``` + +When policy is enabled, `schema apply` is authorized through the +`schema_apply` action and is typically limited to admins on protected `main`. + +## Deprecated names + +The CLI was renamed to align with the HTTP server's canonical endpoint +names (`POST /query`, `POST /mutate`) and the `query` keyword in the GQ +language. The previous spellings keep working forever; invocations emit a +one-line warning to stderr and otherwise behave identically. + +| Old (deprecated) | New (canonical) | Migration | +|--------------------------|---------------------|----------------------------------------------------------| +| `omnigraph read` | `omnigraph query` | Same flags and behavior. `read` is a visible clap alias. | +| `omnigraph change` | `omnigraph mutate` | Same flags and behavior. `change` is a visible clap alias. | +| `omnigraph query lint` | `omnigraph lint` | Same flags. The argv-level shim rewrites `query lint` to `lint`. | +| `omnigraph query check` | `omnigraph check` | `check` is a visible alias of `omnigraph lint`. | + +The `command:` field in `aliases.` in `~/.omnigraph/config.yaml` accepts +both `read` / `change` (legacy) and `query` / `mutate` (canonical); the two +spellings are interchangeable on the wire via serde aliases. diff --git a/docs/user/cli/reference.md b/docs/user/cli/reference.md new file mode 100644 index 0000000..1d52e45 --- /dev/null +++ b/docs/user/cli/reference.md @@ -0,0 +1,228 @@ +# CLI Reference (`omnigraph`) + +A reference for the `omnigraph` binary's command surface and the per-operator `~/.omnigraph/config.yaml` schema. For a quick-start guide, see [cli.md](index.md). + +Top-level command families and subcommands. Graph-targeting commands accept a positional `file://`/`s3://` URI, `--server ` (an operator-defined server from `~/.omnigraph/config.yaml` by name, or a literal `http(s)://` URL, optionally with `--graph ` for multi-graph servers; exclusive with a positional URI), `--store ` (a single graph's storage directly), or `--profile ` / `$OMNIGRAPH_PROFILE` (a named scope bundle; see [Scopes & profiles](#scopes--profiles-rfc-011)); `cluster` commands use `--config `, while `policy` and `queries` read a cluster's applied state via `--cluster `. A remote server is addressed only with `--server` — a positional `http(s)://` URI is rejected. **`query`/`mutate` are the exception**: their positional is a stored-query *name* (RFC-011 D3), not a graph URI, so they address the graph only via `--store`/`--server`/`--profile`/defaults. + +## Top-level commands + +| Command | Purpose | +|---|---| +| `init` | `--schema ` → initialize a graph (start cluster configs from the [cluster.md](../clusters/index.md) quick-start) | +| `load` | bulk load a branch, local or remote (`--mode overwrite\|append\|merge` is **required** — overwrite is destructive, so there is no default). Without `--from` the target branch must exist; `--from ` forks a missing `--branch` from `` first | +| `ingest` | deprecated alias of `load --from ` (defaults: `--from main --mode merge`); prints a one-line warning to stderr | +| `query ` (alias: `read`) | run a read query. **Catalog lane** (default): `` is a stored query invoked **by name** from the served catalog (served-only — address with `--server`/`--profile`; the verb asserts the query is a read). **Ad-hoc lane**: with `--query ` or `-e`/`--query-string `, runs that source (the positional `` then selects which query in it). No positional graph URI — address via `--store`/`--server`/`--profile`. `read` is the deprecated previous name (one-line stderr warning) | +| `mutate ` (alias: `change`) | run a mutation query; same catalog (by-name, served-only, verb asserts mutation) / ad-hoc (`--query`/`-e`) lanes as `query`. `change` is the deprecated previous name (one-line stderr warning) | +| `alias [args]` | invoke an operator alias — a read-only personal binding (under `aliases:` in `~/.omnigraph/config.yaml`) to a stored query on a named server (RFC-011 D4; replaces the removed `--alias` flag; stored mutations are rejected before execution) | +| `snapshot` | print current snapshot (per-table version + row count) | +| `export` | dump to JSONL on stdout (`--type T`, `--table K` filters) | +| `branch create \| list \| delete \| merge` | branching ops | +| `commit list \| show` | inspect commit graph | +| `schema plan \| apply \| show (alias: get)` | migrations. `apply` refuses a cluster-managed graph (one whose storage is inside a cluster) and points at `cluster apply` — those graphs evolve through the cluster ledger, not a direct apply | +| `lint` (alias: `check`) | offline / graph-backed query validation. Replaces `query lint` / `query check`, which are kept as deprecated argv-level shims that print a one-line warning and rewrite to `omnigraph lint` | +| `cluster validate \| plan \| apply \| approve \| status \| refresh \| import \| force-unlock` | declarative cluster control plane. `validate` checks a local `cluster.yaml` folder and referenced schema/query/policy files; `plan` diffs it against local JSON state at `__cluster/state.json`, annotates dispositions, and embeds real schema-migration previews; `apply` converges the cluster — stored-query/policy catalog writes (content-addressed under `__cluster/resources/`), graph creates, schema updates (soft drops only; `--as` records the actor), and graph deletes behind a digest-bound approval from `cluster approve --as ` (`apply`/`approve` default the actor from `~/.omnigraph/config.yaml`'s `operator.actor` when `--as` is omitted); what apply converges is what an `omnigraph-server --cluster ` deployment serves on its next restart (`--cluster` is the server's only boot source — RFC-011 cluster-only); `status` reads the state ledger; `refresh`/`import` explicitly update local JSON state from read-only graph observations; `force-unlock ` manually removes a held local state lock by exact id | +| `optimize` | non-destructive Lance compaction (skips tables with `Blob` columns or uncovered drift; `--json` reports `skipped`) | +| `repair [--confirm] [--force]` | preview or explicitly publish uncovered manifest/head drift. `--confirm` heals verified maintenance drift and exits non-zero if suspicious/unverifiable drift is refused; `--force --confirm` publishes suspicious/unverifiable drift after operator review | +| `cleanup --keep N --older-than 7d --confirm` | destructive version GC (`--confirm` to execute; also needs `--yes` against a non-local `s3://` target — see *Write diagnostics & destructive confirmation*) | +| `embed` | offline JSONL embedding pipeline | +| `policy validate \| test \| explain` | Cedar tooling against a cluster's applied policies (`--cluster `; `--graph ` picks a graph's bundle when several apply). `test` takes `--tests `; `explain` takes `--actor`/`--action`/`--branch`/`--target-branch` | +| `profile list \| show []` | read-only inspection of `~/.omnigraph/config.yaml` profiles. `list` shows each profile's binding (server/cluster/store) + default graph and marks the `$OMNIGRAPH_PROFILE`-active one; JSON keeps `binding` and adds `scope_kind`, `target`, `valid`, and `error`; `show` resolves one profile's scope (endpoint + default graph), defaulting to the active profile, else the flat operator defaults | +| `version` / `-v` | print `omnigraph 0.3.x` | + +## Command capabilities + +Every command declares the **capability** it needs — what it requires to reach a graph — which determines the addressing flags that apply: + +- **`any`** — `query`, `mutate`, `load`, `ingest`, `branch *`, `snapshot`, `export`, `commit *`, `schema show`, `schema apply`. Run against a graph **served (via a server) or embedded (direct against a store)**: accept a positional `file://`/`s3://` URI, `--server ` (+ `--graph ` for multi-graph servers), `--store `, or `--profile `. A remote server is addressed with `--server` — a positional `http(s)://` URI does **not** dispatch to one. +- **`served`** — `graphs list`. Requires a server (accepts `--server` / `--profile`). +- **`direct`** — `init`, `optimize`, `repair`, `cleanup`, `schema plan`, `lint`. Need **direct storage access** (`file://` / `s3://`), never through a server. They accept a positional `URI`, but **not** `--server`, and a remote (`http(s)://`) URI is rejected. `optimize` / `repair` / `cleanup` additionally accept **`--cluster --graph `** (`--cluster` is a cluster directory or storage-root URI, named via `clusters:` in `~/.omnigraph/config.yaml` or a literal root), which resolves the graph's storage URI from the served cluster state (so you needn't know the `/graphs/.omni` layout). `--graph` is the one graph selector across all scopes — on these three verbs it picks the cluster graph; on the other `direct` verbs it does not apply. +- **`control`** — `cluster *` via `--config `; `policy *` and `queries *` via `--cluster ` or a cluster profile. +- **`local`** — `alias`, `embed`, `login`, `logout`, `profile`, `version`. Address no explicit graph scope. + +These restrictions are enforced and reported, not silent: + +- A scope flag on a verb that can't consume it fails loudly rather than being silently dropped — `--server` outside a served scope, `--cluster` outside cluster-scoped verbs, or `--graph` where no multi-graph scope applies, e.g.: ``optimize is a direct (storage-native) command; --server addresses a served graph and does not apply. Pass a storage URI, or --cluster --graph .`` +- A `direct` verb pointed at a remote URI fails loudly, e.g.: ``optimize is a direct (storage-native) command and needs direct storage access; the resolved target is a remote server (https://…). Pass the graph's file:// or s3:// URI.`` +- A data verb pointed at a positional `http(s)://` URI fails loudly: ``a remote graph must be addressed with --server — a positional (or --uri) http(s):// URL no longer dispatches to a server.`` +- `init` into an **established cluster's** storage layout (`/graphs/.omni` where `` holds `__cluster/state.json`) is refused — graphs in a cluster are created by `cluster apply` (which records ledger / recovery / approvals), not `init`. + +To maintain a server-backed graph, run the `direct` verbs from a host with storage access against the graph's storage URI (a positional URI, or `--cluster … --graph …`), out-of-band from the serving process — there are no server routes for `optimize` / `repair` / `cleanup` by design. + +`omnigraph --help` lists commands with a **capability legend** at the bottom (any / served / direct / control / local). + +## Write diagnostics & destructive confirmation + +Two global flags make writes self-documenting and guard the dangerous ones (RFC-011 Decision 9): + +- **Every write echoes its resolved target to stderr** — `omnigraph load → s3://acme/brain/graphs/knowledge.omni (direct, remote)` — so you catch a scope that resolved somewhere unexpected (e.g. *prod*) before it lands. Applies to `load`, `ingest`, `mutate`, `branch create|delete|merge`, `schema apply`, `optimize`, `repair`, `cleanup`. The line is stderr, so `--json` consumers reading stdout are unaffected; suppress it with **`--quiet`**. +- **Destructive writes against a non-local scope require confirmation.** `cleanup`, overwrite `load` (`--mode overwrite`), and `branch delete` proceed freely against a local (`file://`) graph, but when the resolved target is **not local** (a served `http(s)://` graph or an `s3://` store/cluster) they require explicit consent: pass **`--yes`** to confirm, an interactive terminal is prompted, and a non-interactive run (no TTY, or `--json`) **refuses with an error** rather than silently destroying. `cleanup` still also requires its existing `--confirm` (preview→execute); `--yes` is the additional non-local consent. + +A "local" target is a bare path or a `file://` URI; `http(s)://`, `s3://`, and other object-store schemes are non-local. + +## Config surfaces + +Two config surfaces with single owners, plus a zero-config tier: + +| Surface | Owner | Location | Declares | +|---|---|---|---| +| Cluster config | the team, in a repo | `cluster.yaml` + checkout ([cluster-config.md](../clusters/config.md)) | what the system **is**: graphs, schemas, queries, policies, storage | +| Operator config | one person | `~/.omnigraph/config.yaml` (override dir with `$OMNIGRAPH_HOME`) | who **I** am: identity, ergonomics | +| Flags / env | per invocation | — | everything, explicitly | + +### `~/.omnigraph/config.yaml` (operator) + +```yaml +operator: + actor: act-andrew # default identity for the --as cascade: --as > operator.actor > none +servers: # operator-owned endpoints; names key the credentials + prod: + url: https://graph.example.com # no tokens in this file, ever +defaults: + output: table # read format default, below --json/--format/alias + server: prod # the everyday SERVED scope when no address is given (RFC-011) + # store: file:///data/dev.omni # OR a zero-flag LOCAL default (mutually + # # exclusive with `server`); the local-dev + # # counterpart of `server` + default_graph: knowledge # graph selected in a server/cluster scope +clusters: # admin-only: managed-cluster storage roots (RFC-011). + brain: # the ONLY place a storage root lives in this file. + root: s3://acme/clusters/brain +profiles: # named scope bundles (RFC-011); pick with --profile + staging: { server: staging, default_graph: knowledge } # a served scope + brain-admin: { cluster: brain, default_graph: knowledge } # a direct cluster scope +``` + +Absent file = empty layer. Unknown keys warn and load (a file written for a +newer CLI works on an older one). Override the config directory with +`$OMNIGRAPH_HOME`. + +#### Scopes & profiles (RFC-011) + +A command resolves a **scope** — a server, a cluster, or a store — then selects a +graph in it; the served-vs-direct access path is derived from the scope, not +toggled. The scope comes from one of (highest precedence first): an explicit +address (a positional URI, `--server`, or `--store `); a named +`--profile ` (or `$OMNIGRAPH_PROFILE`); or the flat `defaults.server` + +`defaults.default_graph` (a served default) **or** `defaults.store` (a zero-flag +*local* default — mutually exclusive with `defaults.server`). A **profile** binds +exactly one of `server` / `cluster` / `store` plus an optional default graph — +config data, not state: every command resolves its scope fresh, there is no +sticky "current" mode. Inspect what is defined with `omnigraph profile list` and +`omnigraph profile show []` (read-only). + +- `--store ` addresses a single graph's storage directly (ad-hoc / break-glass). +- A `cluster`-bound profile reaches `optimize` / `repair` / `cleanup` for a managed + graph (resolving its storage root from `clusters:`), the same as + `--cluster --graph `. A `--graph` flag overrides the profile's default. +- A `server`-bound scope on a maintenance verb, or a `cluster`-bound scope on a + data verb, is rejected with a message pointing at the right addressing. +- **No graph selected (RFC-011 D7).** When a scope has no `--graph` and no + `default_graph`, the CLI never silently picks: + - **Cluster scope** — exactly **one** applied graph is used automatically; + **several** errors and lists the candidates (from the served catalog). + - **Server scope** — a multi-graph server (any non-empty `GET /graphs`, even a + single entry) errors and lists the candidates: you must pass `--graph `. + A single-graph / flat server (405 on `/graphs`), or one whose `/graphs` is + policy-gated or unreachable, uses its bare URL as before. + +`--target`, `--cluster-graph`, and the positional-`http(s)://`→remote dispatch +have been **removed** (`--graph` is now the one graph selector across server and +cluster scopes); operator `defaults`/`--profile` supply the no-flag scope and an +explicit address always wins. + +#### Credentials keyed by server name + +`omnigraph login ` stores a bearer token in +`~/.omnigraph/credentials` (created `0600`; group/world-readable files are +refused). Token from `--token`, or — preferred, keeps it out of shell +history — one line on stdin: `echo $TOKEN | omnigraph login prod`. +`omnigraph logout ` removes it (idempotent). + +#### Operator aliases — bindings, not content + +An operator alias is a personal name for *invoking a stored query on a +named server* — it carries no query content (the stored query in the +catalog is the team's contract; the alias, its defaults, and its name are +yours): + +```yaml +aliases: + triage: + server: intel-dev # names an entry under servers: + graph: spike # optional (multi-graph servers) + query: weekly_triage # the STORED query's name — never a file + args: [since] # positional args -> params, in order + params: { limit: 20 } # fixed defaults; positionals/--params win + format: table +``` + +`omnigraph alias triage 2026-06-01` invokes +`POST /graphs/spike/queries/weekly_triage` with the keyed +credential. Aliases live in their own `alias` namespace (RFC-011 Decision 4), +so an alias can never shadow — or be shadowed by — a built-in verb. (The old +`--alias ` flag on `query`/`mutate` was removed.) + +A remote command whose URL prefix-matches an operator server's `url` (the +`gh` host model — no flags needed) resolves its token through: + +| Order | Source | +|---|---| +| 1 | `OMNIGRAPH_TOKEN_` env (`prod` → `OMNIGRAPH_TOKEN_PROD`) | +| 2 | `[]` section in `~/.omnigraph/credentials` | +| 3 | the default `OMNIGRAPH_BEARER_TOKEN` env | + +A keyed token is only ever sent to the server it is keyed to: a URL matching no +operator server falls back to `OMNIGRAPH_BEARER_TOKEN` alone. + +## Cluster config preview + +```bash +omnigraph cluster validate --config company-brain +omnigraph cluster plan --config company-brain --json +omnigraph cluster apply --config company-brain --json +omnigraph cluster approve graph. --config company-brain --as +omnigraph cluster status --config company-brain --json +omnigraph cluster refresh --config company-brain --json +omnigraph cluster import --config company-brain --json +omnigraph cluster force-unlock --config company-brain --json +``` + +`--config` is a directory containing `cluster.yaml`; it defaults to `.`. +Stage 3A accepts graphs, schemas, stored queries, and policy bundle file +references. `cluster plan` reads local JSON state from +`/__cluster/state.json`; a missing file means empty state. Plan, +apply, refresh, and import acquire `__cluster/lock.json` by default and release +it before returning. `cluster apply` executes only stored-query/policy catalog +writes (content-addressed under `__cluster/resources/`) and requires an +existing `state.json`; graph/schema changes are deferred with warnings, and +applied resources do not serve traffic until an `omnigraph-server --cluster +` restart picks them up. `cluster status` reads state only and reports any existing +lock metadata. `force-unlock` removes a lock only when the supplied id exactly +matches the lock file. `refresh` requires an existing `state.json`; `import` +creates one only when it is missing. Both observe declared graphs read-only at +`/graphs/.omni`. External state backends, graph/schema +apply, automatic stale-lock breaking, `plan --refresh`, pipelines, UI specs, +embeddings, aliases, and bindings are reserved for later stages. See +[cluster-config.md](../clusters/config.md). + +## Output formats (`query` command, alias: `read`) + +- `json` — pretty-printed object with metadata + rows +- `jsonl` — one metadata line then one JSON object per row +- `csv` — RFC 4180-ish quoting +- `table` — fitted text table, honors `table_max_column_width` + `table_cell_layout` +- `kv` — grouped per-row key/value blocks + +## Param resolution + +Precedence (high to low): explicit `--params` / `--params-file`, alias positional args. JS-safe-integer handling is built in (`is_js_safe_integer_i64`, `JS_MAX_SAFE_INTEGER_U64`) so 64-bit ids round-trip safely through JSON clients. + +## Bearer token resolution (CLI) + +1. `graphs..bearer_token_env` +2. `OMNIGRAPH_BEARER_TOKEN` global env +3. `auth.env_file` referenced `.env` + +## Duration parsing (cleanup) + +`s | m | h | d | w` units, e.g. `--older-than 7d`. diff --git a/docs/user/cluster-config.md b/docs/user/clusters/config.md similarity index 86% rename from docs/user/cluster-config.md rename to docs/user/clusters/config.md index 59c9207..8f8caf4 100644 --- a/docs/user/cluster-config.md +++ b/docs/user/clusters/config.md @@ -1,9 +1,7 @@ # Cluster Config -**Status:** Phase 5 — cluster-booted serving (`omnigraph-server --cluster`). - > New to the cluster tooling? Start with the operator how-to guide, -> [cluster.md](cluster.md) — this document is the reference. +> [cluster.md](index.md) — this document is the reference. Cluster config is the future control-plane configuration surface for a whole OmniGraph deployment. In this stage, OmniGraph can validate a local @@ -15,7 +13,8 @@ catalog writes, **graph creation** (a declared graph that does not exist yet is initialized by apply at the derived root), **schema updates** (soft drops only), and — behind an explicit, digest-bound **approval** — **graph deletion**. It does not perform data-loss schema migrations, start servers, -or serve anything it applies: the server still boots from `omnigraph.yaml`. +or run data loads. A server can boot from the applied ledger with +`omnigraph-server --cluster `. ## Commands @@ -33,33 +32,31 @@ omnigraph cluster force-unlock --config company-brain --json `--config` points at a directory, not a file. The directory must contain `cluster.yaml`. When omitted, it defaults to the current directory. -## Relationship to `omnigraph.yaml` +## Relationship to `~/.omnigraph/config.yaml` -`cluster.yaml` does not replace `omnigraph.yaml`, and the two never describe -the same fact. `omnigraph.yaml` is the permanent **per-operator** layer (CLI -defaults, the operator's identity and credential references, graph targets -for data-plane commands); `cluster.yaml` is the shared desired state of a +`cluster.yaml` and the per-operator `~/.omnigraph/config.yaml` never describe +the same fact. The operator config is the permanent **per-operator** layer +(the operator's identity and credential references, named servers/clusters, +profiles, and CLI defaults); `cluster.yaml` is the shared desired state of a whole deployment, read only by the `cluster` commands via `--config`. The exact contract: -- **Cluster commands read `omnigraph.yaml` for exactly one thing**: the - `cli.actor` default used by `apply`/`approve` when `--as` is omitted — - operator identity is a per-operator fact. With `--as` present, no config - is read at all. Nothing else (its graph set, targets, bind, queries, - policies) ever influences a cluster command; a malformed `omnigraph.yaml` - breaks only the no-flag actor lookup, loudly. -- **A `--cluster` server reads `omnigraph.yaml` for nothing** — not even the - implicit current-directory search runs (mode-inference rule 0). Boot from - cluster state XOR `omnigraph.yaml`, never a merge. -- **The other direction is ergonomics, not coupling**: a per-operator - `omnigraph.yaml` may point `graphs..uri` at a cluster's derived root - (`company-brain/graphs/knowledge.omni`) so data-plane commands can use - `--target ` — an ordinary local path, no special handling. +- **Cluster commands read the operator config for exactly one thing**: the + `operator.actor` default used by `apply`/`approve` when `--as` is omitted — + operator identity is a per-operator fact. With `--as` present, the operator + config is not needed. Nothing else in it influences a cluster command. +- **No legacy `omnigraph.yaml`**: the CLI does not read `omnigraph.yaml` at + all, and a `--cluster` server reads only the cluster catalog — boot is + cluster-only. +- **The other direction is ergonomics, not coupling**: per-operator + data-plane commands address a cluster graph by its derived storage root + (`company-brain/graphs/knowledge.omni`) with `--store ` — an ordinary + local path, no special handling. ## Supported `cluster.yaml` -Stage 3A accepts only this resource subset: +The current config surface accepts this resource subset: ```yaml version: 1 @@ -70,9 +67,18 @@ state: backend: cluster lock: true +providers: + embedding: + default: + kind: openai-compatible + base_url: https://openrouter.ai/api/v1 + model: openai/text-embedding-3-large + api_key: ${OPENROUTER_API_KEY} + graphs: knowledge: schema: knowledge.pg + embedding_provider: default queries: queries/ # discover every `query ` in queries/*.gq policies: @@ -101,6 +107,17 @@ updates all of its queries together. Paths are relative to the config directory — the cluster is one explicit folder, so no `./` prefixes are needed. +`providers.embedding.` defines a query-time embedding provider profile +for cluster-served graphs. A graph opts in with `embedding_provider: `; +bare names normalize to `provider.embedding.`. Supported provider +`kind` values are `openai-compatible` (default/OpenRouter-compatible), +`openai` (OpenAI's own host), `gemini`, and `mock`. Real providers require +`api_key: ${ENV_VAR}`; inline secrets are rejected. The env var is resolved +only when a `--cluster` server boots, so `cluster validate`, `plan`, and +`apply` do not need deployment secrets. `mock` is deterministic and does not +require `api_key`. Vector dimensions stay schema-driven by the target +`Vector(N)` column, not the provider profile. + `storage:` (optional) is the **storage root URI** for everything the cluster stores — the state ledger, lock, content-addressed catalog, recovery sidecars, approval artifacts, and the derived graph roots @@ -135,10 +152,12 @@ operation is active. - stored-query parsing and query-name matching - stored-query type-checking against the desired schema - policy `applies_to` graph references +- embedding provider profiles and graph `embedding_provider` references -Fields reserved for later phases, such as `pipelines`, `embeddings`, `ui`, -`aliases`, and `bindings`, fail with a typed diagnostic instead of being -silently ignored. +Fields reserved for later phases, such as `pipelines`, top-level +`embeddings`, `ui`, `aliases`, and `bindings`, fail with a typed diagnostic +instead of being silently ignored. Under `providers`, only `embedding` is +supported today; other provider namespaces fail as unsupported config. ## Planning @@ -158,9 +177,21 @@ resource is planned as a create. If present, the file must use this shape: "applied_revision": { "config_digest": "...", "resources": { - "graph.knowledge": { "digest": "..." }, "schema.knowledge": { "digest": "..." }, "query.knowledge.find_experts": { "digest": "..." }, + "provider.embedding.default": { + "digest": "...", + "embedding_profile": { + "kind": "openai-compatible", + "base_url": "https://openrouter.ai/api/v1", + "model": "openai/text-embedding-3-large", + "api_key": "${OPENROUTER_API_KEY}" + } + }, + "graph.knowledge": { + "digest": "...", + "embedding_provider": "provider.embedding.default" + }, "policy.base": { "digest": "...", "applies_to": ["cluster", "graph.knowledge"] @@ -236,12 +267,11 @@ Deletes remove the resource from state; their old payload blobs stay on disk (garbage collection is a later stage). Re-running a converged apply is a no-op: no state write, no revision change (`state_written: false`). -**Applied means serving — for deployments that opt in.** A server started -with `--cluster ` boots from the applied revision (see +**Applied means serving.** A server started with `--cluster ` boots from +the applied revision (see [Serving from the cluster](#serving-from-the-cluster-the-mode-switch)); it -picks up newly applied state on its next restart. Deployments still booting -from `omnigraph.yaml` are untouched: for them, applied means recorded in the -catalog, nothing more. +picks up newly applied state on its next restart. Until that restart, applied +means recorded in the catalog, nothing more. ### Graph creation diff --git a/docs/user/cluster.md b/docs/user/clusters/index.md similarity index 82% rename from docs/user/cluster.md rename to docs/user/clusters/index.md index 0d6dac5..c59ff9d 100644 --- a/docs/user/cluster.md +++ b/docs/user/clusters/index.md @@ -7,8 +7,8 @@ destructive changes, and recovering from crashes. It is a **how-to**. The reference for every `cluster.yaml` key, command flag, state-file field, and diagnostic code is -[cluster-config.md](cluster-config.md); the HTTP surface is -[server.md](server.md). +[cluster-config.md](config.md); the HTTP surface is +[server.md](../operations/server.md). ## The model in one paragraph @@ -102,7 +102,7 @@ curl -H 'authorization: Bearer s3cret' \ Bearer tokens and the bind address are deliberately *not* cluster facts — they are per-replica, set by flag or environment -([server.md](server.md#modes) for the token sources). +([server.md](../operations/server.md#modes) for the token sources). ## 2. The day-2 loop: edit → plan → apply → restart @@ -117,7 +117,7 @@ omnigraph cluster apply --config company-brain --as andrew `--as ` attributes the run: it is recorded in recovery sidecars and audit entries and threaded into the engine's commit history. Set -`cli: { actor: }` in your per-operator `omnigraph.yaml` to make it the +`operator: { actor: }` in your `~/.omnigraph/config.yaml` to make it the default when `--as` is omitted (the flag always wins; `approve` requires one of the two). @@ -237,26 +237,53 @@ with an in-flight apply. directory; boot is read-only. Roll out a change by `apply` once, then restarting replicas (serving is static per process — there is no hot reload yet). Container/cloud recipes (AWS ECS+EFS, Railway volumes): - [deployment.md](deployment.md#cluster-mode-in-containers-aws-railway). + [deployment.md](../deployment.md#cluster-mode-in-containers-aws-railway). - **The directory is the deployable unit**: config, catalog, ledger, approvals, and graph data all live under it. Back it up as a whole; version the *config files* (not `__cluster/` or `graphs/`) in git. - **CI-driven convergence**: `validate` and `plan --json` are read-only and safe in pipelines; gate `apply --as ci` on plan review. Approvals are the human step by design — keep `cluster approve` out of automation. -- **`omnigraph.yaml` still has a job**: per-operator settings — your - `cli.actor` default for `--as`, CLI defaults, credentials, and data-plane - ergonomics (point `graphs..uri` at a derived root like - `company-brain/graphs/knowledge.omni` to use `--target ` for - loads). It just no longer describes the deployment — a server boots from - one source or the other, never a merge of both. +- **`~/.omnigraph/config.yaml` is the per-operator config**: your + `operator.actor` default for `--as`, named servers/clusters, credentials, + profiles, and data-plane ergonomics (address a cluster graph by its derived + root like `company-brain/graphs/knowledge.omni` with `--store` for loads). The + cluster directory's `cluster.yaml` is the **sole deployment declaration** — the + server boots from the cluster only. + +## 7. Maintaining a cluster graph + +Storage maintenance (`optimize` / `repair` / `cleanup`) is **not** a control-plane +operation — it runs out-of-band, with direct storage access, against the graph's +roots. Address a cluster graph by name instead of hand-typing its storage path: + +```bash +omnigraph optimize --cluster ./company-brain --graph knowledge +omnigraph cleanup --cluster ./company-brain --graph knowledge --keep 10 --confirm +# --cluster also takes the storage-root URI directly (config-free), and a +# `clusters:` name from ~/.omnigraph/config.yaml: +omnigraph optimize --cluster s3://bucket/clusters/company-brain --graph knowledge +``` + +The graph's storage URI is resolved from the **served cluster state** (the same +truth a `--cluster` server boots from); a graph that hasn't been applied yet is +not resolvable. Run these from a host with storage access — there are no server +routes for them. Conversely, **`init` refuses** a cluster-managed path: graphs in +a cluster are created by `cluster apply`, not by hand. + +If the cluster has exactly **one** applied graph you can omit `--graph` — it is +used automatically. With **several**, omitting `--graph` errors and lists the +candidates (RFC-011 D7); it never picks one for you. + +Against an **`s3://`-backed cluster** the resolved graph storage is non-local, so a +destructive `cleanup` additionally requires **`--yes`** (an interactive prompt +otherwise, refusal without a TTY) on top of `--confirm` — see [cli-reference.md](../cli/reference.md)'s +*Write diagnostics & destructive confirmation*. Every maintenance run also echoes +its resolved target to stderr (suppress with `--quiet`). ## What the control plane does not do (yet) - **No hot reload** — applied changes serve on the next restart. -- **No S3-hosted cluster directories** — the config dir, ledger, catalog, - and derived graph roots are local-filesystem paths today. (Individual - *graphs* on S3 are a server feature outside cluster mode.) - **No data operations** — rows move through `omnigraph load / ingest / mutate` against the graph roots, with branches and merges as usual. - **Stored-query exposure is all-or-nothing per cluster** — every applied @@ -266,4 +293,4 @@ with an in-flight apply. reserved and rejected loudly. For the full reference — every key, flag, status, disposition, and -diagnostic — see [cluster-config.md](cluster-config.md). +diagnostic — see [cluster-config.md](config.md). diff --git a/docs/user/concepts/index.md b/docs/user/concepts/index.md new file mode 100644 index 0000000..8bc3d7e --- /dev/null +++ b/docs/user/concepts/index.md @@ -0,0 +1,49 @@ +# Concepts + +OmniGraph is a typed property-graph engine built as a coordination layer over the +[Lance](https://lance.org) columnar storage format. It gives you a schema-checked +graph with vector, full-text, and graph queries in one runtime, plus Git-style +branches and commits across the whole graph. + +## The data model + +- A graph has **node types** and **edge types**, declared in a + [schema](../schema/index.md). +- Each node type and each edge type is stored as its **own Lance dataset** — + columnar, versioned, on local disk or object storage. +- A single `__manifest` table coordinates all of those datasets, so the graph has + one coherent version even though it spans many datasets. + +This split is what lets a graph commit be **atomic across every type at once**: a +publish flips every relevant dataset's version together in one manifest write, so +readers never see a half-applied change. See [storage](storage.md) for the layout. + +## Two layers: inherited vs. added + +Throughout the docs, capabilities are framed as **L1** (inherited from Lance) or +**L2** (added by OmniGraph): + +| | L1 — from Lance | L2 — added by OmniGraph | +|---|---|---| +| Storage | Columnar Arrow datasets on object storage | Per-type datasets coordinated as one graph | +| Versioning | Per-dataset versions + time travel | [Snapshots](../branching/time-travel.md) across all types at once | +| Branches | Per-dataset branches | [Graph-level branches](../branching/index.md), atomic across types | +| Commits | Per-dataset commits | [Commit DAG](../branching/index.md) for the whole graph; three-way [merge](../branching/merge.md) | +| Indexes | Scalar / vector / full-text indexes | Built per relevant column; graph topology index for traversal | +| Search | Vector + full-text primitives | [`nearest` / `bm25` / `rrf`](../search/index.md) in one query, plus graph traversal | +| Querying | — | The [`.gq` query language](../queries/index.md) and [`.pg` schema language](../schema/index.md) | + +## How the pieces fit + +- The **schema** (`.pg`) and **query** (`.gq`) languages are compiled to a typed + intermediate representation. +- The **engine** runs queries and mutations against Lance, coordinates the manifest, + maintains the commit graph, and builds indexes. +- The **CLI** ([`omnigraph`](../cli/index.md)) and the + **HTTP server** ([`operations/server.md`](../operations/server.md)) are two front + ends over the same engine, so embedded and remote behavior match. +- [Cedar policy](../operations/policy.md) enforcement is engine-wide — every writer + goes through the same authorization gate regardless of front end. + +For deployment-scale topics — multi-graph servers, control-plane operations, +recovery — see [clusters](../clusters/index.md). diff --git a/docs/user/storage.md b/docs/user/concepts/storage.md similarity index 53% rename from docs/user/storage.md rename to docs/user/concepts/storage.md index 9cc2356..68bfbcc 100644 --- a/docs/user/storage.md +++ b/docs/user/concepts/storage.md @@ -7,47 +7,45 @@ Every node type and every edge type is its own Lance dataset: - **Columnar Arrow storage**: each property is a column; nullable per Arrow schema. - **Fragments**: data is partitioned into fragments; new writes create new fragments. - **Manifest versioning**: every commit produces a new dataset version; old versions remain readable. -- **Stable row IDs**: `enable_stable_row_ids: true` is set on every Lance dataset OmniGraph creates — node and edge data tables, `__manifest`, `_graph_commits.lance`, `_graph_commit_recoveries.lance`, and any future system tables. This is an architectural invariant: the flag is one-way at dataset create per Lance's row-id-lineage spec, so a future change that introduces a Lance dataset must preserve it. Consequences: `_row_created_at_version` and `_row_last_updated_at_version` are available on every dataset (load-bearing for change-feed validators); `CreateIndex × Rewrite` is not a retryable conflict, so indices survive `omnigraph optimize` without needing the Fragment Reuse Index; readers must use a Lance build that recognises the flag (our pinned 4.0.0 is fine). Pre-0.4.x graphs created before this code path settled may have datasets without the flag and cannot be retrofitted in place — the supported path is dump-and-reload. The `stage_overwrite` rewrite path (used by `schema_apply`) preserves the flag through `Operation::Overwrite`; pinned by `stage_overwrite_preserves_stable_row_ids` in `crates/omnigraph/tests/staged_writes.rs`. +- **Stable row IDs**: stable row IDs are enabled on every Lance dataset OmniGraph creates — node and edge data tables, `__manifest`, the commit-graph datasets, and any future system tables. This is an architectural invariant: the flag is one-way at dataset create, so a future change that introduces a Lance dataset must preserve it. Consequences: `_row_created_at_version` and `_row_last_updated_at_version` are available on every dataset (load-bearing for change-feed validators); indices survive `omnigraph optimize`. Pre-0.4.x graphs created before this code path settled may have datasets without the flag and cannot be retrofitted in place — the supported path is dump-and-reload. The rewrite path used by `schema_apply` preserves the flag. - **Append / delete / `merge_insert`**: native Lance write modes. - **Per-dataset branches** (Lance native): copy-on-write at the dataset level. -- **Object-store agnostic**: file://, s3://, gs://, az://, http (read-only via Lance) — OmniGraph wires file:// and s3:// (`storage.rs`). +- **Object-store agnostic**: file://, s3://, gs://, az://, http (read-only via Lance) — OmniGraph wires file:// and s3://. ## L2 — Multi-dataset coordination via `__manifest` OmniGraph is **not** a single Lance dataset; it is a *graph* of datasets coordinated through one append-only manifest table. - **Manifest table**: `__manifest/` Lance dataset. -- **Layout** (`db/manifest/layout.rs`, `db/manifest/state.rs`): +- **Layout**: - `nodes/{fnv1a64-hex(type_name)}` — one Lance dataset per node type - `edges/{fnv1a64-hex(edge_type_name)}` — one Lance dataset per edge type - `__manifest/` — the catalog of all sub-tables and their published versions - `_graph_commits.lance` / `_graph_commit_actors.lance` — the commit graph and its actor map - - (legacy `_graph_runs.lance` / `_graph_run_actors.lance` from pre-v0.4.0 graphs are inert; the run state machine was removed in MR-771. The v2→v3 manifest migration sweeps stale `__run__*` branches on first write-open; the inert dataset bytes themselves remain until a `delete_prefix` storage primitive lands) + - (legacy `_graph_runs.lance` / `_graph_run_actors.lance` from pre-v0.4.0 graphs are inert; the run state machine was removed. The internal schema migration sweeps stale `__run__*` branches on first write-open; the inert dataset bytes themselves remain until a prefix-delete storage primitive lands) - **Manifest row schema** (`object_id, object_type, location, metadata, base_objects, table_key, table_version, table_branch, row_count`): - `object_type` ∈ `table | table_version | table_tombstone` - `table_key` ∈ `node: | edge:` - `table_branch` is `null` for the main lineage and the branch name otherwise - **Snapshot reconstruction**: latest visible `table_version` per `(table_key, table_branch)` minus tombstones — rows where `object_type = table_tombstone`, whose own `table_version` (acting as the tombstone version) is `>= the entry's table_version`. -- **Atomic publish**: multi-dataset commits publish via a `ManifestBatchPublisher` so a single write to `__manifest` flips all the new sub-table versions visible at once. -- **Row-level CAS on the merge-insert join key**: `object_id` carries `lance-schema:unenforced-primary-key=true` so Lance's bloom-filter conflict resolver rejects two concurrent commits that land the same `object_id` row. Without this annotation, Lance's transparent rebase would admit silent duplicates of `version:T@v=N` from racing publishers (see `.context/merge-insert-cas-granularity.md`). -- **Optimistic concurrency control on publish**: `ManifestBatchPublisher::publish` accepts a `expected_table_versions: HashMap` map. Each entry asserts the manifest's current latest non-tombstoned version for that table is exactly what the caller observed; mismatches surface as `OmniError::Manifest` with `ManifestConflictDetails::ExpectedVersionMismatch { table_key, expected, actual }`. Empty map preserves the legacy "best-effort publish" semantics. The publisher uses `conflict_retries(0)` against Lance and owns retry itself (`PUBLISHER_RETRY_BUDGET = 5`), re-running the pre-check on each iteration so concurrent advances surface as `ExpectedVersionMismatch` rather than being silently rebased through. +- **Atomic publish**: multi-dataset commits publish so that a single write to `__manifest` flips all the new sub-table versions visible at once. +- **Row-level CAS on the merge-insert join key**: `object_id` carries an unenforced-primary-key annotation so Lance's bloom-filter conflict resolver rejects two concurrent commits that land the same `object_id` row. Without this annotation, Lance's transparent rebase would admit silent duplicates from racing publishers. +- **Optimistic concurrency control on publish**: a publish asserts the manifest's current latest non-tombstoned version for each touched table is exactly what the caller observed; mismatches surface as an `ExpectedVersionMismatch` manifest conflict naming the table and the expected/actual versions. Concurrent advances surface as a conflict rather than being silently rebased through. -### Internal schema versioning (`db/manifest/migrations.rs`) +### Internal schema versioning -The on-disk shape of `__manifest` is reconciled with the binary via a single stamp + dispatcher. `INTERNAL_MANIFEST_SCHEMA_VERSION` declares the shape this binary writes; the on-disk stamp `omnigraph:internal_schema_version` lives in the manifest dataset's schema-level metadata (Lance `update_schema_metadata`). +The on-disk shape of `__manifest` is reconciled with the binary via a single version stamp held in the manifest dataset's schema-level metadata. -- **`init_manifest_graph`** stamps the current version at creation, so newly initialized graphs never need migration. -- **Publisher open-for-write path** (`load_publish_state`) calls `migrate_internal_schema(&mut dataset)` before reading state. When the on-disk stamp matches the binary, this is a single metadata read with no writes; otherwise the dispatcher walks `match`-arm steps forward (1→2, 2→3, …) until the stamp matches, then proceeds with the publish. Reads stay side-effect-free. +- **Graph creation** stamps the current version, so newly initialized graphs never need migration. +- **The open-for-write path** migrates the on-disk stamp before reading state. When the stamp matches the binary, this is a single metadata read with no writes; otherwise the migration walks steps forward (1→2, 2→3, …) until the stamp matches, then proceeds with the publish. Reads stay side-effect-free. - **Forward-version protection**: a stamp *higher* than the binary's known version triggers a clear "upgrade omnigraph first" error. An old binary cannot clobber a newer schema by silently treating "unknown stamp" as "missing stamp". -- **Idempotency**: each migration step is safe to re-run. A crash between two metadata updates inside a single step leaves the partial state; the next open re-runs the step and the second update lands. The dispatcher itself is a cheap stamp-read on the steady-state path. - -Adding a new on-disk shape change is one constant bump (`INTERNAL_MANIFEST_SCHEMA_VERSION`), one match arm in `migrate_internal_schema`, and one test. No code outside this module branches on the stamp. +- **Idempotency**: each migration step is safe to re-run. A crash between two metadata updates inside a single step leaves the partial state; the next open re-runs the step and the second update lands. | Stamp | Shape change | |---|---| -| v1 (implicit, pre-stamp) | `__manifest.object_id` had no PK annotation; publisher had no row-level CAS protection. | -| v2 | `__manifest.object_id` carries `lance-schema:unenforced-primary-key=true`; row-level CAS engaged. Stamped as `omnigraph:internal_schema_version=2`. | -| v3 | One-time sweep of legacy `__run__*` staging branches (pre-v0.4.0 Run state machine, removed MR-771) off `__manifest`. Runs at `Omnigraph::open(ReadWrite)` and on publish. Stamped as `omnigraph:internal_schema_version=3`. | +| v1 (implicit, pre-stamp) | `__manifest.object_id` had no PK annotation; no row-level CAS protection. | +| v2 | `__manifest.object_id` carries an unenforced-primary-key annotation; row-level CAS engaged. | +| v3 | One-time sweep of legacy `__run__*` staging branches (pre-v0.4.0 Run state machine, removed) off `__manifest`. Runs at read-write open and on publish. | ## On-disk layout @@ -92,20 +90,20 @@ flowchart TB - **Graph root** is one directory (or S3 prefix). Everything below is part of one OmniGraph graph. - **`__manifest/`** is a Lance dataset whose rows describe which sub-table version is published at which graph-branch. Reading a snapshot starts here. - **`nodes/`** and **`edges/`** are sibling directories holding one Lance dataset per declared type. Names are `fnv1a64-hex` of the type name to keep paths fixed-length and case-safe. -- **`_graph_commits.lance`** is an L2 dataset that records the graph-level commit DAG, with a paired `_graph_commit_actors.lance` for the actor map. (Pre-v0.4.0 graphs also have inert `_graph_runs.lance` / `_graph_run_actors.lance` from the removed Run state machine; the v2→v3 migration sweeps their stale `__run__*` branches, and the dataset bytes are reclaimed once `delete_prefix` lands.) -- **`_graph_commit_recoveries.lance`** — one row per recovery sweep action. Joined to `_graph_commits.lance` by `graph_commit_id`; the linked commit row carries `actor_id=omnigraph:recovery`. Operators correlate recoveries with the original mutations they rolled forward / back via this join. See `crates/omnigraph/src/db/recovery_audit.rs`. -- **`__recovery/{ulid}.json`** — transient sidecar files written by the five migrated writers (`MutationStaging::finalize`, `schema_apply`, `branch_merge`, `ensure_indices`, `optimize_all_tables`) before Phase B begins, deleted after Phase C succeeds. A sidecar persisting after process exit means the writer crashed in the Phase B → Phase C window; the next `Omnigraph::open` recovery sweep processes it. Steady-state directory is empty. See `crates/omnigraph/src/db/manifest/recovery.rs`. +- **`_graph_commits.lance`** is an L2 dataset that records the graph-level commit DAG, with a paired `_graph_commit_actors.lance` for the actor map. (Pre-v0.4.0 graphs also have inert `_graph_runs.lance` / `_graph_run_actors.lance` from the removed Run state machine; the internal schema migration sweeps their stale `__run__*` branches, and the dataset bytes are reclaimed once a prefix-delete primitive lands.) +- **`_graph_commit_recoveries.lance`** — one row per crash-recovery action. Joined to `_graph_commits.lance` by `graph_commit_id`; the linked commit row carries `actor_id=omnigraph:recovery`. Operators correlate recoveries with the original mutations they rolled forward / back via this join. +- **`__recovery/{ulid}.json`** — transient sidecar files written by a writer before it advances the underlying dataset, deleted once the matching manifest publish succeeds. A sidecar persisting after process exit means the writer crashed mid-commit; the next read-write open processes it. Steady-state directory is empty. - **`_refs/branches/{name}.json`** is graph-level branch metadata — pointers from a branch name to the manifest version it heads. - **Inside each Lance dataset** (orange): the standard Lance directory layout. `_versions/{n}.manifest` records every commit; `data/` holds the actual Arrow fragments; `_indices/{uuid}/` holds index segments with their own `fragment_bitmap` for partial coverage; `_refs/` holds Lance-native per-dataset branches and tags. The split — L2 owns the cross-dataset catalog; L1 owns the per-dataset internals — means that schema work (which adds or removes datasets) updates `__manifest`, while data work (which adds fragments) updates `_versions/` inside the affected dataset and then bumps `__manifest`. -## URI scheme support (`storage.rs`) +## URI scheme support | Scheme | Backend | Notes | |---|---|---| -| local path / `file://` | `ObjectStorageAdapter` over `object_store::LocalFileSystem` | Normalized to absolute paths; relative and dot-segment paths are lexically absolutized | -| `s3://bucket/prefix` | `ObjectStorageAdapter` over `object_store` S3 | Honors `AWS_ENDPOINT_URL_S3`, `AWS_ALLOW_HTTP`, `AWS_S3_FORCE_PATH_STYLE` | +| local path / `file://` | local filesystem | Normalized to absolute paths; relative and dot-segment paths are lexically absolutized | +| `s3://bucket/prefix` | S3 object store | Honors `AWS_ENDPOINT_URL_S3`, `AWS_ALLOW_HTTP`, `AWS_S3_FORCE_PATH_STYLE` | | `http(s)://host:port` | HTTP client to `omnigraph-server` | Used by CLI as a target, not a storage backend | ## Object-store env vars (S3-compatible) diff --git a/docs/user/constants.md b/docs/user/constants.md deleted file mode 100644 index f523042..0000000 --- a/docs/user/constants.md +++ /dev/null @@ -1,40 +0,0 @@ -# Constants & Tunables (cheat sheet) - -| Name | Value | Where | -|---|---|---| -| `MANIFEST_DIR` | `__manifest` | `db/manifest/layout.rs` | -| Commit graph dir | `_graph_commits.lance` | `db/commit_graph.rs` | -| Run registry dir (legacy, removed MR-771) | `_graph_runs.lance` | inert post-v0.4.0; bytes remain until a `delete_prefix` primitive lands | -| Run branch prefix (legacy, removed MR-771/MR-770) | `__run__` | swept off `__manifest` by the v2→v3 migration; no longer a reserved name | -| Schema apply lock | `__schema_apply_lock__` | `db/mod.rs` | -| Manifest publisher retry budget | `PUBLISHER_RETRY_BUDGET = 5` | `db/manifest/publisher.rs` | -| Internal manifest schema version | `INTERNAL_MANIFEST_SCHEMA_VERSION = 3` | `db/manifest/migrations.rs` | -| Merge stage batch | `MERGE_STAGE_BATCH_ROWS = 8192` | `exec/merge.rs` | -| Maintenance concurrency | `OMNIGRAPH_MAINTENANCE_CONCURRENCY=8` | `db/omnigraph/optimize.rs` | -| Lance blob compaction support | `LANCE_SUPPORTS_BLOB_COMPACTION = false` | `db/omnigraph/optimize.rs` | -| Graph index cache size | `8` (LRU) | `runtime_cache.rs` | -| Expand indexed-path frontier ceiling | `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER=1024` | `exec/query.rs` | -| Expand indexed-path hop ceiling | `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS=6` | `exec/query.rs` | -| Expand CSR-build cost factor | `CSR_BUILD_FACTOR = 1.5` | `exec/query.rs` | -| Expand mode override | `OMNIGRAPH_TRAVERSAL_MODE` (`indexed`\|`csr`; unset = cost-based auto) | `exec/query.rs` | -| Default body limit | `1 MB` | `omnigraph-server/lib.rs` | -| Ingest body limit | `32 MB` | `omnigraph-server/lib.rs` | -| Engine embed model | `gemini-embedding-2-preview` | `omnigraph/embedding.rs` | -| Compiler embed model | `text-embedding-3-small` | `omnigraph-compiler/embedding.rs` | -| Embed timeout | `30 000 ms` | both clients | -| Embed retries | `4` | both clients | -| Embed retry backoff | `200 ms` | both clients | -| LANCE memory pool default | `1 GB` (raised in v0.3.0) | runtime | - -**Expand traversal dispatch.** With `OMNIGRAPH_TRAVERSAL_MODE` unset, the engine -chooses the indexed (per-hop BTREE) vs CSR (whole-graph in-memory) path with a -cost model over cheap manifest counts (frontier size, |E|, source-vertex count, -hops) plus the index-coverage signal: the indexed path is preferred when its -frontier-relative work beats building the CSR (≈ when `hops × frontier` is a -small fraction of the source-vertex set), and CSR is preferred for dense/deep -traversals or when the BTREE coverage is degraded and a full scan would be paid -per hop. The two ceilings bound the **initial dispatch** frontier/hops (beyond -them CSR is always used); they are not a hard per-hop bound — the cost model -*estimates* total indexed work as ~`hops × frontier × fanout`, so dense fan-out is -priced toward CSR rather than capped mid-traversal. The override flag forces a path (the `auto` result is identical either way; -only the path differs). diff --git a/docs/user/deployment.md b/docs/user/deployment.md index ece7b5d..a0d8e9f 100644 --- a/docs/user/deployment.md +++ b/docs/user/deployment.md @@ -13,13 +13,10 @@ Omnigraph supports two broad deployment shapes: The server binary and container image expose the same HTTP surface. -The server also has two **boot sources**: `omnigraph.yaml` (graph targets -declared in the per-operator config) or a **cluster directory** -(`omnigraph-server --cluster `), which serves the cluster control +The server has a single **boot source**: a **cluster directory** +(`omnigraph-server --cluster `), which serves the cluster control plane's applied revision — see -[cluster-config.md](cluster-config.md#serving-from-the-cluster-the-mode-switch). -The two are exclusive per deployment; switching is a restart with a different -flag. +[cluster-config.md](clusters/config.md#serving-from-the-cluster-the-mode-switch). ## Binary Deployment @@ -30,25 +27,29 @@ Build or install: On Windows, the binaries are `omnigraph.exe` and `omnigraph-server.exe`. -Run against a local graph: +The server boots from a cluster only (RFC-011) — there is no positional +`` / single-graph boot. Point it at a local cluster directory: ```bash -omnigraph-server graph.omni --bind 0.0.0.0:8080 +omnigraph-server --cluster ./company-brain --bind 0.0.0.0:8080 ``` -Run against an object-store-backed graph: +Or boot config-free from an object-storage-rooted cluster: ```bash OMNIGRAPH_SERVER_BEARER_TOKEN="change-me" \ AWS_REGION="us-east-1" \ -omnigraph-server s3://my-bucket/graphs/example/releases/2026-04-10-v0.1.0 \ +omnigraph-server --cluster s3://my-bucket/clusters/company-brain \ --bind 0.0.0.0:8080 ``` +The server serves every graph in the cluster's applied revision under +`/graphs/{id}/...`. See [clusters](clusters/index.md) for authoring and +applying a cluster. + ## Cluster Mode in Containers (AWS, Railway) -A cluster-booted deployment has **two shapes** since the `storage:` root -(RFC-006): +A cluster-booted deployment has **two shapes** since the `storage:` root: - **Bucket, no volume (preferred for cloud)** — the cluster's ledger, catalog, and graph data live under an object-storage root @@ -81,10 +82,8 @@ docker run -d \ -p 8080:8080 ``` -`OMNIGRAPH_CLUSTER` is exclusive: combining it with `OMNIGRAPH_TARGET_URI`, -`OMNIGRAPH_CONFIG`, or `OMNIGRAPH_TARGET` fails fast (exit 64), the same -rule the server itself enforces. The image also ships the `omnigraph` CLI, -so the day-2 loop runs in-container with no `omnigraph.yaml`: +`OMNIGRAPH_CLUSTER` is the server's only boot source. The image also +ships the `omnigraph` CLI, so the day-2 loop runs in-container: ```bash docker exec -it sh -c \ @@ -105,10 +104,10 @@ docker exec -it sh -c \ `omnigraph cluster apply --as --config /var/lib/omnigraph/cluster` → force a new deployment (restart). -For a deployment that doesn't need the cluster control plane, the classic -stateless shape — `OMNIGRAPH_TARGET_URI=s3://bucket/graph.omni`, no volume — -remains the simplest AWS architecture (see Binary/Container Deployment -above). +For a stateless, volume-free deployment, root the cluster on object +storage and boot config-free with +`OMNIGRAPH_CLUSTER=s3://bucket/clusters/` (the bucket-no-volume +shape above) — the simplest AWS architecture. ### Railway @@ -130,49 +129,46 @@ above). unvalidated** — boot is lock-free read-only so it should compose, but it is not yet exercised by tests. -## One-Command Local RustFS Bootstrap +## Testing against S3 locally -The easiest local S3-backed deployment path is: +To exercise the S3 storage path without a cloud account, run any S3-compatible +store in Docker and point the standard `AWS_*` environment at it. RustFS is +shown; MinIO works the same way. ```bash -curl -fsSL https://raw.githubusercontent.com/ModernRelay/omnigraph/main/scripts/local-rustfs-bootstrap.sh | bash +docker run -d --name omnigraph-s3 -p 9000:9000 \ + -e RUSTFS_ACCESS_KEY=omnigraph -e RUSTFS_SECRET_KEY=omnigraph \ + -e RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS=true \ + rustfs/rustfs:latest /data + +export AWS_ACCESS_KEY_ID=omnigraph AWS_SECRET_ACCESS_KEY=omnigraph \ + AWS_REGION=us-east-1 AWS_ENDPOINT_URL_S3=http://127.0.0.1:9000 \ + AWS_ALLOW_HTTP=true AWS_S3_FORCE_PATH_STYLE=true + +# create the bucket once (any S3 client works) +aws --endpoint-url "$AWS_ENDPOINT_URL_S3" s3 mb s3://omnigraph-local ``` -The bootstrap: +Now an `s3://…` URI works anywhere a graph or cluster root is expected. Root a +cluster on the bucket and serve it config-free: -- starts a local RustFS-backed object store -- creates a bucket and S3-backed Omnigraph graph -- loads the checked-in context fixture -- starts `omnigraph-server` on `127.0.0.1:8080` +```bash +# cluster.yaml +# version: 1 +# storage: s3://omnigraph-local/clusters/demo +# graphs: { demo: { schema: schema.pg } } -Supported behavior: +omnigraph cluster validate --config . +omnigraph cluster import --config . +omnigraph cluster apply --config . --as you +omnigraph load --data seed.jsonl --mode merge \ + s3://omnigraph-local/clusters/demo/graphs/demo.omni +omnigraph-server --cluster s3://omnigraph-local/clusters/demo \ + --bind 127.0.0.1:8080 --unauthenticated +``` -- downloads the rolling `edge` binary when one exists for the current platform -- otherwise clones `ModernRelay/omnigraph` and builds from source -- reuses an existing RustFS container if it is already running - -Useful overrides: - -- `WORKDIR=/path/to/state` -- `BUCKET=omnigraph-local` -- `PREFIX=graphs/context` -- `RESET_REPO=1` to delete an existing partially initialized graph prefix before recreating it -- `BIND=127.0.0.1:8080` -- `RUSTFS_CONTAINER_NAME=omnigraph-rustfs-demo` - -The bootstrap expects: - -- Docker -- `curl` -- either a matching release asset or a local Rust toolchain plus `git` - -If `aws` is not installed, the script attempts a user-local AWS CLI install via -`python3 -m pip`. Docker Desktop or another Docker daemon must already be -running. - -If a previous bootstrap left objects behind under the selected `PREFIX` but did -not finish initializing the graph, rerun with `RESET_REPO=1` or choose a new -`PREFIX`. +The same `AWS_*` contract applies to a production object store — swap the +endpoint and credentials. CI exercises this path against containerized RustFS. ## Container Deployment @@ -182,23 +178,24 @@ Build the image: docker build -t omnigraph-server:local . ``` -Run against a local graph: +The server boots from a cluster only (RFC-011). Run against a cluster +directory on a mounted volume: ```bash docker run --rm -p 8080:8080 \ - -v "$PWD/graph.omni:/data/graph.omni" \ + -v "$PWD/company-brain:/var/lib/omnigraph/cluster" \ omnigraph-server:local \ - /data/graph.omni --bind 0.0.0.0:8080 + --cluster /var/lib/omnigraph/cluster --bind 0.0.0.0:8080 ``` -Run against an S3-backed graph: +Run config-free against an object-storage-rooted cluster: ```bash docker run --rm -p 8080:8080 \ -e OMNIGRAPH_SERVER_BEARER_TOKEN="change-me" \ -e AWS_REGION="us-east-1" \ omnigraph-server:local \ - s3://my-bucket/graphs/example/releases/2026-04-10-v0.1.0 \ + --cluster s3://my-bucket/clusters/company-brain \ --bind 0.0.0.0:8080 ``` @@ -209,27 +206,14 @@ When no positional args are given, the image entrypoint | Var | Effect | |---|---| -| `OMNIGRAPH_TARGET_URI` | Graph URI, passed as the positional argument. | -| `OMNIGRAPH_CONFIG` | Path to an `omnigraph.yaml`, passed as `--config`. Used to supply a `policy.file` (Cedar authorization). The config file and any relative `policy.file` must be mounted into the container. | -| `OMNIGRAPH_TARGET` | Graph name to select from the config's `graphs:` block (with `OMNIGRAPH_CONFIG`, when no `OMNIGRAPH_TARGET_URI`). | +| `OMNIGRAPH_CLUSTER` | Cluster boot source — a config directory or a storage-root URI, forwarded as `--cluster`. The only boot source. | | `OMNIGRAPH_BIND` | Listen address (default `0.0.0.0:8080`). | -`OMNIGRAPH_TARGET_URI` and `OMNIGRAPH_CONFIG` **compose**: set both to keep the -graph URI in the env var while loading policy from the config file (the -positional URI wins over any `graphs:` entry). To enable Cedar policy on a -container otherwise driven by `OMNIGRAPH_TARGET_URI`, mount the config dir and -add `OMNIGRAPH_CONFIG`: - -```bash -docker run --rm -p 8080:8080 \ - -e OMNIGRAPH_SERVER_BEARER_TOKEN="change-me" \ - -e OMNIGRAPH_TARGET_URI="s3://my-bucket/graphs/example/releases/2026-04-10-v0.1.0" \ - -e OMNIGRAPH_CONFIG="/etc/omnigraph/omnigraph.yaml" \ - -v "$PWD/config:/etc/omnigraph:ro" \ - omnigraph-server:local -# /etc/omnigraph/omnigraph.yaml contains `policy: { file: policy.yaml }`; -# policy.yaml (+ optional policy.tests.yaml) sit beside it in the mount. -``` +Per-graph and server-level Cedar policy come from the cluster's applied +revision (authored in `cluster.yaml` and published with `cluster apply`), +not from a separate config file. The cluster docker shapes — volume vs. +config-free object-storage root — are detailed under +[Cluster Mode in Containers](#cluster-mode-in-containers-aws-railway) above. ## Auth diff --git a/docs/user/embeddings.md b/docs/user/embeddings.md deleted file mode 100644 index 382e683..0000000 --- a/docs/user/embeddings.md +++ /dev/null @@ -1,31 +0,0 @@ -# Embeddings - -OmniGraph has **two** embedding clients with different defaults and purposes. - -## Compiler-side client (`omnigraph-compiler/src/embedding.rs`) — query-time normalization - -- Default model: `text-embedding-3-small` (OpenAI-style schema) -- Env: `NANOGRAPH_EMBED_MODEL`, `OPENAI_API_KEY`, `OPENAI_BASE_URL` (default `https://api.openai.com/v1`), `NANOGRAPH_EMBEDDINGS_MOCK`, `NANOGRAPH_EMBED_TIMEOUT_MS=30000`, `NANOGRAPH_EMBED_RETRY_ATTEMPTS=4`, `NANOGRAPH_EMBED_RETRY_BACKOFF_MS=200` -- Methods: `embed_text(input, expected_dim)`, `embed_texts(inputs, expected_dim)` -- Mock mode: deterministic FNV-1a + xorshift64 → L2-normalized vectors - -## Engine-side client (`omnigraph/src/embedding.rs`) — runtime ingest - -- Model: `gemini-embedding-2-preview` -- Env: `GEMINI_API_KEY`, `OMNIGRAPH_GEMINI_BASE_URL` (default Google generativelanguage v1beta), `OMNIGRAPH_EMBED_TIMEOUT_MS=30000`, `OMNIGRAPH_EMBED_RETRY_ATTEMPTS=4`, `OMNIGRAPH_EMBED_RETRY_BACKOFF_MS=200`, `OMNIGRAPH_EMBEDDINGS_MOCK` -- Two task types: `embed_query_text` (RETRIEVAL_QUERY) and `embed_document_text` (RETRIEVAL_DOCUMENT) -- Exponential backoff with retryable detection (timeouts, 429, 5xx) - -## Schema integration - -Mark a Vector property with `@embed("source_text_property")`. At ingest, the engine pulls the source text and writes the embedding into the vector column. Stored as L2-normalized FixedSizeList(Float32, dim). - -## CLI `omnigraph embed` (offline file pipeline) - -Operates on **JSONL files** (not on a graph). Three modes (mutually exclusive): - -- (default) `fill_missing` — only embed rows whose target field is empty -- `--reembed-all` — overwrite all -- `--clean` — strip embeddings - -Inputs are either a single seed manifest YAML or `--input/--output/--spec`. Selectors `--type T`, `--select T:field=value` filter rows. Streams JSONL → JSONL. diff --git a/docs/user/index.md b/docs/user/index.md index 956fa0b..cabd98a 100644 --- a/docs/user/index.md +++ b/docs/user/index.md @@ -2,44 +2,68 @@ **Audience:** users, CLI users, HTTP clients, and self-hosting operators -This is the public-facing entry point. These docs should describe behavior, -commands, configuration, and operational contracts without requiring knowledge -of MRs, internal recovery mechanics, or contributor-only invariants. +This is the public-facing entry point. These docs describe behavior, commands, +configuration, and operational contracts without requiring knowledge of internal +recovery mechanics or contributor-only invariants. They are organized by topic — +start with install, then follow the section that matches your task. -## Start Here +## Start here | Goal | Read | |---|---| | Install OmniGraph | [install.md](install.md) | -| Run the CLI locally | [cli.md](cli.md) | -| Look up every CLI flag and config field | [cli-reference.md](cli-reference.md) | -| Deploy and operate a cluster (how-to guide) | [cluster.md](cluster.md) | -| Validate and plan cluster config | [cluster-config.md](cluster-config.md) | -| Write schemas | [schema-language.md](schema-language.md) | -| Read schema-lint diagnostic codes | [schema-lint.md](schema-lint.md) | -| Write queries and mutations | [query-language.md](query-language.md) | -| Use embeddings | [embeddings.md](embeddings.md) | +| Run the core loop end to end | [quickstart.md](quickstart.md) | +| Understand the model | [concepts/index.md](concepts/index.md) | +| Run the CLI | [cli/index.md](cli/index.md) | +| Look up every CLI flag and config field | [cli/reference.md](cli/reference.md) | -## Operate A Graph +## Schema & queries | Goal | Read | |---|---| -| Understand graph layout and URI support | [storage.md](storage.md) | -| Work with branches, commits, and snapshots | [branches-commits.md](branches-commits.md) | -| Coordinate multi-query workflows | [transactions.md](transactions.md) | -| Read diffs and change feeds | [changes.md](changes.md) | -| Build and use indexes | [indexes.md](indexes.md) | -| Compact and clean old versions | [maintenance.md](maintenance.md) | -| Interpret errors and output formats | [errors.md](errors.md) | +| Write schemas (the `.pg` language) | [schema/index.md](schema/index.md) | +| Read schema-lint diagnostic codes | [schema/lint.md](schema/lint.md) | +| Write queries (the `.gq` language) | [queries/index.md](queries/index.md) | +| Write data — inserts, updates, deletes | [mutations/index.md](mutations/index.md) | +| Use vector / full-text / hybrid search | [search/index.md](search/index.md) | +| Generate embeddings | [search/embeddings.md](search/embeddings.md) | +| Build and use indexes | [search/indexes.md](search/indexes.md) | -## Run The Server +## Branching & version control + +| Goal | Read | +|---|---| +| Work with branches and commits | [branching/index.md](branching/index.md) | +| Read past versions (time travel) | [branching/time-travel.md](branching/time-travel.md) | +| Merge branches and resolve conflicts | [branching/merge.md](branching/merge.md) | +| Coordinate multi-query workflows | [branching/transactions.md](branching/transactions.md) | +| Read diffs and change feeds | [branching/changes.md](branching/changes.md) | + +## Operations | Goal | Read | |---|---| | Deploy the binary or container | [deployment.md](deployment.md) | -| Use HTTP endpoints | [server.md](server.md) | -| Configure Cedar authorization | [policy.md](policy.md) | -| Track actors and audit behavior | [audit.md](audit.md) | +| Use HTTP endpoints | [operations/server.md](operations/server.md) | +| Compact, repair, and clean old versions | [operations/maintenance.md](operations/maintenance.md) | +| Configure Cedar authorization | [operations/policy.md](operations/policy.md) | +| Track actors and audit behavior | [operations/audit.md](operations/audit.md) | +| Interpret errors and output formats | [operations/errors.md](operations/errors.md) | + +## Clusters + +| Goal | Read | +|---|---| +| Deploy and operate a cluster (how-to) | [clusters/index.md](clusters/index.md) | +| Reference every `cluster.yaml` key and command | [clusters/config.md](clusters/config.md) | + +## Concepts & reference + +| Goal | Read | +|---|---| +| Understand the model and L1/L2 framing | [concepts/index.md](concepts/index.md) | +| Understand graph layout and URI support | [concepts/storage.md](concepts/storage.md) | +| Look up constants and tunables | [reference/constants.md](reference/constants.md) | ## Releases @@ -48,7 +72,6 @@ changes between versions, not for contributor design history. ## Boundary -User docs should focus on stable behavior. If a paragraph needs to explain -internal sidecars, Lance API blockers, MR numbers, test strategy, or review -rules, it probably belongs in [docs/dev/index.md](../dev/index.md) or a developer-area document -instead. +User docs focus on stable behavior. If a paragraph needs to explain internal +sidecars, Lance API blockers, or test strategy, it probably belongs in +[docs/dev/index.md](../dev/index.md) or a developer-area document instead. diff --git a/docs/user/indexes.md b/docs/user/indexes.md deleted file mode 100644 index df898c4..0000000 --- a/docs/user/indexes.md +++ /dev/null @@ -1,26 +0,0 @@ -# Indexes - -## L1 — Lance index types OmniGraph exposes - -| Index | Use | Notes | -|---|---|---| -| **BTREE scalar** | range / equality on any scalar | created on `@key`, `@index(...)`, and on key columns by `ensure_indices()` | -| **Inverted (FTS)** | `search`, `fuzzy`, `match_text`, `bm25` | created on text columns referenced by FTS queries | -| **Vector** | `nearest()` k-NN | Lance picks IVF_PQ vs HNSW family by configuration; OmniGraph stores as FixedSizeList(Float32, dim) | - -## L2 — OmniGraph orchestration - -- `ensure_indices()` / `ensure_indices_on(branch)` — idempotent build of BTREE + inverted indexes for the current head; safe to re-run. -- Indexes are built on the *branch head* (not on a snapshot), so reads always see the current index state. -- **Lazy branch forking for indexes**: a branch that hasn't mutated a sub-table doesn't need its own index — the main lineage's index is reused until the first write triggers a copy-on-write fork. -- Vector index parameters (metric, nlist, nprobe, etc.) are not exposed in the schema; they default at the Lance layer and are picked up automatically when an index is asked for on a Vector column. - -## L2 — Graph topology index (`graph_index/mod.rs`) - -This is OmniGraph-specific (not Lance): - -- `TypeIndex`: dense `u32 ↔ String id` mapping per node type. -- `CsrIndex`: Compressed Sparse Row representation of edges per edge type — `offsets[i]..offsets[i+1]` slices into `targets`. -- `GraphIndex { type_indices, csr (out), csc (in) }` — built on demand from a snapshot's edge tables, **lazily**: only when an `Expand` the planner routes to the CSR path (dense / large frontier) or an `AntiJoin` actually needs it. -- Cached in `RuntimeCache::graph_indices` (LRU, max 8 entries, keyed by snapshot id + edge table versions). -- Selective `Expand`s resolve neighbors from the persisted `src`/`dst` BTREE instead (one indexed scan per hop) and never trigger the CSR build; see [query-language](query-language.md) → Expand. Pure scans, and queries served entirely by the indexed traversal path, skip it. diff --git a/docs/user/maintenance.md b/docs/user/maintenance.md deleted file mode 100644 index e69bba3..0000000 --- a/docs/user/maintenance.md +++ /dev/null @@ -1,47 +0,0 @@ -# Maintenance: Optimize, Repair & Cleanup - -`db/omnigraph/optimize.rs` and `db/omnigraph/repair.rs`. - -## `optimize_all_tables(db)` — non-destructive - -- Lance `compact_files()` on every node + edge table on `main`, then **publishes the compacted version to the `__manifest`** so the manifest's `table_version` tracks the compacted Lance HEAD. Reads pin the manifest version, so without this publish compaction would be invisible to readers *and* would break the HEAD-vs-manifest precondition of the next schema apply / strict update/delete ("stale view … refresh and retry"). The publish advances the graph version (a system-attributed commit) only for tables that actually compacted. -- Rewrites small fragments into fewer large ones; old fragments remain reachable via older manifests until `cleanup` runs. -- Each table's compact→publish runs under its per-`(table, main)` write queue (serializing with concurrent mutations — compaction is a Lance `Rewrite` op that retryable-conflicts with a concurrent merge/update/delete on overlapping fragments). The Lance-HEAD-before-manifest-publish gap is covered by a `SidecarKind::Optimize` recovery sidecar (loose-match): a crash in that window rolls the compacted version forward on the next `Omnigraph::open` (compaction is content-preserving, so roll-forward is always safe). -- **Requires a recovered graph.** `optimize` refuses (errors) when an unresolved recovery sidecar is present under `__recovery` — operating on an unrecovered graph could publish a partial write the open-time recovery sweep would roll back. Reopen the graph to run the recovery sweep, then re-run `optimize`. -- **Uncovered drift is skipped, not interpreted.** If a table's Lance HEAD is ahead of the version recorded in `__manifest` and no recovery sidecar covers that movement, `optimize` reports `skipped: Some(DriftNeedsRepair)` with the manifest/head versions and leaves the table untouched. Run `omnigraph repair` to classify and explicitly publish that drift. -- Bounded by `OMNIGRAPH_MAINTENANCE_CONCURRENCY` (default 8). -- Returns `[TableOptimizeStats { table_key, fragments_removed, fragments_added, committed, skipped, manifest_version, lance_head_version }]`. -- **Blob tables are skipped.** A table that declares any `Blob` property is not compacted: it is reported with `skipped: Some(BlobColumnsUnsupportedByLance)` (and logged via `tracing::warn`) instead of compacted, and the rest of the sweep proceeds normally. The current Lance `compact_files` mis-decodes blob-v2 columns under its forced `BlobHandling::AllBinary` read; **reads and writes are unaffected** — only compaction is. This is gated by `LANCE_SUPPORTS_BLOB_COMPACTION` (`db/omnigraph/optimize.rs`) and removed when the upstream Lance fix lands (see [docs/dev/lance.md](../dev/lance.md)). Consequence: fragment count and deleted-row space on blob tables are not reclaimed until then; query results are never affected. - -## `repair_all_tables(db, options)` — explicit - -- Handles **uncovered manifest/head drift**: a table's Lance HEAD is ahead of the manifest pin and no recovery sidecar records the writer intent. -- Preview by default. `omnigraph repair --json ` reports each table's `classification`, `action`, manifest/head versions, Lance operation names, and any classification error. `--confirm` publishes only verified maintenance drift; if any suspicious or unverifiable table is refused, the CLI prints the per-table output and exits non-zero. `--force --confirm` also publishes suspicious or unverifiable drift after operator review. -- Classifies drift by reading Lance transactions from `manifest_version + 1` through `lance_head_version`. Only `ReserveFragments` and `Rewrite` are verified maintenance. Semantic operations such as `Append`, `Delete`, `Update`, `Merge`, or missing transaction history are not auto-healed. -- Publishes repair by advancing `__manifest` to the existing Lance HEAD; it does **not** rewrite Lance data. If the publish succeeds, normal reads and strict writes use the repaired version. If it fails, no new data-side partial state was created. -- Requires a clean recovery state. Pending `__recovery` sidecars still belong to automatic sidecar recovery, not manual repair. - -## `cleanup_all_tables(db, options)` — destructive - -- Lance `cleanup_old_versions()` per table. -- Removes manifests (and their unique fragments) older than the retention policy. -- `CleanupPolicyOptions { keep_versions: Option, older_than: Option }` — at least one is required. -- Returns `[TableCleanupStats { table_key, bytes_removed, old_versions_removed, error }]`. -- **Fault-isolated per table.** A single table's transient failure (version GC or - orphan reclaim) is recorded on that table's stats row (`error: Some(..)`, logged - via `tracing`) and never aborts the healthy tables — cleanup is the convergence - backstop, so it does as much as it can and converges on re-run. The CLI reports - any failed tables; rerun `cleanup` to retry them. -- CLI guards with `--confirm`; without it, prints a preview line. -- **Recovery floor:** `--keep < 3` may garbage-collect Lance versions that the open-time recovery sweep needs as a rollback target (the sweep restores to the branch's manifest-pinned table version, which is HEAD-1 in the typical Phase B → Phase C drift case). Default `--keep 10` is safe. -- **Orphaned-branch reconciliation:** before the version GC, cleanup runs `reconcile_orphaned_branches`, which `force_delete_branch`es any per-table or commit-graph Lance branch absent from the manifest branch list. These orphans arise when a `branch_delete` flips the manifest authority but a downstream best-effort reclaim does not complete (see [branches-commits.md](branches-commits.md)). The reconciler is authority-derived and idempotent (it no-ops once nothing is orphaned), runs regardless of the `keep_versions` / `older_than` values (those gate version GC only), and never reclaims `main` or system-branch forks. Reclaimed forks are logged via `tracing::info`. - -## Tombstones - -Logical sub-table delete markers in `__manifest`; `tombstone_object_id(table_key, version)` excludes a sub-table version from snapshot reconstruction. - -## Internal schema migrations (`db/manifest/migrations.rs`) - -Version evolutions of the on-disk `__manifest` shape are reconciled automatically on the first write under a new binary. `INTERNAL_MANIFEST_SCHEMA_VERSION` declares the shape the binary expects; the on-disk stamp `omnigraph:internal_schema_version` (Lance schema-level metadata) records the on-disk shape. The publisher's open-for-write path calls `migrate_internal_schema` before reading state; reads are side-effect-free. No operator action is required for in-place upgrades. See [storage.md → Internal schema versioning](storage.md) for the full mechanism. - -A binary opening a manifest stamped at a version *higher* than it knows about refuses to publish with a clear "upgrade omnigraph first" error — old binaries cannot clobber a newer schema. diff --git a/docs/user/mutations/index.md b/docs/user/mutations/index.md new file mode 100644 index 0000000..2602ae5 --- /dev/null +++ b/docs/user/mutations/index.md @@ -0,0 +1,52 @@ +# Mutations + +Write statements live inside a `query` declaration whose body is one or more +mutation statements (the [query language](../queries/index.md) covers the read +shape and shared declaration syntax). + +``` +query onboard($name: String, $title: String) { + insert Person { name: $name, title: $title } +} +``` + +An edge type is inserted the same way — its endpoint columns are just +properties in the assignment block (`insert WorksAt { person: $p, org: $o }`). + +## Statements + +- `insert { prop: , … }` +- `update set { prop: , … } where ` +- `delete where ` + +`` is a literal, `$param`, or `now()`. + +## Atomicity + +A change query publishes **one commit** at the end of the query. Multiple +insert/update statements accumulate in memory and commit together — a mid-query +failure leaves the graph untouched. See [transactions](../branching/transactions.md) +for the per-query atomicity contract and [branches](../branching/index.md) for +multi-query workflows. + +## Inserts/updates and deletes cannot mix in one query + +A single change query must be **either insert/update-only or delete-only**. +Mixing the two is rejected at parse time, before any I/O: + +> `mutation '' on the same query mixes inserts/updates and deletes; split +> into separate mutations: (1) inserts and updates, then (2) deletes.` + +Run two separate queries instead — the inserts/updates first, then the deletes. +The restriction exists because inserts/updates and deletes commit through +different paths today, and mixing them in one query creates ordering hazards +(e.g. a same-row insert-then-delete, or a cascading delete of a just-inserted +edge). Keeping the two kinds in separate queries keeps each one atomic and +correct. + +## Bulk loading + +For loading data from files rather than inline statements, use +[`omnigraph load`](../cli/index.md) (`--mode overwrite|append|merge`) — it is the +single bulk-write command and applies the same schema validation and atomic +publish as inline mutations. diff --git a/docs/user/operations/audit.md b/docs/user/operations/audit.md new file mode 100644 index 0000000..7e8b24d --- /dev/null +++ b/docs/user/operations/audit.md @@ -0,0 +1,46 @@ +# Audit & Actor Tracking + +Every write in OmniGraph records **who made it**. The actor id is persisted on the +graph commit, so the commit history is an audit trail of which actor changed the +graph and when. + +## Where the actor comes from + +The actor is resolved differently depending on the front end, but it always lands +on the commit: + +- **HTTP server** — the actor is resolved **server-side from the bearer token**. A + client cannot set its own actor id; it is derived from the authenticated token. + See [policy](policy.md) for how tokens map to actors. +- **CLI / embedded** — the actor is self-declared through one resolution chain: + + 1. `--as ` on the command, + 2. then `operator.actor` in `~/.omnigraph/config.yaml` (see the + [CLI reference](../cli/reference.md)), + 3. otherwise none. + +This difference is intentional: storage credentials imply a self-declared actor, +while a server resolves the actor from a token it trusts. + +## Reading the audit trail + +Actor ids are stored on each commit in the [commit graph](../branching/index.md). +List commits to see who made each change: + +```bash +omnigraph commit list graph.omni +``` + +System-initiated writes use reserved actor ids — for example, automatic recovery +of an interrupted write records `omnigraph:recovery`, so operator changes and +machine repairs are distinguishable in the history: + +```bash +omnigraph commit list --filter actor=omnigraph:recovery graph.omni +``` + +## What is tracked + +Every successful publish — load, change, branch merge, and schema apply — appends a +commit carrying the resolving actor. Because publishes are atomic, the actor on a +commit is exactly the actor responsible for that whole change. diff --git a/docs/user/errors.md b/docs/user/operations/errors.md similarity index 76% rename from docs/user/errors.md rename to docs/user/operations/errors.md index 8373b0d..48f1fc9 100644 --- a/docs/user/errors.md +++ b/docs/user/operations/errors.md @@ -9,7 +9,7 @@ - `Manifest(ManifestError { kind: BadRequest|NotFound|Conflict|Internal, details: Option, … })` - `ManifestConflictDetails::ExpectedVersionMismatch { table_key, expected, actual }` — caller's `expected_table_versions` did not match the manifest's current latest non-tombstoned version (set by `OmniError::manifest_expected_version_mismatch`). - `ManifestConflictDetails::RowLevelCasContention` — Lance row-level CAS rejected the publish because a concurrent writer landed the same `object_id`. Retried internally by the publisher; only surfaces if the retry budget exhausts. - - **D₂ parse-time rejection** (MR-794): a single mutation query that mixes inserts/updates with deletes errors out *before any I/O* with kind `BadRequest`. Message: `mutation '' on the same query mixes inserts/updates and deletes; split into separate mutations: (1) inserts and updates, then (2) deletes`. See [docs/user/query-language.md](query-language.md) for the rule and [docs/dev/writes.md](../dev/writes.md) for the underlying staged-write rationale. + - **D₂ parse-time rejection**: a single mutation query that mixes inserts/updates with deletes errors out *before any I/O* with kind `BadRequest`. Message: `mutation '' on the same query mixes inserts/updates and deletes; split into separate mutations: (1) inserts and updates, then (2) deletes`. See [query-language.md](../queries/index.md) for the rule. - `MergeConflicts(Vec)` Compiler-side `NanoError` covers parse / catalog / type / storage / plan / execution / arrow / lance / IO / manifest / unique-constraint, each with structured spans (`SourceSpan { start, end }`) for ariadne-style diagnostics. diff --git a/docs/user/operations/maintenance.md b/docs/user/operations/maintenance.md new file mode 100644 index 0000000..161e5d6 --- /dev/null +++ b/docs/user/operations/maintenance.md @@ -0,0 +1,50 @@ +# Maintenance: Optimize, Repair & Cleanup + +**Addressing.** `optimize`, `repair`, and `cleanup` are **direct** (storage-native) CLI commands: they run with direct storage access against a positional `file://`/`s3://` URI or **`--cluster --graph `** (which resolves the graph's storage URI from the served cluster state, so you needn't know the `/graphs/.omni` layout). They never run through a server, and reject `--server` or a remote (`http(s)://`) URI with a declared error. There are no server routes for them by design — to maintain a server-backed graph, run them out-of-band against the graph's storage URI. See the *Command capabilities* section of [cli-reference.md](../cli/reference.md). + +## `optimize` — non-destructive + +- Compacts every node + edge table on `main`, then reindexes them, then **publishes the resulting version to the `__manifest`** so the manifest's recorded version tracks the compacted-and-reindexed state. Reads pin the manifest version, so without this publish the work would be invisible to readers *and* would break the version precondition of the next schema apply / strict update/delete ("stale view … refresh and retry"). The publish advances the graph version (a system-attributed commit) only for tables that actually changed. +- Rewrites small fragments into fewer large ones; old fragments remain reachable via older versions until `cleanup` runs. +- **Reindex (index coverage maintenance).** A scalar/FTS/vector index only covers the fragments it was built over. Rows appended after the index was built (e.g. by `load --mode merge`, whose commit does not rebuild an already-existing index) are scanned unindexed, and compaction itself rewrites fragments out of an index's coverage. `optimize` runs Lance's incremental `optimize_indices` after compaction to fold those fragments back in (a delta merge, not a full retrain), restoring full coverage so equality/range/traversal predicates stay index-accelerated. This is why a table with **no compaction work but stale index coverage still commits** a new version under `optimize`. Run `optimize` on a cadence at least as frequent as your freshness window so recently-loaded rows do not linger in the unindexed flat-scan tail. +- **Create declared-but-missing indexes (the index reconciler).** `@index`/`@key` declares intent; `schema apply` records it but builds nothing, and `load`/`mutate` defer a column that cannot be built yet (a `Vector` column with no trainable vectors). `optimize` materializes any such declared-but-unbuilt index over the compacted layout — so it is the convergence path for an `@index` added after data exists, or a vector index whose embeddings arrived via a later `embed`. A column still not buildable (no vectors yet) is reported on the table's stat as `pending_indexes` (visible in `--json`), not treated as a failure; the next `optimize` retries. So `optimize` is the single operator-facing index reconciler: it compacts, restores coverage, **and** builds declared-but-missing indexes. +- Each table's compact→reindex→publish serializes with concurrent mutations on the same table. A crash mid-operation is recovered automatically on the next open (both compaction and reindex are content-preserving, so roll-forward is always safe). +- **Requires a recovered graph.** `optimize` refuses (errors) when a pending crash-recovery operation is present — operating on an unrecovered graph could publish a partial write that recovery would roll back. Reopen the graph to run recovery, then re-run `optimize`. +- **Uncovered drift is skipped, not interpreted.** If a table's underlying version is ahead of the version recorded in `__manifest` and no crash-recovery record covers that movement, `optimize` reports `skipped: DriftNeedsRepair` with the manifest/head versions and leaves the table untouched. Run `omnigraph repair` to classify and explicitly publish that drift. +- Bounded by `OMNIGRAPH_MAINTENANCE_CONCURRENCY` (default 8). +- Returns per-table stats: `table_key, fragments_removed, fragments_added, committed, skipped, manifest_version, lance_head_version, pending_indexes` (the last lists any declared `@index` column the reconciler could not build this run, with the reason — e.g. a vector column with no trainable vectors yet). +- **Blob tables are skipped.** A table that declares any `Blob` property is not compacted: it is reported with `skipped: BlobColumnsUnsupportedByLance` (and logged) instead of compacted, and the rest of the sweep proceeds normally. **Reads and writes are unaffected** — only compaction is. Consequence: fragment count and deleted-row space on blob tables are not reclaimed; query results are never affected. A skipped blob table is also **not reindexed** in the same sweep (the skip happens before the reindex step), so its index coverage on appended rows is not refreshed by `optimize` today. + +## `repair` — explicit + +- Handles **uncovered manifest/head drift**: a table's underlying version is ahead of the manifest pin and no crash-recovery record explains the movement. +- Preview by default. `omnigraph repair --json ` reports each table's `classification`, `action`, manifest/head versions, underlying operation names, and any classification error. `--confirm` publishes only verified maintenance drift; if any suspicious or unverifiable table is refused, the CLI prints the per-table output and exits non-zero. `--force --confirm` also publishes suspicious or unverifiable drift after operator review. +- Classifies drift by reading the table's transaction history from `manifest_version + 1` through the current head. Only fragment-reservation and rewrite (compaction) operations are verified maintenance. Semantic operations such as append, delete, update, merge, or missing transaction history are not auto-healed. +- Publishes repair by advancing `__manifest` to the existing head; it does **not** rewrite data. If the publish succeeds, normal reads and strict writes use the repaired version. If it fails, no new data-side partial state was created. +- Requires a clean recovery state. A pending crash-recovery operation still belongs to automatic recovery, not manual repair. + +## `cleanup` — destructive + +- Garbage-collects old versions per table. +- Removes versions (and their unique fragments) older than the retention policy. +- Policy options `keep_versions` and `older_than` — at least one is required. +- Returns per-table stats: `table_key, bytes_removed, old_versions_removed, error`. +- **Fault-isolated per table.** A single table's transient failure (version GC or + orphan reclaim) is recorded on that table's stats row (with an `error`) and logged, + and never aborts the healthy tables — cleanup is the convergence + backstop, so it does as much as it can and converges on re-run. The CLI reports + any failed tables; rerun `cleanup` to retry them. +- CLI guards with `--confirm`; without it, prints a preview line. +- **Non-local consent (RFC-011 D9).** Against a non-local target (an `s3://` store/cluster), `cleanup` additionally requires `--yes` on top of `--confirm`: a TTY is prompted, and a non-interactive run (no TTY, or `--json`) refuses rather than destroying. A local (`file://`) target needs only `--confirm`. The same `--yes` gate applies to overwrite `load` and `branch delete`; every maintenance run echoes its resolved target to stderr (suppress with `--quiet`). +- **Recovery floor:** `--keep < 3` may garbage-collect versions that crash recovery needs as a rollback target. Default `--keep 10` is safe. +- **Orphaned-branch reconciliation:** before the version GC, cleanup reclaims any per-table or commit-graph branch absent from the manifest branch list. These orphans arise when a `branch_delete` flips the manifest authority but a downstream best-effort reclaim does not complete (see [branches-commits.md](../branching/index.md)). The reconciler is idempotent (it no-ops once nothing is orphaned), runs regardless of the `keep_versions` / `older_than` values (those gate version GC only), and never reclaims `main` or system-branch forks. Reclaimed forks are logged. + +## Tombstones + +Logical sub-table delete markers in `__manifest` that exclude a sub-table version from snapshot reconstruction. + +## Internal schema migrations + +Version evolutions of the on-disk `__manifest` shape are reconciled automatically on the first write under a new binary. An on-disk stamp records the shape; the binary migrates it forward before reading state, and reads are side-effect-free. No operator action is required for in-place upgrades. See [storage.md → Internal schema versioning](../concepts/storage.md) for the full mechanism. + +A binary opening a manifest stamped at a version *higher* than it knows about refuses to publish with a clear "upgrade omnigraph first" error — old binaries cannot clobber a newer schema. diff --git a/docs/user/policy.md b/docs/user/operations/policy.md similarity index 52% rename from docs/user/policy.md rename to docs/user/operations/policy.md index 91684d8..c6096d0 100644 --- a/docs/user/policy.md +++ b/docs/user/operations/policy.md @@ -13,14 +13,14 @@ Per-graph actions (bind to `Omnigraph::Graph::""`): 5. `branch_create` 6. `branch_delete` 7. `branch_merge` -8. `admin` — reserved for policy-management surfaces (hot reload, audit log, approvals). No call site today; see MR-724 for the reservation rationale. +8. `admin` — reserved for policy-management surfaces (hot reload, audit log, approvals). No call site today. 9. `invoke_query` — gates invoking a server-side stored query (the `queries:` registry). Graph-scoped (like `admin`) — per-branch access is enforced by the inner `read` / `change` gate, so a rule that sets `branch_scope` on `invoke_query` is rejected. Coarse in this release: an `invoke_query` allow rule permits any stored query on the graph; a future, additive refinement adds an optional per-query-name scope without changing rules written against the coarse action. Enforced at `POST /queries/{name}` (see [server](server.md)). A stored *mutation* is double-gated: `invoke_query` to reach the tool, plus `change` for the write itself (the engine `_as` writers still enforce per the query body). Server-scoped action (v0.6.0+; binds to `Omnigraph::Server::"root"`): 10. `graph_list` — `GET /graphs` registry enumeration (multi-graph mode) -Server-scoped actions cannot use `branch_scope` or `target_branch_scope` — they operate on the registry, not on a graph's branches. A rule cannot mix server-scoped and per-graph actions; split into separate rules. (Runtime `graph_create` / `graph_delete` are reserved but not shipped in v0.6.0; operators add/remove graphs by editing `omnigraph.yaml` and restarting.) +Server-scoped actions cannot use `branch_scope` or `target_branch_scope` — they operate on the registry, not on a graph's branches. A rule cannot mix server-scoped and per-graph actions; split into separate rules. (Runtime `graph_create` / `graph_delete` over HTTP are reserved but not shipped; operators add/remove graphs by editing the cluster's `cluster.yaml`, running `omnigraph cluster apply`, and restarting the server.) ## Scope kinds @@ -28,38 +28,34 @@ Server-scoped actions cannot use `branch_scope` or `target_branch_scope` — the - `target_branch_scope` — applied to destination (`schema_apply`, branch ops, run ops) - `protected_branches` — named list with special rules; rule scopes are `any | protected | unprotected` -## Per-graph vs. server-level policy (multi-graph mode) +## Per-graph vs. server-level policy -In multi mode (`omnigraph.yaml` with a non-empty `graphs:` map), policy files attach at two levels: +A server boots from a cluster (`--cluster `), and the cluster's +`cluster.yaml` declares its policy bundles in a `policies:` section. Each bundle +names the scopes it `applies_to`: a graph id (per-graph rules — `read`, `change`, +`branch_*`, `schema_apply`) or the literal `cluster` (server-level rules — +`graph_list`). ```yaml -server: - policy: - file: server-policy.yaml # server-level: graph_list - -graphs: +# cluster.yaml +policies: + base: + file: base.policy.yaml + applies_to: [cluster, knowledge] # cluster-level + the `knowledge` graph alpha: - uri: s3://tenant-bucket/alpha - policy: - file: policies/alpha.yaml # per-graph: read, change, branch_*, schema_apply - beta: - uri: s3://tenant-bucket/beta - # no per-graph policy → no engine-layer Cedar enforcement on beta + file: policies/alpha.yaml + applies_to: [alpha] # per-graph: alpha only ``` -**Config follows graph identity, not server mode.** A graph served by **name** -(`--target ` or `server.graph`) uses its own `graphs..policy.file`, -exactly as in multi-graph mode. Top-level `policy.file` applies only to an -**anonymous** graph — one served by a bare `` with no `graphs:` entry. -Serving a **named** graph (single- or multi-graph mode) while top-level -`policy.file` (or `queries:`) is populated **refuses boot**, naming the block, -since the top-level value would otherwise be silently shadowed by the per-graph -block. Move per-graph rules to `graphs..policy.file` and `graph_list` -rules to `server.policy.file`. +A graph with no bundle bound to it has no engine-layer Cedar enforcement. Each +graph's HTTP request flows through its bound bundle; the management endpoint +(`GET /graphs`) flows through the `cluster`-scoped bundle. When no bundle binds +`cluster`, `GET /graphs` is denied in every runtime state, including +`--unauthenticated`; with bearer tokens configured it returns 403 after admission +control because `graph_list` is not a `read`-equivalent action. The operator must +bind a `cluster`-scoped bundle granting `graph_list` to expose `/graphs`. -Each graph's HTTP request flows through its own per-graph policy. The management endpoint (`GET /graphs`) flows through the server-level policy. When `server.policy.file` is unset, `GET /graphs` is denied in every runtime state, including `--unauthenticated`; with bearer tokens configured, it returns 403 after admission control because `graph_list` is not a `read`-equivalent action. The operator must explicitly authorize via `server-policy.yaml` to expose `/graphs`. - -Example server-level policy: +Example `cluster`-scoped bundle: ```yaml version: 1 @@ -72,40 +68,28 @@ rules: actions: [graph_list] ``` -## Configuration +Each per-graph rule may use at most one of `branch_scope` or +`target_branch_scope`. Server-scoped rules (`graph_list`) take neither — they +have no branch context. -`omnigraph.yaml`: +## Actor for direct-engine writes -```yaml -policy: - file: policy.yaml # Cedar rules + groups - tests: policy.tests.yaml # declarative test cases - -cli: - actor: act-andrew # default actor for CLI direct-engine writes -``` - -Each per-graph rule may use at most one of `branch_scope` or `target_branch_scope`. Server-scoped rules (`graph_list`) take neither — they have no branch context. - -`cli.actor` is the default actor identity for CLI direct-engine writes -when `policy.file` is configured. Override per-invocation with `--as -` (top-level flag) — `--as` wins, otherwise `cli.actor` is used, -otherwise no actor. With policy configured and neither set, the -engine-layer footgun guard intentionally denies the write (silent bypass -via "I forgot the actor" is exactly what the guard prevents). Remote -HTTP writes ignore both — they resolve their actor server-side from the -bearer token. +The default actor identity for CLI direct-engine (`--store`) writes is +`operator.actor` in `~/.omnigraph/config.yaml`. Override per-invocation with +`--as ` — `--as` wins, otherwise `operator.actor`, otherwise no actor. +Remote HTTP writes ignore both — they resolve their actor server-side from the +bearer token. (Direct-store access carries no Cedar policy under RFC-011; policy +lives in the cluster/server.) ## CLI -Policy tooling resolves its graph like server single-mode policy: `cli.graph` -wins, otherwise `server.graph` is used, otherwise the top-level `policy.file` -is validated/tested/explained as the anonymous policy. +Policy tooling reads a cluster's applied policy bundles: pass `--cluster `, +and `--graph ` to pick a graph's bundle when several apply. - `omnigraph policy validate` — parse + count actors, exit 1 on parse error. -- `omnigraph policy test` — run cases in `policy.tests.yaml`, exit 1 on any expectation mismatch. +- `omnigraph policy test --tests ` — run the declarative cases in `` against the selected bundle, exit 1 on any expectation mismatch. - `omnigraph policy explain --actor … --action … [--branch …] [--target-branch …]` — show decision and matched rule. -- `omnigraph --as ` — set the actor for the duration of one invocation. Effective for `change`, `load` (and its deprecated `ingest` alias), `branch create|delete|merge`, and `schema apply` against local URIs. No-op against remote HTTP URIs (actor is bearer-token-resolved server-side). +- `omnigraph --as ` — set the actor for the duration of one invocation. Effective for `change`, `load` (and its deprecated `ingest` alias), `branch create|delete|merge`, and `schema apply` against a direct (`--store`) graph. **Rejected** on a served write (`--server`): the actor is bearer-token-resolved server-side, so `--as` can't set it there. ## Enforcement @@ -113,42 +97,38 @@ Policy is a property of the **engine**, not the transport. Every mutating write — `mutate_as`, `load_as` (the deprecated `ingest_as` shims route through it), `apply_schema_as`, `branch_create_as`, `branch_create_from_as`, `branch_delete_as`, -`branch_merge_as` — calls `Omnigraph::enforce(action, scope, actor)` at -the head of the method. The gate fires identically whether the call +`branch_merge_as` — consults the policy gate at the head of the method. +The gate fires identically whether the call originates from the HTTP server, the CLI, or an embedded SDK consumer. -When no `PolicyChecker` is installed (the dev/embedded default) the gate +When no policy is installed (the dev/embedded default) the gate is a strict no-op; when one is installed and the call site forgets to thread an actor through, the gate fails closed rather than silently bypassing. -## Server runtime states (MR-723) +## Server runtime states The HTTP server classifies its startup configuration into one of three states based on whether bearer tokens are configured and whether a policy file is set. The state determines what happens to a request that -reaches `authorize_request()` without a matching policy permit. +reaches the authorization gate without a matching policy permit. | State | Tokens | Policy file | Behavior | |---|---|---|---| | **Open** | no | no | Every request is permitted. Refuses to start unless `--unauthenticated` or `OMNIGRAPH_UNAUTHENTICATED=1` is set — the operator must explicitly opt in. | | **DefaultDeny** | yes | no | Every authenticated request for an action other than `read` is rejected with HTTP 403. Closes the "tokens but forgot the policy file" trap — an operator who sets up auth and forgot to point at a policy file used to ship the illusion of protection. | -| **PolicyEnabled** | yes | yes | Authenticated requests that reach a configured policy engine are evaluated by Cedar. Server-scoped actions still require `server.policy.file`. | +| **PolicyEnabled** | yes | yes | Authenticated requests that reach a configured policy engine are evaluated by Cedar. Server-scoped actions still require a `cluster`-scoped policy bundle. | -The classifier is `classify_server_runtime_state` in -`crates/omnigraph-server/src/lib.rs`; it returns `Err` for the "no -tokens, no policy, no flag" cell and for "policy file, no tokens" so the -server refuses to start instead of silently shipping an open instance or -a policy-protected server that can only 401. Tests pin every cell of the -matrix and the State-2 deny path. +The server refuses to start for the "no tokens, no policy, no flag" cell +and for "policy file, no tokens" — instead of silently shipping an open +instance or a policy-protected server that can only 401. -Server-side, `authorize_request()` still runs at the HTTP boundary — +Server-side, request authorization still runs at the HTTP boundary — that's where actor identity is resolved from the bearer token and where admission control / per-actor rate limits live. Engine-layer enforcement is the **defense in depth** layer: it catches CLI direct-engine writes, embedded SDK consumers, and any future transport that hasn't (or won't) -re-implement HTTP's authorize_request. Both layers consult the same -Cedar policy via the same `PolicyChecker` trait, so decisions cannot -disagree. +re-implement the HTTP boundary's authorization. Both layers consult the same +Cedar policy, so decisions cannot disagree. ## Coarse vs. fine enforcement @@ -157,19 +137,19 @@ responsibilities: | Layer | Question it answers | Where it fires | |---|---|---| -| **Engine-layer (coarse)** | Can this actor invoke this action against this branch / branch-transition? | `Omnigraph::enforce(action, scope, actor)` at the head of every `_as` writer; one Cedar decision per call. | -| **Query-layer (fine)** | For the rows / types this action actually touches, which can the actor see or modify? | Per-row predicates pushed into DataFusion at plan time. **Not yet implemented — see MR-725.** | +| **Engine-layer (coarse)** | Can this actor invoke this action against this branch / branch-transition? | The policy gate at the head of every `_as` writer; one Cedar decision per call. | +| **Query-layer (fine)** | For the rows / types this action actually touches, which can the actor see or modify? | Per-row predicates pushed into the query plan. **Not yet implemented.** | -The engine-layer gate keeps `ResourceScope` deliberately at branch -granularity (`Graph`, `Branch`, `TargetBranch`, `BranchTransition`). +The engine-layer gate keeps its resource scope deliberately at branch +granularity (graph, branch, target branch, branch transition). Per-type and per-row authority is the query-layer's job; conflating them -in `ResourceScope` would create two places per-type policy could be +in the engine-layer scope would create two places per-type policy could be evaluated and a drift surface between them. ## Actor identity (signed-claim-only) The actor identity used for every policy decision comes from the matched bearer token — never from a client-supplied request header, query parameter, or body field. The server resolves the token at the auth middleware boundary, looks up the actor it was minted for, and overwrites whatever the handler may have placed in the policy request. Clients cannot set `actor_id` directly. -This is intentional. Trusting client-supplied identity for authorization is "asking the attacker if they're an admin" — Supabase's RLS history names the same footgun. The chokepoint lives in `authorize_request` in `crates/omnigraph-server/src/lib.rs` and is named in `docs/dev/invariants.md` Hard Invariant 11. A regression test asserts the contract: a request with `Authorization: Bearer ` plus `X-Actor-Id: actor-B` always evaluates as actor A, never as actor B. +This is intentional. Trusting client-supplied identity for authorization is "asking the attacker if they're an admin" — Supabase's RLS history names the same footgun. The chokepoint lives at the server's auth boundary: a request with `Authorization: Bearer ` plus `X-Actor-Id: actor-B` always evaluates as actor A, never as actor B. If you find yourself wanting to let clients override `actor_id` for impersonation, delegation, or service-account flows — that's a feature, but it needs explicit design (e.g., signed delegation claims, an `On-Behalf-Of` audit trail). It is not a convenience knob. diff --git a/docs/user/server.md b/docs/user/operations/server.md similarity index 53% rename from docs/user/server.md rename to docs/user/operations/server.md index 391b7ae..bd14e1e 100644 --- a/docs/user/server.md +++ b/docs/user/operations/server.md @@ -1,74 +1,67 @@ # HTTP Server (`omnigraph-server`) -Axum 0.8 + tokio + utoipa-generated OpenAPI. **Two modes** (v0.6.0+): single-graph (legacy) and multi-graph (MR-668), with **two boot sources** for multi mode: `omnigraph.yaml` or — exclusively — a cluster directory (`--cluster`, RFC-005). Mode is inferred from CLI args + config shape. +Axum 0.8 + tokio + utoipa-generated OpenAPI. **Cluster-only boot** (RFC-011): the server always boots from a cluster (`--cluster `) and serves N graphs (N ≥ 1) under cluster routes. There is no longer a single-graph flat-route mode, no positional `` boot, no `--target`, and no `omnigraph.yaml`-`graphs:`-map boot. All HTTP is nested under `/graphs/{graph_id}/...`; `/healthz` and the management `/graphs` enumeration stay flat. -## Modes +## Boot -### Single-graph mode (legacy) +### Cluster boot (the only boot) -`omnigraph-server ` or `omnigraph-server --target --config omnigraph.yaml`. Routes are flat — `/snapshot`, `/read`, `/branches`, etc. +```bash +omnigraph-server --cluster --bind 0.0.0.0:8080 +``` -**Config follows graph identity.** A bare `` is an *anonymous* graph and uses the **top-level** `policy.file` / `queries:`. A graph chosen by **name** (`--target` / `server.graph`) uses its own `graphs..{policy.file, queries}` — the same block multi-graph mode uses. ⚠️ *Changed from v0.6.0, which always used top-level config in single mode: a named-graph config that puts `policy`/`queries` at top-level now **refuses boot** and points you at `graphs..…` (move the block there). Bare-`` single mode is unchanged.* +`omnigraph-server --cluster ` boots from the cluster catalog's +**applied revision**. The server resolves that revision into per-graph +startup configs (id, URI, optional per-graph policy, stored-query +registry) plus an optional server-level policy, then opens every +configured graph in parallel at startup (bounded concurrency = 4, +fail-fast on the first open error). Routing is always multi-graph — +requests to bare flat protected paths (`/read`, `/snapshot`, …) return +404; the served surface is `/graphs/{graph_id}/...`. See +[cluster-config.md](../clusters/config.md#serving-from-the-cluster-the-mode-switch) +for what is read and the fail-fast readiness rules. -### Multi-graph mode (v0.6.0+) - -`omnigraph-server --config omnigraph.yaml` with a non-empty `graphs:` map and **no** single-mode selector (no `server.graph`, no ``, no `--target`). The server opens every configured graph in parallel at startup (bounded concurrency = 4, fail-fast on the first open error). Routes are nested under `/graphs/{graph_id}/...`. Bare flat paths return 404 in multi mode. - -### Cluster-booted multi mode (Phase 5) - -`omnigraph-server --cluster ` boots from the cluster catalog's **applied -revision** (`state.json` + content-addressed blobs) instead of -`omnigraph.yaml` — an exclusive boot source: combining it with ``, -`--target`, or `--config` is a startup error, and `omnigraph.yaml` is never -read in this mode. Always multi-graph routing. See -[cluster-config.md](cluster-config.md#serving-from-the-cluster-the-mode-switch) -for what is read and the fail-fast readiness rules. `--bind`, -`--unauthenticated`, and the bearer-token env vars work identically. - -Mode inference: - -0. CLI `--cluster ` → **multi, cluster-booted** (exclusive; a scheme-qualified argument reads the ledger straight from the storage root, no local config) -1. CLI positional `` → single -2. CLI `--target ` → single -3. `server.graph` in config → single -4. `--config` + non-empty `graphs:` + no single-mode selector → **multi** -5. otherwise → error with migration hint +A scheme-qualified argument (`s3://…`) reads the ledger straight from the +storage root, with no local config directory. `--bind`, +`--unauthenticated`, and the bearer-token env vars all apply. ### Stored-query validation at startup -If a graph declares a `queries:` registry (see [cli-reference](cli-reference.md)), the server **loads and type-checks every stored query against that graph's live schema at startup** and **refuses to boot** if any query references a type or property the schema lacks — the same fail-loud posture as a malformed policy file, so schema drift surfaces at the deploy boundary rather than at invocation. Two MCP-exposed queries claiming the same tool name is likewise a boot error. Non-blocking advisories (e.g. an MCP-exposed query with a vector parameter an agent cannot supply) are logged. Validate offline before deploying with `omnigraph queries validate`. Discover the exposed queries as a typed tool catalog with `GET /queries`, and invoke one over HTTP with `POST /queries/{name}` (both below). +If a graph declares a `queries:` registry (see [cli-reference](../cli/reference.md)), the server **loads and type-checks every stored query against that graph's live schema at startup** and **refuses to boot** if any query references a type or property the schema lacks — the same fail-loud posture as a malformed policy file, so schema drift surfaces at the deploy boundary rather than at invocation. Two MCP-exposed queries claiming the same tool name is likewise a boot error. Non-blocking advisories (e.g. an MCP-exposed query with a vector parameter an agent cannot supply) are logged. Validate offline before deploying with `omnigraph queries validate`. Discover the exposed queries as a typed tool catalog with `GET /queries`, and invoke one over HTTP with `POST /queries/{name}` (both below). ## Endpoint inventory -Per-graph endpoints — same body shape across modes; URLs differ: +Per-graph endpoints — all nested under `/graphs/{id}/...`. `{id}` is the +graph id from the cluster's applied revision: -| Method | Single-mode path | Multi-mode path | Auth | Action | Handler | -|---|---|---|---|---|---| -| GET | `/healthz` | `/healthz` | none | — | `server_health` | -| GET | `/openapi.json` | `/openapi.json` | none | — | `server_openapi` (strips security if auth disabled; in multi mode emits cluster paths with `cluster_` operation-id prefix) | -| GET | `/snapshot?branch=` | `/graphs/{id}/snapshot?branch=` | bearer + `read` | snapshot of branch | `server_snapshot` | -| POST | `/query` | `/graphs/{id}/query` | bearer + `read` | inline read query (canonical; clean field names `query`/`name`; mutations → 400) | `server_query` | -| POST | `/read` | `/graphs/{id}/read` | bearer + `read` | **deprecated** alias of `/query` (legacy field names `query_source`/`query_name`, byte-stable response; carries `Deprecation: true` + `Link: ; rel="successor-version"`) | `server_read` | -| POST | `/export` | `/graphs/{id}/export` | bearer + `export` | NDJSON stream | `server_export` | -| POST | `/mutate` | `/graphs/{id}/mutate` | bearer + `change` | mutation (canonical; `query`/`name`; accepts legacy `query_source`/`query_name` as serde aliases) | `server_mutate` | -| POST | `/change` | `/graphs/{id}/change` | bearer + `change` | **deprecated** alias of `/mutate` (carries `Deprecation: true` + `Link: ; rel="successor-version"`) | `server_change` | -| GET | `/queries` | `/graphs/{id}/queries` | bearer + `read` | list the `mcp.expose` stored queries as a typed tool catalog | `server_list_queries` | -| POST | `/queries/{name}` | `/graphs/{id}/queries/{name}` | bearer + `invoke_query` (+ `change` for a stored mutation) | invoke a named query from the `queries:` registry; deny == 404 | `server_invoke_query` | -| GET | `/schema` | `/graphs/{id}/schema` | bearer + `read` | get current `.pg` source | `server_schema_get` | -| POST | `/schema/apply` | `/graphs/{id}/schema/apply` | bearer + `schema_apply` (target=`main`) | migrate | `server_schema_apply` | -| POST | `/ingest` | `/graphs/{id}/ingest` | bearer + `branch_create` (only when `from` is set and the branch is created) + `change` | bulk load; branch creation is opt-in via `from` — without it a missing `branch` is a 404, never an implicit fork | `server_ingest` (32 MB body limit) | -| GET | `/branches` | `/graphs/{id}/branches` | bearer + `read` | list branches | `server_branch_list` | -| POST | `/branches` | `/graphs/{id}/branches` | bearer + `branch_create` | create | `server_branch_create` | -| DELETE | `/branches/{branch}` | `/graphs/{id}/branches/{branch}` | bearer + `branch_delete` | delete | `server_branch_delete` | -| POST | `/branches/merge` | `/graphs/{id}/branches/merge` | bearer + `branch_merge` | merge `source → target` | `server_branch_merge` | -| GET | `/commits?branch=` | `/graphs/{id}/commits?branch=` | bearer + `read` | list | `server_commit_list` | -| GET | `/commits/{commit_id}` | `/graphs/{id}/commits/{commit_id}` | bearer + `read` | show | `server_commit_show` | +| Method | Path | Auth | Action | +|---|---|---|---| +| GET | `/healthz` | none | — | +| GET | `/openapi.json` | none | — (strips security if auth disabled; emits the nested cluster paths with `cluster_` operation-id prefix) | +| GET | `/graphs/{id}/snapshot?branch=` | bearer + `read` | snapshot of branch | +| POST | `/graphs/{id}/query` | bearer + `read` | inline read query (canonical; clean field names `query`/`name`; mutations → 400) | +| POST | `/graphs/{id}/read` | bearer + `read` | **deprecated** alias of `/query` (legacy field names `query_source`/`query_name`, byte-stable response; carries `Deprecation: true` + `Link: ; rel="successor-version"`) | +| POST | `/graphs/{id}/export` | bearer + `export` | NDJSON stream | +| POST | `/graphs/{id}/mutate` | bearer + `change` | mutation (canonical; `query`/`name`; accepts legacy `query_source`/`query_name` as serde aliases) | +| POST | `/graphs/{id}/change` | bearer + `change` | **deprecated** alias of `/mutate` (carries `Deprecation: true` + `Link: ; rel="successor-version"`) | +| GET | `/graphs/{id}/queries` | bearer + `read` | list the `mcp.expose` stored queries as a typed tool catalog | +| POST | `/graphs/{id}/queries/{name}` | bearer + `invoke_query` (+ `change` for a stored mutation) | invoke a named query from the `queries:` registry; deny == 404 | +| GET | `/graphs/{id}/schema` | bearer + `read` | get current `.pg` source | +| POST | `/graphs/{id}/schema/apply` | bearer + `schema_apply` (target=`main`) | disabled for cluster-backed serving; returns 409 and points operators at `omnigraph cluster apply` + restart | +| POST | `/graphs/{id}/load` | bearer + `branch_create` (only when `from` is set and the branch is created) + `change` | bulk load (canonical); branch creation is opt-in via `from` — without it a missing `branch` is a 404, never an implicit fork (32 MB body limit) | +| POST | `/graphs/{id}/ingest` | bearer + `branch_create` (only when `from` is set and the branch is created) + `change` | **deprecated** alias of `/load` (carries `Deprecation: true` + `Link: ; rel="successor-version"`) (32 MB body limit) | +| GET | `/graphs/{id}/branches` | bearer + `read` | list branches | +| POST | `/graphs/{id}/branches` | bearer + `branch_create` | create | +| DELETE | `/graphs/{id}/branches/{branch}` | bearer + `branch_delete` | delete | +| POST | `/graphs/{id}/branches/merge` | bearer + `branch_merge` | merge `source → target` | +| GET | `/graphs/{id}/commits?branch=` | bearer + `read` | list | +| GET | `/graphs/{id}/commits/{commit_id}` | bearer + `read` | show | -Server-level management endpoints (v0.6.0+): +Server-level management endpoints: -| Method | Path | Auth | Action | Handler | -|---|---|---|---|---| -| GET | `/graphs` | bearer + `graph_list` on `Server::"root"` | list registered graphs | `server_graphs_list` (405 in single mode) | +| Method | Path | Auth | Action | +|---|---|---|---| +| GET | `/graphs` | bearer + `graph_list` on `Server::"root"` | list registered graphs | ### Stored-query catalog (`GET /queries`) @@ -87,17 +80,17 @@ Invoke a curated, server-side stored query by **name** — the source comes from - **Requires an explicit policy grant when auth is on.** In default-deny mode (bearer tokens but no `policy.file`), only `read` is permitted, so *every* `/queries/{name}` call returns `404` until an `invoke_query` rule is configured. - A stored mutation cannot target a `snapshot` (`400`); a parameter type error is a structured `400` naming the parameter. -## Adding and removing graphs (multi mode) +## Adding and removing graphs -Runtime add/remove via API is **not** exposed in v0.6.0 — neither -`POST /graphs` nor `DELETE /graphs/{id}` is implemented. Operators add -or remove graphs by stopping the server, editing the `graphs:` map in -`omnigraph.yaml`, then restarting. The server treats `omnigraph.yaml` -as operator-owned configuration and never writes it. +Runtime add/remove via API is **not** exposed — neither `POST /graphs` +nor `DELETE /graphs/{id}` is implemented. Operators add or remove graphs +by running `cluster apply` against the cluster (which publishes a new +applied revision) and restarting the server so it boots from the new +revision. The server treats the cluster source as operator-owned and +never writes it. -A future release may introduce a managed registry (Lance-backed, -catalog-style: reserve → init → publish with recovery sidecars) and -re-expose runtime mutation on top of it. +A future release may introduce a managed registry and re-expose runtime +mutation on top of it. ## Inline read queries (`POST /query`) @@ -138,8 +131,8 @@ channels: - **Response headers (RFC 9745)**: every response carries `Deprecation: true`. - **Response headers (RFC 8288)**: every response carries a `Link` header pointing at the canonical successor: - `Link: ; rel="successor-version"` for `/read`, and - `Link: ; rel="successor-version"` for `/change`. SDKs and HTTP + `Link: ; rel="successor-version"` for `/read`, and + `Link: ; rel="successor-version"` for `/change`. SDKs and HTTP proxies can pick the successor up automatically. Migration is purely cosmetic on the client side — swap the URL path, leave @@ -153,7 +146,7 @@ Only `/export` streams (`application/x-ndjson`, MPSC channel + `Body::from_strea Uniform `ErrorOutput { error, code?, merge_conflicts[], manifest_conflict? }` with `code ∈ unauthorized | forbidden | bad_request | not_found | conflict | too_many_requests | internal`. Merge conflicts attach structured `MergeConflictOutput { table_key, row_id?, kind, message }`. -`manifest_conflict` is set on **publisher CAS rejections** (HTTP 409): the +`manifest_conflict` is set on **concurrent-write rejections** (HTTP 409): the caller's pre-write view of one table's manifest version was stale. `ManifestConflictOutput { table_key, expected, actual }` tells the client which table to refresh and retry. This is the conflict shape produced by @@ -168,8 +161,8 @@ Disjoint `(table, branch)` writes from different actors now run concurrently, guarded only by the engine's per-(table, branch) write queue. To keep one heavy actor from exhausting shared capacity (Lance I/O, manifest -churn, network), the server gates mutating handlers through a -`WorkloadController` configured per-process from environment variables: +churn, network), the server gates mutating handlers through per-process +admission limits configured from environment variables: | Env var | Default | Purpose | |---|---|---| @@ -198,7 +191,7 @@ admission-gated. ## Auth model (`bearer + SHA-256`) - Tokens are SHA-256 hashed on startup; plaintext is never persisted in memory. -- Constant-time comparison via `subtle::ConstantTimeEq`. +- Constant-time comparison. - Three sources, in precedence: 1. `OMNIGRAPH_SERVER_BEARER_TOKENS_AWS_SECRET` — AWS Secrets Manager (build with `--features aws`) 2. `OMNIGRAPH_SERVER_BEARER_TOKENS_FILE` or `OMNIGRAPH_SERVER_BEARER_TOKENS_JSON` — JSON `{actor_id: token, …}` @@ -208,7 +201,7 @@ admission-gated. policy file without tokens is also rejected at startup. In open mode `/openapi.json` strips the security scheme. -See [deployment.md](deployment.md) for token-source operational details. +See [deployment.md](../deployment.md) for token-source operational details. ## Tracing & observability @@ -226,4 +219,4 @@ See [deployment.md](deployment.md) for token-source operational details. admission control" above). No global rate limiter is configured; add `tower_http::limit` if a graph-wide cap is needed. - Pagination — none (commits/branches return everything; export streams). -- Runtime graph add/remove — edit `omnigraph.yaml` and restart. +- Runtime graph add/remove — run `cluster apply` and restart. diff --git a/docs/user/queries/index.md b/docs/user/queries/index.md new file mode 100644 index 0000000..c8a70c5 --- /dev/null +++ b/docs/user/queries/index.md @@ -0,0 +1,65 @@ +# Query Language (`.gq`) + +## Query declarations + +``` +query ($p1: T1, $p2: T2?, …) + @description("…") @instruction("…") { + … +} +``` + +Two body shapes: + +- **Read**: `match { … } return { … } [order { … }] [limit N]` — covered on this page. +- **Mutation**: one or more of `insert | update | delete` statements — see [mutations](../mutations/index.md). + +Multi-modal search functions (`nearest`, `bm25`, `rrf`, …) used inside `match`, +`return`, and `order` are documented on the [search](../search/index.md) page. + +Param types reuse all schema scalars; trailing `?` makes a param optional. The compiler reserves `$__nanograph_now` for `now()`. + +## MATCH clauses + +- **Binding**: `$x: NodeType { prop: , … }` +- **Traversal**: `$src EDGE_NAME { min, max? } $dst` — variable-length paths via hop bounds; default 1..1 if bounds omitted. +- **Filter**: ` ` with operators `>=`, `<=`, `!=`, `>`, `<`, `=`, and string `contains`. +- **Negation**: `not { clause+ }` — desugars to anti-join over the inner pipeline. + +## RETURN clause + +`return { [as ], … }` with expressions: + +- Variable / property access: `$x`, `$x.prop` +- Literals: string, int, float, bool, list +- `now()` +- Aggregates: `count`, `sum`, `avg`, `min`, `max` +- [Search functions](../search/index.md) (so you can return a score column) +- `AliasRef` — re-use a previous projection alias + +## ORDER & LIMIT + +- `order { [asc|desc], … }` — supports plain expressions and `nearest(...)`. +- `limit ` — required when there is a `nearest(...)` ordering. +- **Total, deterministic order.** Rows with equal user-sort keys are broken by the bound entities' key columns (`.id`, ascending) appended as a final tie-break, so the result is a *total* order — reproducible across runs, and `order … limit N` returns a deterministic top-N even when ties straddle the cutoff. (Aggregate results have no entity-key columns; their group rows are already distinct on the projected group keys.) +- **NULL placement** is *nulls-first ascending, nulls-last descending* (i.e. `nulls_first = !descending`): a NULL sorts as if smaller than any value. + +Write statements (`insert` / `update` / `delete`) are documented on the +[mutations](../mutations/index.md) page. + +## Traversal execution + +Variable-length traversals (`Expand`) are executed one of two ways, chosen per-expand by a cost model over cheap manifest counts (frontier size, edge count, source-vertex count, hops) plus index coverage: selective traversals (small frontier relative to the source set) resolve neighbors from the persisted `src`/`dst` BTREE (one indexed scan per hop); dense / deep / large-frontier traversals — or those whose BTREE coverage is degraded so a full scan would be paid per hop — use an in-memory CSR adjacency index. Both produce identical results. The `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER` / `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS` ceilings bound the *initial dispatch* frontier/hops (beyond them CSR is always used); the cost model estimates total indexed work as ~`hops × frontier × fanout` and prices dense fan-out toward CSR — they are not a hard per-hop bound. `OMNIGRAPH_TRAVERSAL_MODE=indexed|csr` forces a mode (see [constants](../reference/constants.md)). + +## Linting & validation + +Codes seen so far: + +- **Q000** (Error): parse error +- **L201** (Warning): nullable property never set by any UPDATE — "{type}.{prop} exists in schema but no update query sets it" +- (Warning): mutation declares no params — hardcoded mutations are easy to miss +- Plus all type errors from type checking (undefined types, mismatched operators, undefined edges, etc.) + +Lint output reports an overall status, per-query results (name, kind, status, any error and warnings), and structured findings (severity, code, message, and the type/property/query they apply to). + +CLI exits non-zero only on `status = Error`. diff --git a/docs/user/query-language.md b/docs/user/query-language.md deleted file mode 100644 index bcab67c..0000000 --- a/docs/user/query-language.md +++ /dev/null @@ -1,113 +0,0 @@ -# Query Language (`.gq`) - -Pest grammar at `crates/omnigraph-compiler/src/query/query.pest`. AST in `query/ast.rs`. Type checker in `query/typecheck.rs`. Lowering in `ir/lower.rs`. - -## Query declarations - -``` -query ($p1: T1, $p2: T2?, …) - @description("…") @instruction("…") { - … -} -``` - -Two body shapes: - -- **Read**: `match { … } return { … } [order { … }] [limit N]` -- **Mutation**: one or more of `insert | update | delete` statements - -Param types reuse all schema scalars; trailing `?` makes a param optional. The compiler reserves `$__nanograph_now` for `now()`. - -## MATCH clauses - -- **Binding**: `$x: NodeType { prop: , … }` -- **Traversal**: `$src EDGE_NAME { min, max? } $dst` — variable-length paths via hop bounds; default 1..1 if bounds omitted. -- **Filter**: ` ` with operators `>=`, `<=`, `!=`, `>`, `<`, `=`, and string `contains`. -- **Negation**: `not { clause+ }` — desugars to anti-join over the inner pipeline. - -## Search clauses (multi-modal) - -Used inside MATCH or as expressions inside RETURN/ORDER: - -| Function | Purpose | Underlying Lance facility | -|---|---|---| -| `nearest($x.vec, $q)` | k-NN vector search (cosine) | Lance vector index (IVF / HNSW) | -| `search(field, q)` | Generic FTS | Inverted index | -| `fuzzy(field, q [, max_edits])` | Levenshtein-tolerant text search | Inverted index | -| `match_text(field, q)` | Pattern match | Inverted index | -| `bm25(field, q)` | BM25 scoring | Inverted index | -| `rrf(rank_a, rank_b [, k])` | Reciprocal Rank Fusion of two rankings (default k=60) | OmniGraph fuses scored rankings | - -`nearest()` requires a `LIMIT`; the compiler resolves the query vector via the param map (or via the runtime embedding client when bound to a text input). - -## RETURN clause - -`return { [as ], … }` with expressions: - -- Variable / property access: `$x`, `$x.prop` -- Literals: string, int, float, bool, list -- `now()` -- Aggregates: `count`, `sum`, `avg`, `min`, `max` -- All search functions above (so you can return a score column) -- `AliasRef` — re-use a previous projection alias - -## ORDER & LIMIT - -- `order { [asc|desc], … }` — supports plain expressions and `nearest(...)`. -- `limit ` — required when there is a `nearest(...)` ordering. -- **Total, deterministic order.** Rows with equal user-sort keys are broken by the bound entities' key columns (`.id`, ascending) appended as a final tie-break, so the result is a *total* order — reproducible across runs, and `order … limit N` returns a deterministic top-N even when ties straddle the cutoff. (Aggregate results have no entity-key columns; their group rows are already distinct on the projected group keys.) -- **NULL placement** is *nulls-first ascending, nulls-last descending* (i.e. `nulls_first = !descending`): a NULL sorts as if smaller than any value. - -## Mutation statements - -- `insert { prop: , … }` -- `update set { prop: , … } where ` -- `delete where ` - -`` is a literal, `$param`, or `now()`. Multi-statement mutations execute atomically (added in v0.2.0). - -### D₂ — mixed insert/update + delete is rejected at parse time - -A single mutation query must be **either insert/update-only or delete-only**. Mixed → rejected before any I/O with the message: - -> `mutation '' on the same query mixes inserts/updates and deletes; split into separate mutations: (1) inserts and updates, then (2) deletes. This restriction lifts when Lance exposes a two-phase delete API (tracked: MR-793 / Lance-upstream).` - -Reason: under the staged-write rewire (MR-794), inserts and updates accumulate in memory and commit at end-of-query, while deletes still inline-commit (Lance v6.0.1 has no public two-phase delete). Mixing creates ordering hazards (same-row insert→delete becomes a no-op because the staged insert isn't visible to delete; cascading deletes of just-inserted edges break referential integrity by silent design). Until the MR-A Lance v7 bump migrates `delete_where` to staged (`DeleteBuilder::execute_uncommitted` first ships in `v7.0.0-beta.10`), the parse-time rejection keeps both paths atomic and correct. See [docs/dev/writes.md](../dev/writes.md), [docs/dev/lance.md](../dev/lance.md), and [docs/dev/invariants.md](../dev/invariants.md). - -## IR (Intermediate Representation) - -`QueryIR { name, params, pipeline: Vec, return_exprs, order_by, limit }` - -Pipeline operations: - -- `NodeScan { variable, type_name, filters }` -- `Expand { src_var, dst_var, edge_type, direction (Out|In), dst_type, min_hops, max_hops, dst_filters }` — destination filters are pushed *into* the expand so Lance scalar pushdown can prune. Executed one of two ways, chosen per-expand by a cost model over cheap manifest counts (frontier size, |E|, source-vertex count, hops) plus index coverage: selective traversals (small frontier relative to the source set) resolve neighbors from the persisted `src`/`dst` BTREE (one indexed scan per hop); dense / deep / large-frontier traversals — or those whose BTREE coverage is degraded so a full scan would be paid per hop — use the in-memory CSR adjacency index. Both produce identical results. The `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER` / `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS` ceilings bound the *initial dispatch* frontier/hops (beyond them CSR is always used); the cost model estimates total indexed work as ~`hops × frontier × fanout` and prices dense fan-out toward CSR — they are not a hard per-hop bound. `OMNIGRAPH_TRAVERSAL_MODE=indexed|csr` forces a mode (see [constants](constants.md)). -- `Filter { left, op, right }` -- `AntiJoin { outer_var, inner: Vec }` — for `not { … }` - -Lowering: - -1. Partition MATCH clauses (bindings, traversals, filters, negations). -2. Identify "deferred" bindings (a destination of a traversal that has filters) so the Expand can carry the filter as a pushdown. -3. Emit NodeScan for the first binding, then Expand operations, then remaining Filter operations, then AntiJoins for negations. -4. Translate RETURN / ORDER expressions; preserve LIMIT. - -## Linting & validation (`query/lint.rs`) - -Codes seen so far: - -- **Q000** (Error): parse error -- **L201** (Warning): nullable property never set by any UPDATE — "{type}.{prop} exists in schema but no update query sets it" -- (Warning): mutation declares no params — hardcoded mutations are easy to miss -- Plus all type errors from `typecheck_query_decl()` (undefined types, mismatched operators, undefined edges, etc.) - -Output: - -``` -QueryLintOutput { status, schema_source, query_path, - queries_processed, errors, warnings, infos, - results: [{ name, kind, status, error?, warnings[] }], - findings: [{ severity, code, message, type_name?, property?, query_names[] }] } -``` - -CLI exits non-zero only on `status = Error`. diff --git a/docs/user/quickstart.md b/docs/user/quickstart.md new file mode 100644 index 0000000..ae98e7c --- /dev/null +++ b/docs/user/quickstart.md @@ -0,0 +1,84 @@ +# Quickstart + +This walks the core loop end to end: define a schema, initialize a graph, load +data, query it, and use a branch. It uses a local file-backed graph; swap the +path for an `s3://…` URI to run the same flow against object storage. + +[Install](install.md) the `omnigraph` CLI first. + +## 1. Write a schema + +A schema (`.pg`) declares your node and edge types. Save this as `schema.pg`: + +``` +node Person { + name: String + title: String? +} +``` + +See the [schema language](schema/index.md) for types, constraints, and edges. + +## 2. Initialize the graph + +```bash +omnigraph init --schema schema.pg graph.omni +``` + +`init` creates an empty graph at the given URI with your schema applied. + +## 3. Load data + +`load` is the single bulk-write command. `--mode` is required +(`overwrite | append | merge`): + +```bash +omnigraph load --data people.jsonl --mode overwrite graph.omni +``` + +`people.jsonl` is newline-delimited JSON, one record per line. For finer-grained +or inline writes, see [mutations](mutations/index.md). + +## 4. Query + +Write a query (`.gq`) — save as `queries.gq`: + +```gq +query find_people($title: String) { + match { $p: Person { title: $title } } + return { $p.name } +} +``` + +Run it: + +```bash +omnigraph query find_people --query queries.gq \ + --params '{"title":"Engineer"}' --format table --store graph.omni +``` + +The query name is positional; `--query` points at the `.gq` source and +`--store` addresses the graph's storage directly. + +The [query language](queries/index.md) covers `match`/`return`/`order`, and +[search](search/index.md) covers vector and full-text search. + +## 5. Work on a branch + +Branches isolate changes until you merge them — Git-style, across the whole graph: + +```bash +omnigraph branch create review/new-hires graph.omni +omnigraph load --data new-hires.jsonl --mode append --branch review/new-hires graph.omni +# inspect the branch, then integrate it +omnigraph branch merge review/new-hires --into main graph.omni +``` + +See [branches & commits](branching/index.md) and [merging](branching/merge.md). + +## Next steps + +- [CLI reference](cli/reference.md) — every command and flag. +- [Schema language](schema/index.md) and [query language](queries/index.md). +- [Operating a cluster](clusters/index.md) and [running the server](operations/server.md) + for multi-graph, multi-user deployments. diff --git a/docs/user/reference/constants.md b/docs/user/reference/constants.md new file mode 100644 index 0000000..ec19f4d --- /dev/null +++ b/docs/user/reference/constants.md @@ -0,0 +1,42 @@ +# Constants & Tunables (cheat sheet) + +| Name | Value | Area | +|---|---|---| +| `MANIFEST_DIR` | `__manifest` | manifest layout | +| Commit graph dir | `_graph_commits.lance` | commit graph | +| Run registry dir (legacy, removed) | `_graph_runs.lance` | inert post-v0.4.0; bytes remain until a prefix-delete primitive lands | +| Run branch prefix (legacy, removed) | `__run__` | swept off `__manifest` by the internal schema migration; no longer a reserved name | +| Schema apply lock | `__schema_apply_lock__` | schema apply | +| Manifest publisher retry budget | `PUBLISHER_RETRY_BUDGET = 5` | manifest publish | +| Internal manifest schema version | `INTERNAL_MANIFEST_SCHEMA_VERSION = 3` | manifest migrations | +| Merge stage batch | `MERGE_STAGE_BATCH_ROWS = 8192` | merge execution | +| Maintenance concurrency | `OMNIGRAPH_MAINTENANCE_CONCURRENCY=8` | optimize/cleanup | +| Lance blob compaction support | `LANCE_SUPPORTS_BLOB_COMPACTION = false` | optimize | +| Graph index cache size | `8` (LRU) | runtime cache | +| Expand indexed-path frontier ceiling | `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER=1024` | traversal | +| Expand indexed-path hop ceiling | `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS=6` | traversal | +| Expand CSR-build cost factor | `CSR_BUILD_FACTOR = 1.5` | traversal | +| Expand mode override | `OMNIGRAPH_TRAVERSAL_MODE` (`indexed`\|`csr`; unset = cost-based auto) | traversal | +| Default body limit | `1 MB` | HTTP server | +| Ingest body limit | `32 MB` | HTTP server | +| Default embed provider/model | `openai-compatible` / `openai/text-embedding-3-large` | engine embedding | +| OpenAI-direct embed model | `text-embedding-3-large` | engine embedding | +| Gemini-direct embed model | `gemini-embedding-2` | engine embedding | +| Embed deadline | `OMNIGRAPH_EMBED_DEADLINE_MS=60000` | engine embedding | +| Embed timeout | `OMNIGRAPH_EMBED_TIMEOUT_MS=30000` | engine embedding | +| Embed retries | `OMNIGRAPH_EMBED_RETRY_ATTEMPTS=4` | engine embedding | +| Embed retry backoff | `OMNIGRAPH_EMBED_RETRY_BACKOFF_MS=200` | engine embedding | +| LANCE memory pool default | `1 GB` (raised in v0.3.0) | runtime | + +**Expand traversal dispatch.** With `OMNIGRAPH_TRAVERSAL_MODE` unset, the engine +chooses the indexed (per-hop BTREE) vs CSR (whole-graph in-memory) path with a +cost model over cheap manifest counts (frontier size, |E|, source-vertex count, +hops) plus the index-coverage signal: the indexed path is preferred when its +frontier-relative work beats building the CSR (≈ when `hops × frontier` is a +small fraction of the source-vertex set), and CSR is preferred for dense/deep +traversals or when the BTREE coverage is degraded and a full scan would be paid +per hop. The two ceilings bound the **initial dispatch** frontier/hops (beyond +them CSR is always used); they are not a hard per-hop bound — the cost model +*estimates* total indexed work as ~`hops × frontier × fanout`, so dense fan-out is +priced toward CSR rather than capped mid-traversal. The override flag forces a path (the `auto` result is identical either way; +only the path differs). diff --git a/docs/user/schema-language.md b/docs/user/schema/index.md similarity index 60% rename from docs/user/schema-language.md rename to docs/user/schema/index.md index 4250676..105281c 100644 --- a/docs/user/schema-language.md +++ b/docs/user/schema/index.md @@ -1,7 +1,5 @@ # Schema Language (`.pg`) -Pest grammar at `crates/omnigraph-compiler/src/schema/schema.pest`. AST at `schema/ast.rs`. Catalog at `catalog/mod.rs`. - ## Top-level declarations - `interface { property* }` — reusable property contracts. @@ -47,37 +45,28 @@ Edge bodies only allow `@unique` and `@index`. - `@` or `@()` on any declaration or property. - Known annotations: - - `@embed` on a Vector property — names the *source* property whose text gets embedded into this vector at ingest (`embed_sources` map in NodeType). + - `@embed("source_property")` on a Vector property — records which String property is the embedding source for query-time `nearest($v, "string")` auto-embedding. It is a catalog annotation; it does **not** populate the vector at ingest (supply vectors in load data, or pre-fill via the offline `omnigraph embed` pipeline). An optional `model="…"` kwarg (`@embed("source_property", model="openai/text-embedding-3-large")`) records the embedding model so a `nearest()` query whose embedder uses a different model is rejected loudly; `model` is the only supported kwarg. See [search/embeddings.md](../search/embeddings.md). - `@description("…")`, `@instruction("…")` on query declarations (carried through to clients). - Custom annotations are accepted by the parser and surfaced in catalog metadata; unrecognized annotations don't fail compilation. -## Catalog construction +## Table layout -- Pass 0: collect interfaces. -- Pass 1: collect nodes, expand `implements`, build constraint and `@embed` mappings, build the Arrow schema for each node table (`id: Utf8` plus all properties; blob columns get `LargeBinary`). -- Pass 2: collect edges, validate that `from_type` / `to_type` exist, normalize edge names case-insensitively for lookup, validate constraints for edges. Edge Arrow schema: `id: Utf8, src: Utf8, dst: Utf8` plus edge properties. - -## Schema IR & stable type IDs - -- `SCHEMA_IR_VERSION = 1` (`catalog/schema_ir.rs`). -- Each interface/node/edge currently gets a `stable_type_id` from a kind+name hash. -- Rename-preserving accepted IDs are an architectural invariant, but the current hash-on-name implementation is a known gap until migration carries IDs across `@rename_from`. -- Serialized as JSON for diff/migration plans. +- Each node type compiles to a table with an `id: Utf8` column plus all declared properties (blob columns are stored as `LargeBinary`); `implements` clauses expand the interface's properties into the node. +- Each edge type compiles to a table with `id: Utf8, src: Utf8, dst: Utf8` plus the edge's own properties. Edge endpoint types (`from`/`to`) must exist, and edge names are matched case-insensitively. ## Schema migration planning -`plan_schema_migration(accepted, desired) -> SchemaMigrationPlan { supported, steps[] }` with step types: +A migration plan compares the accepted schema against the desired one and reports whether the change is supported plus the ordered steps it requires: -- `AddType { type_kind, name }` -- `RenameType { type_kind, from, to }` -- `AddProperty { type_kind, type_name, property_name, property_type }` -- `RenameProperty { type_kind, type_name, from, to }` -- `AddConstraint { type_kind, type_name, constraint }` -- `UpdateTypeMetadata { … annotations }` -- `UpdatePropertyMetadata { … annotations }` -- `UnsupportedChange { entity, reason }` (forces `supported=false`) +- Add a type +- Rename a type +- Add a property +- Rename a property +- Add a constraint +- Update type or property metadata (annotations) +- Unsupported change (reports the entity and reason; forces the plan to unsupported) -`apply_schema()` returns `SchemaApplyResult { supported, applied, manifest_version, steps }` and is gated by an internal `__schema_apply_lock__` system branch so concurrent schema applies serialize. +Applying a plan reports whether it was supported, the steps applied, and the resulting manifest version. Concurrent schema applies serialize so they can't interleave. ## Destructive drops — `--allow-data-loss` diff --git a/docs/user/schema-lint.md b/docs/user/schema/lint.md similarity index 58% rename from docs/user/schema-lint.md rename to docs/user/schema/lint.md index a1495fd..6635e9f 100644 --- a/docs/user/schema-lint.md +++ b/docs/user/schema/lint.md @@ -2,29 +2,26 @@ The migration planner emits **code-tagged diagnostics** for every schema change it rejects. Codes have the form `OG-XXX-NNN` and identify the rule (not the message); operators reference them in suppression directives, severity overrides, and CI reports. -This page is the catalog of codes shipped today. The chassis behind it is tracked in [MR-694](https://linear.app/modernrelay/issue/MR-694). +This page is the catalog of codes shipped today. -## What's shipped in v0 +## What's shipped -- Stable code attached to every rejection the planner emits (today: 5 of 17 paths — the rest carry `code: None` and are tagged as future work). +- Stable code attached to every rejection the planner emits (today: 5 of 17 paths — the rest are tagged as future work). - Code appears in the user-visible error message: `[OG-DS-104] removing property 'Person.age' is not supported …`. - CLI `omnigraph schema plan` shows the code on `unsupported change …` lines. -- Tests in `tests/schema_apply.rs` assert on codes, not on free-text prose. ## What's not shipped yet -- Severity configuration in `omnigraph.yaml` (planned: `lint: { OG-DS-103: error }`). +- Severity configuration (planned: `lint: { OG-DS-103: error }`). - `@allow(OG-XXX-NNN, "rationale")` suppression directives. -- Pre-migration checks (the `migration_check { … }` block — MR-941). -- The CD / VE / LK / NM families (MR-942..945). -- CI integration (MR-946). -- Cost-class annotations (MR-944). +- Pre-migration checks (the `migration_check { … }` block). +- The CD / VE / LK / NM families. +- CI integration. +- Cost-class annotations. -See the parent chassis issue (MR-694) for the design and the per-family sub-issues for what's planned. +## Code catalog -## Code catalog (v0) - -The chassis defines ten families. Today only DS and MF have emitted codes. The remaining families are reserved for future PRs. +The chassis defines ten families. Today only DS and MF have emitted codes. The remaining families are reserved for future releases. | Code | Family | Tier | Default severity | Meaning | |---|---|---|---|---| @@ -37,24 +34,22 @@ The chassis defines ten families. Today only DS and MF have emitted codes. The r | `OG-MF-104` | Maybe-fail | validated | error | tighten nullable to non-nullable (reserved) | | `OG-MF-106` | Maybe-fail | destructive | error | narrowing scalar type | -The full code catalog source of truth lives in `crates/omnigraph-compiler/src/lint/codes.rs`. CI-level invariants (uniqueness, format, family coverage) are unit-tested in the same module. - ## Families The ten chassis families: | Prefix | Family | Status | |---|---|---| -| **DS** | Destructive (data-loss) | shipped, v0 | -| **MF** | Maybe-fail / data-dependent | shipped, v0 | -| **CD** | Constraint deletion (relaxation warning) | tracked in MR-942 | +| **DS** | Destructive (data-loss) | shipped | +| **MF** | Maybe-fail / data-dependent | shipped | +| **CD** | Constraint deletion (relaxation warning) | planned | | **BC** | Backward-incompatible (rename) | implicit in `@rename_from`; codify later | -| **NM** | Naming conventions | tracked in MR-945 | -| **OW** | Ownership (per-resource Cedar) | tracked in MR-722 | -| **NL** | Non-linear (branch-merge divergence) | stubbed in MR-947 | -| **VE** | Vector / embedding | tracked in MR-943 | -| **ED** | Edge / graph topology | tracked in MR-701, MR-943 | -| **LK** | Lock duration / cost | tracked in MR-944 | +| **NM** | Naming conventions | planned | +| **OW** | Ownership (per-resource Cedar) | planned | +| **NL** | Non-linear (branch-merge divergence) | planned | +| **VE** | Vector / embedding | planned | +| **ED** | Edge / graph topology | planned | +| **LK** | Lock duration / cost | planned | ## Prior art diff --git a/docs/user/search/embeddings.md b/docs/user/search/embeddings.md new file mode 100644 index 0000000..e69d928 --- /dev/null +++ b/docs/user/search/embeddings.md @@ -0,0 +1,112 @@ +# Embeddings + +OmniGraph embeds text through a **single, provider-independent client** resolved from one +`EmbeddingConfig { provider, model, base_url, api_key }`. The same resolved config is used by the query-time +auto-embed of a string in `nearest($v, "string")` and by the offline `omnigraph embed` file pipeline, so +query vectors and document vectors share one model and one vector space. + +## Providers + +| `provider` | Wire shape | Use it for | +|---|---|---| +| `openai-compatible` (default) | `POST {base}/embeddings`, bearer auth, `{model, input, dimensions}` | **OpenRouter** (the default gateway — one key for many models), OpenAI direct, or a self-hosted endpoint (vLLM / Ollama / LM Studio) | +| `gemini` | `POST {base}/models/{model}:embedContent`, `x-goog-api-key`, with `RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT` task types | Reaching Google's `generativelanguage` API directly | +| `mock` | none — deterministic offline vectors | Tests and local dev without a key | + +Vectors are stored L2-normalized as `FixedSizeList(Float32, dim)`; the requested output dimension is driven by +the target column width and sent as Gemini `outputDimensionality` / OpenAI `dimensions`. + +## Configuration (cluster) + +Cluster-served graphs can pin their query-time embedder in `cluster.yaml`: + +```yaml +providers: + embedding: + default: + kind: openai-compatible + base_url: https://openrouter.ai/api/v1 + model: openai/text-embedding-3-large + api_key: ${OPENROUTER_API_KEY} + +graphs: + knowledge: + schema: knowledge.pg + embedding_provider: default +``` + +`embedding_provider` references `providers.embedding.`; bare names are +normalized to that typed ref. The server resolves `${ENV_VAR}` only when it +boots from the applied cluster ledger, so `cluster validate`, `plan`, and +`apply` do not need provider secrets. Inline API keys are rejected. `mock` +needs no key. Vector dimensions stay schema-driven by the target `Vector(N)` +column. + +Direct single-graph serving, embedded callers, and the offline +`omnigraph embed` pipeline use environment configuration unless they inject an +`EmbeddingConfig` directly. + +## Configuration (environment) + +| Variable | Meaning | +|---|---| +| `OMNIGRAPH_EMBED_PROVIDER` | `openai-compatible` (default, → OpenRouter) \| `openai` (→ OpenAI's own host) \| `gemini` \| `mock` | +| `OMNIGRAPH_EMBED_BASE_URL` | endpoint base; defaults `https://openrouter.ai/api/v1` (`openai-compatible`/unset), `https://api.openai.com/v1` (`openai`), `https://generativelanguage.googleapis.com/v1beta` (`gemini`) | +| `OMNIGRAPH_EMBED_MODEL` | model id; defaults `openai/text-embedding-3-large` (OpenRouter), `text-embedding-3-large` (`openai`), `gemini-embedding-2` (`gemini`) | +| `OPENROUTER_API_KEY` / `OPENAI_API_KEY` | api key for `openai-compatible` (OpenRouter preferred) | +| `GEMINI_API_KEY` | api key for `gemini` | +| `OMNIGRAPH_EMBED_DEADLINE_MS` | total wall-clock budget for one embed call across all retries (default `60000`; `0` = unbounded) | +| `OMNIGRAPH_EMBED_TIMEOUT_MS` | per-request HTTP timeout (default `30000`) | +| `OMNIGRAPH_EMBED_RETRY_ATTEMPTS` / `OMNIGRAPH_EMBED_RETRY_BACKOFF_MS` | retry policy (defaults `4` / `200`) | +| `OMNIGRAPH_EMBEDDINGS_MOCK` | set truthy to force the deterministic mock provider | + +The default zero-config path is OpenRouter: set `OPENROUTER_API_KEY` and run. Reaching Gemini takes +`OMNIGRAPH_EMBED_PROVIDER=gemini` plus `GEMINI_API_KEY`. + +### Behavior notes + +- **Bounded latency.** Each embed call is wrapped in `OMNIGRAPH_EMBED_DEADLINE_MS`, so a degraded + provider cannot hang a read for the full retry envelope. +- **Reuse.** The query path builds the client once per graph handle (on the first `nearest($v, "string")` + that needs embedding) and reuses it, keeping the provider connection pool warm. A graph that never embeds + needs no provider key. +- **Observability.** Embed calls emit `tracing` events under `target = "omnigraph::embedding"` (provider, + model, dim, attempt, elapsed, outcome). + +## `@embed` schema annotation + +Mark a Vector property with `@embed("source_text_property")`. This is a **catalog annotation** consumed by the +query typechecker and linter: it records which String property is the embedding source and lets +`nearest($v, "string")` auto-embed a query string for comparison against that vector column. + +Optionally record the model that produced the stored vectors: +`@embed("source_text_property", model="openai/text-embedding-3-large")`. When a model is recorded, a +`nearest($v, "string")` query is **rejected with a typed error** unless the resolved query embedder uses the +same model — so stored and query vectors are guaranteed same-space instead of silently ranking across spaces. +To fix a mismatch, set `OMNIGRAPH_EMBED_MODEL` (and the matching provider) to the recorded model, or re-embed. +The recorded model is the literal string, so `openai/text-embedding-3-large` (via OpenRouter) and +`text-embedding-3-large` (OpenAI direct) are distinct identities; use the matching string. Changing a recorded +model is a loud `schema apply` refusal (treat it as a re-embed migration). `@embed` without a model keeps +working with no validation. `model` is the only supported `@embed` argument; any other is a parse error. + +**It does not embed at ingest.** Stored vectors are supplied directly in your load data, or pre-filled by the +offline `omnigraph embed` pipeline below. (Ingest-time execution of `@embed` is a planned enhancement.) + +## CLI `omnigraph embed` (offline file pipeline) + +Operates on **JSONL files** (not on a graph), using the same resolved provider config. Three modes (mutually +exclusive): + +- (default) `fill_missing` — only embed rows whose target field is empty +- `--reembed-all` — overwrite all +- `--clean` — strip embeddings + +Inputs are either a single seed manifest YAML or `--input/--output/--spec`. Selectors `--type T`, `--select T:field=value` filter rows. Streams JSONL → JSONL. + +## Migration + +This release has no backwards-compatibility shim (pre-release). The default provider is now OpenRouter, and +the legacy `OMNIGRAPH_GEMINI_BASE_URL` is removed. A graph whose vectors were produced with +`gemini-embedding-2-preview` should either re-embed, or pin the query-time embedder to match by setting +`OMNIGRAPH_EMBED_PROVIDER=gemini` and `OMNIGRAPH_EMBED_MODEL=gemini-embedding-2-preview` (the stored and query +vectors must come from the same model to be comparable). diff --git a/docs/user/search/index.md b/docs/user/search/index.md new file mode 100644 index 0000000..280e9e8 --- /dev/null +++ b/docs/user/search/index.md @@ -0,0 +1,48 @@ +# Search + +OmniGraph runs vector, full-text, and hybrid search in the same runtime as graph +traversal — a single [query](../queries/index.md) can combine a vector `nearest`, +a `bm25` text score, and an `Expand` traversal. Search functions are used inside +`match` (to filter), or as expressions inside `return` / `order` (to score and +rank). + +## Functions + +| Function | Purpose | Backing index | +|---|---|---| +| `nearest($x.vec, $q)` | k-NN vector search (cosine) | vector index (IVF / HNSW) | +| `search(field, q)` | Generic full-text search | inverted (FTS) index | +| `fuzzy(field, q [, max_edits])` | Levenshtein-tolerant text search | inverted index | +| `match_text(field, q)` | Pattern match | inverted index | +| `bm25(field, q)` | BM25 relevance scoring | inverted index | +| `rrf(rank_a, rank_b [, k])` | Reciprocal Rank Fusion of two rankings (default `k=60`) | fuses scored rankings | + +- `nearest()` requires a `limit`. The query vector is resolved from the param map, + or embedded from a text input at runtime via the configured + [embedding client](embeddings.md). +- Scores and ranks propagate as ordinary columns, so you can `return` a score and + `order` by it. + +## Hybrid ranking with `rrf` + +Reciprocal Rank Fusion combines two independent rankings (typically one vector and +one text) into a single fused ranking, without needing the two score scales to be +comparable. Rank each retrieval separately, then fuse: + +```gq +query hybrid($q: String) { + match { $d: Document { } } + return { + $d, + rrf( nearest($d.embedding, $q), bm25($d.body, $q) ) as score + } + order { score desc } + limit 10 +} +``` + +## Indexes and embeddings + +Search functions only work when the backing index exists — see +[indexes](indexes.md) for building vector and inverted indexes, and +[embeddings](embeddings.md) for generating the vectors `nearest` searches over. diff --git a/docs/user/search/indexes.md b/docs/user/search/indexes.md new file mode 100644 index 0000000..57935cd --- /dev/null +++ b/docs/user/search/indexes.md @@ -0,0 +1,43 @@ +# Indexes + +## L1 — Lance index types OmniGraph exposes + +| Index | Use | Notes | +|---|---|---| +| **BTREE scalar** | `=` / range / `IN` / `IS NULL` on a scalar | always on the node `id` and edge `src`/`dst`; and on each one-column `@index`/`@key` property that is an **enum** or an **orderable scalar** (`DateTime`/`Date`/`I32`/`I64`/`U32`/`U64`/`F32`/`F64`/`Bool`) | +| **Inverted (FTS)** | `search`, `fuzzy`, `match_text`, `bm25` | created on **free-text** (non-enum) `String` `@index`/`@key` columns | +| **Vector** | `nearest()` k-NN | Lance picks IVF_PQ vs HNSW family by configuration; OmniGraph stores as FixedSizeList(Float32, dim) | + +The per-property index a column gets is decided by `node_prop_index_kind` (shared +by the builder and the sidecar-pinning coverage check so they cannot drift): +enums and orderable scalars → BTREE, free-text Strings → FTS, `Vector` → vector, +list/`Blob` columns → none. + +> **Free-text Strings are not equality-indexed.** A non-enum `String` column +> (including a `String @key` slug) gets an FTS inverted index, which Lance does +> **not** consult for `=`/range — only for `search`/`match_text`/`bm25`. So an +> equality filter on a free-text String falls back to a full scan. If you filter +> a String identifier by equality on a large table, model it so the value is the +> node id, or track it as a follow-up to also build a BTREE on such columns. + +> **Coverage and cost.** Each indexed column adds index files and build time, and +> an index only covers the fragments it was built over. Rows appended after the +> index was built (e.g. by `ingest --mode merge`) are scanned unindexed until a +> reindex extends coverage; see [maintenance](../operations/maintenance.md) → `optimize`. + +## L2 — OmniGraph orchestration + +- **`@index`/`@key` declares intent; the physical index is derived state.** A migration records the declaration in the catalog/IR and never fails on it — `schema apply` builds **no** indexes (adding an `@index` to an existing column is a pure metadata change that touches no table data). `load`/`mutate` build declared indexes inline as part of the write, but a column that can't be built yet (a `Vector` column with no trainable vectors — IVF k-means needs ≥1 vector, e.g. rows loaded before `embed` runs) is left **pending**, not fatal. Reads stay correct meanwhile: a missing/partial index degrades to a scan (vector search to brute-force). A later `ensure_indices`/`optimize` materializes the pending index once it is buildable. This mirrors how LanceDB builds indexes asynchronously and serves unindexed rows by brute-force. +- `ensure_indices()` / `ensure_indices_on(branch)` — idempotent build of BTREE + inverted + vector indexes for the current head; safe to re-run; returns the columns it had to defer as pending. `optimize` runs it after compaction, so the maintenance cron is the convergence path for deferred indexes. +- Indexes are built on the *branch head* (not on a snapshot), so reads always see the current index state. +- **Lazy branch forking for indexes**: a branch that hasn't mutated a sub-table doesn't need its own index — the main lineage's index is reused until the first write triggers a copy-on-write fork. +- Vector index parameters (metric, nlist, nprobe, etc.) are not exposed in the schema; they default at the Lance layer and are picked up automatically when an index is asked for on a Vector column. + +## L2 — Graph topology index + +This is OmniGraph-specific (not Lance): + +- A Compressed Sparse Row (CSR) adjacency representation of edges, with both out- (CSR) and in- (CSC) directions, plus a dense per-node-type id mapping. +- Built on demand from a snapshot's edge tables, **lazily**: only when an `Expand` the planner routes to the CSR path (dense / large frontier) or an `AntiJoin` actually needs it. +- Cached per snapshot (LRU, keyed by snapshot id + edge table versions), so repeat traversals over the same snapshot reuse it. +- Selective `Expand`s resolve neighbors from the persisted `src`/`dst` BTREE instead (one indexed scan per hop) and never trigger the CSR build; see [query-language](../queries/index.md) → Traversal execution. Pure scans, and queries served entirely by the indexed traversal path, skip it. diff --git a/openapi.json b/openapi.json index 6e3dd03..fb76fae 100644 --- a/openapi.json +++ b/openapi.json @@ -10,14 +10,82 @@ "version": "0.7.0" }, "paths": { - "/branches": { + "/graphs": { + "get": { + "tags": [ + "management" + ], + "summary": "List every graph currently registered with this server (MR-668).", + "description": "Multi-graph mode only. In single mode, the route returns 405 — there's\nno registry to enumerate. Cedar-gated by the server-level policy via\nthe `graph_list` action against `Omnigraph::Server::\"root\"`.\n\nOrder: alphabetical by `graph_id` (server-sorted so clients see\ndeterministic output across requests).", + "operationId": "listGraphs", + "responses": { + "200": { + "description": "List of registered graphs", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GraphListResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "405": { + "description": "Method not allowed (single-graph mode)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/{graph_id}/branches": { "get": { "tags": [ "branches" ], "summary": "List all branches.", "description": "Returns branch names sorted alphabetically. Read-only.", - "operationId": "listBranches", + "operationId": "cluster_listBranches", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "200": { "description": "List of branches", @@ -62,7 +130,18 @@ ], "summary": "Create a new branch.", "description": "Forks `name` off of `from` (defaults to `main`). The new branch shares\ntable data with its parent until it is mutated. Returns 409 if `name`\nalready exists.", - "operationId": "createBranch", + "operationId": "cluster_createBranch", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -142,14 +221,25 @@ ] } }, - "/branches/merge": { + "/graphs/{graph_id}/branches/merge": { "post": { "tags": [ "branches" ], "summary": "Merge one branch into another.", "description": "Merges `source` into `target` (defaults to `main`). Outcome is one of\n`already_up_to_date`, `fast_forward`, or `merged`. Returns 409 with the\nlist of conflicts if the merge cannot be completed; the target is left\nunchanged in that case. **Destructive** to `target` on success.", - "operationId": "mergeBranches", + "operationId": "cluster_mergeBranches", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -229,15 +319,24 @@ ] } }, - "/branches/{branch}": { + "/graphs/{graph_id}/branches/{branch}": { "delete": { "tags": [ "branches" ], "summary": "Delete a branch.", "description": "**Irreversible.** Removes the branch pointer; commits remain reachable\nonly if referenced by another branch. Returns 404 if the branch does not\nexist.", - "operationId": "deleteBranch", + "operationId": "cluster_deleteBranch", "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "branch", "in": "path", @@ -307,14 +406,25 @@ ] } }, - "/change": { + "/graphs/{graph_id}/change": { "post": { "tags": [ "mutations" ], "summary": "**Deprecated** — use [`POST /mutate`](#tag/mutations/operation/mutate) instead.", - "description": "Apply a GQ mutation to a branch. Behavior is unchanged; the route is\nkept indefinitely for back-compat. New integrations should target\n`POST /mutate`, which has identical semantics and a name that pairs\ncleanly with `POST /query`. Responses from this route include\n`Deprecation: true` and `Link: ; rel=\"successor-version\"`\nheaders per RFC 9745 / RFC 8288 so SDKs and proxies can surface the\nsignal.", - "operationId": "change", + "description": "Apply a GQ mutation to a branch. Behavior is unchanged; the route is\nkept indefinitely for back-compat. New integrations should target\n`POST /mutate`, which has identical semantics and a name that pairs\ncleanly with `POST /query`. Responses from this route include\n`Deprecation: true` and `Link: ; rel=\"successor-version\"`\nheaders per RFC 9745 / RFC 8288 so SDKs and proxies can surface the\nsignal.", + "operationId": "cluster_change", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -327,7 +437,7 @@ }, "responses": { "200": { - "description": "Mutation results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", + "description": "Mutation results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", "content": { "application/json": { "schema": { @@ -395,15 +505,24 @@ ] } }, - "/commits": { + "/graphs/{graph_id}/commits": { "get": { "tags": [ "commits" ], "summary": "List commits.", "description": "Filter by `branch` to get the commits on a single branch (most recent\nfirst); omit to list across all branches. Read-only.", - "operationId": "listCommits", + "operationId": "cluster_listCommits", "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "branch", "in": "query", @@ -455,15 +574,24 @@ ] } }, - "/commits/{commit_id}": { + "/graphs/{graph_id}/commits/{commit_id}": { "get": { "tags": [ "commits" ], "summary": "Get a single commit.", "description": "Returns the commit's manifest version, parent commit(s), and creation\nmetadata. Read-only.", - "operationId": "getCommit", + "operationId": "cluster_getCommit", "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "commit_id", "in": "path", @@ -523,14 +651,25 @@ ] } }, - "/export": { + "/graphs/{graph_id}/export": { "post": { "tags": [ "queries" ], "summary": "Stream the contents of a branch as NDJSON.", "description": "Emits one JSON object per line (`application/x-ndjson`). Filter with\n`type_names` (node/edge type names) and/or `table_keys`; both empty\nstreams the entire branch. Suitable for large exports — the response is\nstreamed, not buffered. Read-only.", - "operationId": "export", + "operationId": "cluster_export", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -586,21 +725,52 @@ ] } }, - "/graphs": { - "get": { + "/graphs/{graph_id}/ingest": { + "post": { "tags": [ - "management" + "mutations" ], - "summary": "List every graph currently registered with this server (MR-668).", - "description": "Multi-graph mode only. In single mode, the route returns 405 — there's\nno registry to enumerate. Cedar-gated by the server-level policy via\nthe `graph_list` action against `Omnigraph::Server::\"root\"`.\n\nOrder: alphabetical by `graph_id` (server-sorted so clients see\ndeterministic output across requests).", - "operationId": "listGraphs", + "summary": "**Deprecated** — use [`POST /load`](#tag/mutations/operation/load) instead.", + "description": "Bulk-load NDJSON data into a branch. Behavior is unchanged; the route is\nkept indefinitely for back-compat. New integrations should target\n`POST /load`, which has identical semantics. Responses from this route\ninclude `Deprecation: true` and `Link: ; rel=\"successor-version\"`\nheaders per RFC 9745 / RFC 8288 so SDKs and proxies can surface the signal.", + "operationId": "cluster_ingest", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IngestRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "List of registered graphs", + "description": "Load results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GraphListResponse" + "$ref": "#/components/schemas/IngestOutput" + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -625,8 +795,8 @@ } } }, - "405": { - "description": "Method not allowed (single-graph mode)", + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header", "content": { "application/json": { "schema": { @@ -636,6 +806,7 @@ } } }, + "deprecated": true, "security": [ { "bearer_token": [] @@ -643,36 +814,25 @@ ] } }, - "/healthz": { - "get": { - "tags": [ - "health" - ], - "summary": "Liveness probe.", - "description": "Returns server status and version. Unauthenticated; safe to call from any\ncaller. Use this to confirm the server is reachable before invoking other\nendpoints.", - "operationId": "health", - "responses": { - "200": { - "description": "Server is healthy", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HealthOutput" - } - } - } - } - } - } - }, - "/ingest": { + "/graphs/{graph_id}/load": { "post": { "tags": [ "mutations" ], - "summary": "Bulk-load NDJSON data into a branch.", - "description": "`data` is NDJSON with one record per line. `mode` controls behavior on\nexisting rows: `merge` upserts by id (default), `append` blindly inserts,\n`overwrite` replaces table contents. Branch creation is opt-in by\npresence of `from`: with `from` set, a missing `branch` is created from\nit; without `from`, `branch` must already exist — a missing branch is a\n404, never an implicit fork. **Destructive** when `mode` is `overwrite`\nor when the load produces conflicting writes.", - "operationId": "ingest", + "summary": "Bulk-load NDJSON data into a branch (canonical load endpoint).", + "description": "`data` is NDJSON with one record per line. `mode` controls behavior on\nexisting rows: `merge` upserts by id (default), `append` blindly inserts,\n`overwrite` replaces table contents. Branch creation is opt-in by\npresence of `from`: with `from` set, a missing `branch` is created from\nit; without `from`, `branch` must already exist — a missing branch is a\n404, never an implicit fork. **Destructive** when `mode` is `overwrite`\nor when the load produces conflicting writes.\n\nThe legacy `POST /ingest` route has identical semantics and is kept as a\ndeprecated alias.", + "operationId": "cluster_load", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -685,7 +845,7 @@ }, "responses": { "200": { - "description": "Ingest results", + "description": "Load results", "content": { "application/json": { "schema": { @@ -742,14 +902,25 @@ ] } }, - "/mutate": { + "/graphs/{graph_id}/mutate": { "post": { "tags": [ "mutations" ], "summary": "Apply a GQ mutation to a branch (canonical mutation endpoint).", "description": "Writes to the named `branch` (defaults to `main`). Mutations are atomic\nper call and produce a new commit. Returns counts of nodes and edges\naffected. **Destructive**: on success the branch is updated; rejected\nmutations may still acquire locks briefly. Returns 409 on merge conflict.\n\nPairs with `POST /query` (read-only). The legacy `POST /change` route\nhas identical semantics and is kept as a deprecated alias.", - "operationId": "mutate", + "operationId": "cluster_mutate", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -829,14 +1000,25 @@ ] } }, - "/queries": { + "/graphs/{graph_id}/queries": { "get": { "tags": [ "queries" ], "summary": "List the graph's exposed stored queries as a typed tool catalog.", "description": "Returns the `mcp.expose == true` subset of the `queries:` registry, each\nwith its MCP tool name, read/mutate flag, description/instruction, and\ntyped parameters — enough for a client to register them as tools without\nfetching `.gq` source. Read-gated; the catalog is graph-wide (branch\nindependent — `read` is authorized against `main`). **Not** Cedar-filtered\nper query yet, so it can list a query whose `invoke_query` the caller\nlacks (a known gap until per-query authorization lands).", - "operationId": "list_queries", + "operationId": "cluster_list_queries", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "200": { "description": "Stored-query catalog (the mcp.expose subset, with typed params)", @@ -876,15 +1058,24 @@ ] } }, - "/queries/{name}": { + "/graphs/{graph_id}/queries/{name}": { "post": { "tags": [ "queries" ], "summary": "Invoke a curated, server-side stored query by name.", "description": "The query source comes from the graph's `queries:` registry, not the\nrequest body — callers send only runtime inputs (`params`, `branch`,\n`snapshot`). Gated by the `invoke_query` Cedar action at the boundary;\na stored *mutation* additionally passes the engine's `change` gate\n(double-gated). An actor **without** `invoke_query` cannot tell a denied\nquery from a missing one — both return the same 404, so the catalog\ncan't be probed without the grant. Once `invoke_query` is held, the\ninner `read`/`change` gate may surface a 403 for an existing query the\nactor can't run (the intended double-gate signal).", - "operationId": "invoke_query", + "operationId": "cluster_invoke_query", "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "name", "in": "path", @@ -1000,14 +1191,25 @@ ] } }, - "/query": { + "/graphs/{graph_id}/query": { "post": { "tags": [ "queries" ], "summary": "Execute an inline read query (friendlier-named alternative to `POST /read`).", "description": "Designed for ad-hoc exploration and AI-agent tool-use: short field\nnames (`query`, `name`) match the CLI `-e` flag and the GQ `query`\nkeyword. Mutations (`insert`/`update`/`delete`) are rejected with 400\n-- use `POST /mutate` (or its deprecated alias `POST /change`) for\nwrite queries. Otherwise behaves identically to `POST /read`: same\ntarget semantics (branch xor snapshot), same Cedar action (Read),\nsame response shape.", - "operationId": "query", + "operationId": "cluster_query", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -1067,14 +1269,25 @@ ] } }, - "/read": { + "/graphs/{graph_id}/read": { "post": { "tags": [ "queries" ], "summary": "**Deprecated** — use [`POST /query`](#tag/queries/operation/query) instead.", - "description": "Execute a GQ read query. Behavior is unchanged from prior releases; the\nroute is kept indefinitely for byte-stable back-compat. New integrations\nshould target `POST /query`, which has clean field names (`query` /\n`name`) and a 400-on-mutation guard. Responses from this route include\n`Deprecation: true` and `Link: ; rel=\"successor-version\"`\nheaders per RFC 9745 / RFC 8288 so SDKs and proxies can surface the\nsignal.", - "operationId": "read", + "description": "Execute a GQ read query. Behavior is unchanged from prior releases; the\nroute is kept indefinitely for byte-stable back-compat. New integrations\nshould target `POST /query`, which has clean field names (`query` /\n`name`) and a 400-on-mutation guard. Responses from this route include\n`Deprecation: true` and `Link: ; rel=\"successor-version\"`\nheaders per RFC 9745 / RFC 8288 so SDKs and proxies can surface the\nsignal.", + "operationId": "cluster_read", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -1087,7 +1300,7 @@ }, "responses": { "200": { - "description": "Query results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", + "description": "Query results (response includes `Deprecation: true` + `Link: ; rel=\"successor-version\"`)", "content": { "application/json": { "schema": { @@ -1135,14 +1348,25 @@ ] } }, - "/schema": { + "/graphs/{graph_id}/schema": { "get": { "tags": [ "schema" ], "summary": "Read the current schema source.", "description": "Returns the project's schema as a single string in `.pg` source form.\nUseful for clients that want to introspect available types and tables\nbefore constructing GQ queries. Read-only.", - "operationId": "getSchema", + "operationId": "cluster_getSchema", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "responses": { "200": { "description": "Current schema source", @@ -1182,14 +1406,25 @@ ] } }, - "/schema/apply": { + "/graphs/{graph_id}/schema/apply": { "post": { "tags": [ "mutations" ], "summary": "Apply a schema migration.", - "description": "Diffs `schema_source` against the current schema and applies the resulting\nmigration steps (add/drop type, add/drop column, etc.). **Destructive**:\nsome steps drop data. Returns the list of steps applied; if `applied` is\nfalse the diff was unsupported and no changes were made.", - "operationId": "applySchema", + "description": "Cluster-backed servers reject this route with `409 Conflict`; operators\nmust apply schema changes through `omnigraph cluster apply` and restart.\n\nDiffs `schema_source` against the current schema and applies the resulting\nmigration steps (add/drop type, add/drop column, etc.). **Destructive**:\nsome steps drop data. Returns the list of steps applied; if `applied` is\nfalse the diff was unsupported and no changes were made.", + "operationId": "cluster_applySchema", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -1241,6 +1476,16 @@ } } }, + "409": { + "description": "Schema apply is disabled for cluster-backed serving; use `omnigraph cluster apply` and restart", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, "429": { "description": "Per-actor admission cap exceeded; honor `Retry-After` header", "content": { @@ -1259,15 +1504,24 @@ ] } }, - "/snapshot": { + "/graphs/{graph_id}/snapshot": { "get": { "tags": [ "snapshots" ], "summary": "Read the current snapshot of a branch.", "description": "Returns the manifest version plus per-table metadata (path, version, row\ncount) for every table on the branch. Defaults to `main` when `branch` is\nomitted. Read-only.", - "operationId": "getSnapshot", + "operationId": "cluster_getSnapshot", "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "branch", "in": "query", @@ -1318,6 +1572,28 @@ } ] } + }, + "/healthz": { + "get": { + "tags": [ + "health" + ], + "summary": "Liveness probe.", + "description": "Returns server status and version. Unauthenticated; safe to call from any\ncaller. Use this to confirm the server is reachable before invoking other\nendpoints.", + "operationId": "health", + "responses": { + "200": { + "description": "Server is healthy", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthOutput" + } + } + } + } + } + } } }, "components": { @@ -1813,6 +2089,13 @@ ], "description": "Branch to run against. Defaults to `main`; for a stored mutation the\nwrite targets this branch." }, + "expect_mutation": { + "type": [ + "boolean", + "null" + ], + "description": "The kind the caller expects (RFC-011 Decision 3): `Some(false)` for\n`omnigraph query `, `Some(true)` for `omnigraph mutate `.\nWhen set and it disagrees with the stored query's actual kind, the\nserver rejects the call (400) so the verb asserts the kind. `None`\n(the default) skips the check — preserving older clients and aliases." + }, "params": { "description": "JSON object whose keys match the stored query's declared parameters." }, diff --git a/scripts/local-rustfs-bootstrap.sh b/scripts/local-rustfs-bootstrap.sh deleted file mode 100755 index 2425c77..0000000 --- a/scripts/local-rustfs-bootstrap.sh +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REPO_SLUG="${REPO_SLUG:-ModernRelay/omnigraph}" -SOURCE_REF="${SOURCE_REF:-main}" -RELEASE_CHANNEL="${RELEASE_CHANNEL:-edge}" -WORKDIR="${WORKDIR:-$PWD/.omnigraph-rustfs-demo}" -RUSTFS_CONTAINER_NAME="${RUSTFS_CONTAINER_NAME:-omnigraph-rustfs-demo}" -# Pinned to 1.0.0-beta.8 (2026-06-10), matching CI (.github/workflows/ci.yml). -# beta.4+ has a credentials-policy check that refuses to start when the -# access/secret keys are values it considers "default" (rustfsadmin/rustfsadmin -# here); this script passes RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS=true -# below, so overriding RUSTFS_IMAGE to another tag is safe. -RUSTFS_IMAGE="${RUSTFS_IMAGE:-rustfs/rustfs:1.0.0-beta.8}" -RUSTFS_DATA_DIR="${RUSTFS_DATA_DIR:-$WORKDIR/rustfs-data}" -BUCKET="${BUCKET:-omnigraph-local}" -PREFIX="${PREFIX:-repos/context}" -BIND="${BIND:-127.0.0.1:8080}" -AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-rustfsadmin}" -AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-rustfsadmin}" -AWS_REGION="${AWS_REGION:-us-east-1}" -AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:9000}" -AWS_ENDPOINT_URL_S3="${AWS_ENDPOINT_URL_S3:-$AWS_ENDPOINT_URL}" -AWS_ALLOW_HTTP="${AWS_ALLOW_HTTP:-true}" -AWS_S3_FORCE_PATH_STYLE="${AWS_S3_FORCE_PATH_STYLE:-true}" -FORCE_BUILD="${FORCE_BUILD:-0}" -RESET_REPO="${RESET_REPO:-0}" - -REPO_URI="s3://$BUCKET/$PREFIX" -SERVER_LOG="$WORKDIR/omnigraph-server.log" -SERVER_PID_FILE="$WORKDIR/omnigraph-server.pid" -BIN_DIR="" -FIXTURE_DIR="" -AWS_BIN="" - -log() { - printf '==> %s\n' "$*" -} - -die() { - printf 'error: %s\n' "$*" >&2 - exit 1 -} - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" -} - -repo_root_from_shell() { - if [ -f "$PWD/Cargo.toml" ] && [ -f "$PWD/crates/omnigraph/tests/fixtures/context.pg" ]; then - printf '%s\n' "$PWD" - return 0 - fi - - if [ -n "${BASH_SOURCE[0]:-}" ] && [ -f "${BASH_SOURCE[0]}" ]; then - local candidate - candidate="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - if [ -f "$candidate/Cargo.toml" ] && [ -f "$candidate/crates/omnigraph/tests/fixtures/context.pg" ]; then - printf '%s\n' "$candidate" - return 0 - fi - fi - - return 1 -} - -latest_release_tag() { - local json - json="$(curl -fsSL "https://api.github.com/repos/$REPO_SLUG/releases/latest" 2>/dev/null || true)" - printf '%s' "$json" | sed -n 's/.*"tag_name":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1 -} - -platform_asset_name() { - local os arch - os="$(uname -s)" - arch="$(uname -m)" - - case "$os/$arch" in - Linux/x86_64) - printf 'omnigraph-linux-x86_64.tar.gz\n' - ;; - Darwin/arm64) - printf 'omnigraph-macos-arm64.tar.gz\n' - ;; - *) - return 1 - ;; - esac -} - -checksum_command() { - if command -v shasum >/dev/null 2>&1; then - printf 'shasum -a 256' - return - fi - - if command -v sha256sum >/dev/null 2>&1; then - printf 'sha256sum' - return - fi - - die "missing checksum tool: expected shasum or sha256sum" -} - -release_base_url() { - case "$RELEASE_CHANNEL" in - stable) - printf 'https://github.com/%s/releases/latest/download\n' "$REPO_SLUG" - ;; - edge) - printf 'https://github.com/%s/releases/download/edge\n' "$REPO_SLUG" - ;; - *) - die "unsupported RELEASE_CHANNEL '$RELEASE_CHANNEL' (expected stable or edge)" - ;; - esac -} - -verify_checksum() { - local archive="$1" - local checksum_file="$2" - local expected actual tool - - expected="$(awk '{print $1}' "$checksum_file")" - [ -n "$expected" ] || die "checksum file did not contain a SHA256 digest" - - tool="$(checksum_command)" - actual="$($tool "$archive" | awk '{print $1}')" - - [ "$actual" = "$expected" ] || die "checksum verification failed for $(basename "$archive")" -} - -ensure_aws_cli() { - if command -v aws >/dev/null 2>&1; then - AWS_BIN="$(command -v aws)" - return - fi - - need_cmd python3 - - if ! python3 -m pip --version >/dev/null 2>&1; then - python3 -m ensurepip --upgrade --user >/dev/null 2>&1 || die "aws cli not found and python3 pip bootstrap failed" - fi - - log "Installing a user-local AWS CLI" - python3 -m pip install --user awscli >/dev/null - export PATH="$HOME/.local/bin:$PATH" - - command -v aws >/dev/null 2>&1 || die "aws cli installation succeeded but aws was not found on PATH" - AWS_BIN="$(command -v aws)" -} - -download_fixture_files() { - local ref="$1" - local fixture_target="$WORKDIR/fixtures" - mkdir -p "$fixture_target" - - for file in context.pg context.jsonl; do - curl -fsSL \ - "https://raw.githubusercontent.com/$REPO_SLUG/$ref/crates/omnigraph/tests/fixtures/$file" \ - -o "$fixture_target/$file" || return 1 - done - - FIXTURE_DIR="$fixture_target" -} - -download_release_binaries() { - local asset asset_stem archive_dir archive_path checksum_path base_url - - [ "$FORCE_BUILD" = "1" ] && return 1 - - asset="$(platform_asset_name)" || return 1 - asset_stem="${asset%.tar.gz}" - archive_dir="$WORKDIR/release" - archive_path="$archive_dir/$asset" - checksum_path="$archive_dir/$asset_stem.sha256" - mkdir -p "$archive_dir" "$WORKDIR/bin" - base_url="$(release_base_url)" - - log "Downloading release asset $asset" - curl -fsSL \ - "$base_url/$asset" \ - -o "$archive_path" || return 1 - curl -fsSL \ - "$base_url/$asset_stem.sha256" \ - -o "$checksum_path" || return 1 - verify_checksum "$archive_path" "$checksum_path" || return 1 - tar -C "$WORKDIR/bin" -xzf "$archive_path" || return 1 - - BIN_DIR="$WORKDIR/bin" - if [ "$RELEASE_CHANNEL" = "stable" ]; then - local tag - tag="$(latest_release_tag)" - [ -n "$tag" ] || return 1 - download_fixture_files "$tag" || return 1 - else - download_fixture_files "main" || return 1 - fi -} - -build_from_source() { - local repo_root - repo_root="${1:-}" - - if [ -z "$repo_root" ]; then - need_cmd git - need_cmd cargo - - repo_root="$WORKDIR/source" - if [ ! -d "$repo_root/.git" ]; then - log "Cloning $REPO_SLUG at $SOURCE_REF" - git clone --depth 1 --branch "$SOURCE_REF" "https://github.com/$REPO_SLUG.git" "$repo_root" - fi - fi - - need_cmd cargo - log "Building omnigraph binaries from source" - ( - cd "$repo_root" - cargo build --release --locked -p omnigraph-cli -p omnigraph-server - ) - - BIN_DIR="$repo_root/target/release" - FIXTURE_DIR="$repo_root/crates/omnigraph/tests/fixtures" -} - -setup_binaries() { - local repo_root - repo_root="$(repo_root_from_shell || true)" - - if [ -n "${OMNIGRAPH_BIN_DIR:-}" ]; then - BIN_DIR="$OMNIGRAPH_BIN_DIR" - if [ -n "${OMNIGRAPH_FIXTURE_DIR:-}" ]; then - FIXTURE_DIR="$OMNIGRAPH_FIXTURE_DIR" - elif [ -n "$repo_root" ]; then - FIXTURE_DIR="$repo_root/crates/omnigraph/tests/fixtures" - fi - elif ! download_release_binaries; then - if [ -n "$repo_root" ]; then - build_from_source "$repo_root" - else - build_from_source - fi - fi - - [ -x "$BIN_DIR/omnigraph" ] || die "omnigraph binary not found in $BIN_DIR" - [ -x "$BIN_DIR/omnigraph-server" ] || die "omnigraph-server binary not found in $BIN_DIR" - [ -f "$FIXTURE_DIR/context.pg" ] || die "context fixture schema not found in $FIXTURE_DIR" - [ -f "$FIXTURE_DIR/context.jsonl" ] || die "context fixture data not found in $FIXTURE_DIR" -} - -start_rustfs() { - mkdir -p "$RUSTFS_DATA_DIR" - - if docker ps --format '{{.Names}}' | grep -qx "$RUSTFS_CONTAINER_NAME"; then - log "Reusing existing RustFS container $RUSTFS_CONTAINER_NAME" - return - fi - - if docker ps -a --format '{{.Names}}' | grep -qx "$RUSTFS_CONTAINER_NAME"; then - log "Removing stopped RustFS container $RUSTFS_CONTAINER_NAME" - docker rm -f "$RUSTFS_CONTAINER_NAME" >/dev/null - fi - - log "Starting RustFS on $AWS_ENDPOINT_URL_S3" - docker run -d \ - --name "$RUSTFS_CONTAINER_NAME" \ - -p 9000:9000 \ - -p 9001:9001 \ - -v "$RUSTFS_DATA_DIR:/data" \ - -e RUSTFS_ACCESS_KEY="$AWS_ACCESS_KEY_ID" \ - -e RUSTFS_SECRET_KEY="$AWS_SECRET_ACCESS_KEY" \ - -e RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS=true \ - "$RUSTFS_IMAGE" \ - /data >/dev/null -} - -wait_for_rustfs() { - local attempt - for attempt in $(seq 1 30); do - if "$AWS_BIN" --endpoint-url "$AWS_ENDPOINT_URL_S3" s3api list-buckets >/dev/null 2>&1; then - return - fi - sleep 2 - done - - docker logs "$RUSTFS_CONTAINER_NAME" || true - die "RustFS did not become ready" -} - -ensure_bucket() { - log "Ensuring bucket $BUCKET exists" - "$AWS_BIN" --endpoint-url "$AWS_ENDPOINT_URL_S3" \ - s3api create-bucket --bucket "$BUCKET" >/dev/null 2>&1 || true -} - -graph_prefix_has_objects() { - local key_count - key_count="$("$AWS_BIN" --endpoint-url "$AWS_ENDPOINT_URL_S3" \ - s3api list-objects-v2 \ - --bucket "$BUCKET" \ - --prefix "$PREFIX/" \ - --max-keys 1 \ - --query 'KeyCount' \ - --output text 2>/dev/null || true)" - - [ -n "$key_count" ] && [ "$key_count" != "None" ] && [ "$key_count" != "0" ] -} - -reset_graph_prefix() { - log "Removing existing objects under $REPO_URI" - "$AWS_BIN" --endpoint-url "$AWS_ENDPOINT_URL_S3" \ - s3 rm "s3://$BUCKET/$PREFIX" --recursive >/dev/null -} - -initialize_graph() { - if "$BIN_DIR/omnigraph" snapshot "$REPO_URI" --json >/dev/null 2>&1; then - log "Reusing existing graph at $REPO_URI" - return - fi - - if graph_prefix_has_objects; then - if [ "$RESET_REPO" = "1" ]; then - reset_graph_prefix - else - die "found existing objects under $REPO_URI but could not open an Omnigraph graph there. This usually means a previous bootstrap left a partially initialized prefix. Rerun with RESET_REPO=1 to delete that prefix and recreate it, or set PREFIX to a new value." - fi - fi - - log "Initializing graph at $REPO_URI" - "$BIN_DIR/omnigraph" init --schema "$FIXTURE_DIR/context.pg" "$REPO_URI" - - log "Loading context fixture into $REPO_URI" - "$BIN_DIR/omnigraph" load --data "$FIXTURE_DIR/context.jsonl" "$REPO_URI" -} - -start_server() { - mkdir -p "$WORKDIR" - - if [ -f "$SERVER_PID_FILE" ] && kill -0 "$(cat "$SERVER_PID_FILE")" >/dev/null 2>&1; then - log "Stopping existing server process $(cat "$SERVER_PID_FILE")" - kill "$(cat "$SERVER_PID_FILE")" >/dev/null 2>&1 || true - sleep 1 - fi - - log "Starting omnigraph-server on $BIND" - nohup "$BIN_DIR/omnigraph-server" "$REPO_URI" --bind "$BIND" >"$SERVER_LOG" 2>&1 & - echo "$!" > "$SERVER_PID_FILE" -} - -wait_for_server() { - local bind_host bind_port health_host base_url - bind_host="${BIND%:*}" - bind_port="${BIND##*:}" - health_host="$bind_host" - if [ "$health_host" = "0.0.0.0" ]; then - health_host="127.0.0.1" - fi - base_url="http://$health_host:$bind_port" - - for _ in $(seq 1 30); do - if curl -fsSL "$base_url/healthz" >/dev/null 2>&1; then - printf '%s\n' "$base_url" - return - fi - sleep 1 - done - - cat "$SERVER_LOG" >&2 || true - die "omnigraph-server did not pass /healthz" -} - -print_summary() { - local base_url="$1" - - cat </dev/null 2>&1 || die "docker is installed but the daemon is not reachable; start Docker Desktop or another daemon and rerun" - - export AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY - export AWS_REGION - export AWS_ENDPOINT_URL - export AWS_ENDPOINT_URL_S3 - export AWS_ALLOW_HTTP - export AWS_S3_FORCE_PATH_STYLE - - mkdir -p "$WORKDIR" - - setup_binaries - ensure_aws_cli - start_rustfs - wait_for_rustfs - ensure_bucket - initialize_graph - start_server - print_summary "$(wait_for_server)" -} - -main "$@" diff --git a/skills/omnigraph/SKILL.md b/skills/omnigraph/SKILL.md new file mode 100644 index 0000000..7bf044a --- /dev/null +++ b/skills/omnigraph/SKILL.md @@ -0,0 +1,414 @@ +--- +name: omnigraph +description: Store, retrieve, and query knowledge, memory, and relationships in an Omnigraph graph, and operate a local or remote Omnigraph deployment. Use when the user wants to capture or recall facts, notes, or entities, build or query a knowledge graph or agent memory, or run Omnigraph — and whenever you see Omnigraph CLI commands (omnigraph init/query/mutate/load/schema/lint/embed/branch/commit/login/profile/cluster), .pg schema or .gq query files, s3:// graph URIs, bearer-authed graph endpoints, 504 errors, or a cluster.yaml / omnigraph.yaml / ~/.omnigraph/config.yaml. Covers cluster-mode deployments (cluster.yaml plan/apply, omnigraph-server --cluster), the two config surfaces (cluster.yaml + ~/.omnigraph/config.yaml), schema evolution, query linting, data writes (mutate; load needs --mode/--from), branches, embeddings, Cedar policy, and remote ops. Especially important before schema apply (plan first), any load (--mode required), any .gq/.pg edit (lint after), or any remote write (verify via commit list). +license: MIT (see LICENSE at repo root) +compatibility: Requires omnigraph CLI >= 0.7.0 — the unified `load`, the two config surfaces (cluster.yaml + ~/.omnigraph/config.yaml), and cluster apply/serve all require 0.7.0. +metadata: + author: ModernRelay + version: "0.7.0" + repository: https://github.com/ModernRelay/omnigraph +--- + +# Operating Omnigraph Locally + +This skill captures the operational rules for working with a locally or remotely deployed Omnigraph. Follow them when authoring schema, writing queries, loading data, evolving schema, or automating graph operations. + +## The Seven Rules + +1. **Lint before commit** — `omnigraph lint --schema schema.pg --query queries/foo.gq` validates both sides against each other. No running repo required. +2. **Plan before apply** — never run `schema apply` without a successful `schema plan` first. Apply is destructive; plan is free. (Cluster mode has the same rule with different verbs: `cluster plan` before `cluster apply` — the plan embeds the engine's real migration steps.) +3. **Branches are for data; apply is for schema** — review bulk data loads on a feature branch then merge. Schema changes go straight to `main`: in cluster mode edit the `.pg` and run `cluster apply` (a direct `schema apply` **refuses** a cluster-managed graph); `schema plan`/`apply` is for a non-cluster store. +4. **Pick the right write command** — `mutate` for edits (typechecked, parameterized); `load` for bulk JSONL, local **or** remote, with a **required** `--mode` (`merge` upsert · `append` strict-insert · `overwrite` clean-slate). `load --from ` forks a review branch in one shot; bare `load` needs an existing target branch. +5. **Parameterize everything** — never string-interpolate values into `.gq` bodies or `--params`. Declare `$var: Type` and pass via `--params`. +6. **Expose agent operations as aliases** — not raw CLI invocations. Aliases decouple the operation name from the query implementation. +7. **Verify after every remote write** — compare `commit list --branch main` head before and after. The CLI's exit code is not authoritative on remote graphs; proxies can drop the response while the write commits server-side. See `references/remote-ops.md` for the verification ritual and how to recover from 504s. + +## Essentials: Queries, Mutations, Loads + +The patterns below cover the daily 80% — enough to write correct `.gq` and JSONL without leaving this file. The long tail (multi-hop, negation, aggregations, hybrid search, every decorator) is in [`references/queries.md`](references/queries.md) and [`references/schema.md`](references/schema.md). + +**Comments in `.pg` and `.gq` are `//`, never `#`** (the #1 parse error). + +### Read query (`.gq`) + +```gq +query get_signal($slug: String) { + match { + $s: Signal { slug: $slug } // inline property filter goes in the match block + $s formsPattern $p // edge FormsPattern declared PascalCase, traversed lowerCamelCase + } + return { $s.slug, $s.name, $p.slug } +} +``` + +- **Parameterize, never interpolate.** Declare `$var: Type` in the signature; pass via `--params '{"slug":"sig-foo"}'`. An empty signature still needs parens: `query foo() { ... }`. +- **Edge traversal is lowerCamelCase** even though the schema declares edges PascalCase (`FormsPattern` → `formsPattern`). +- **List/sort** by appending `order { $s.stagingTimestamp desc } limit 50` after `return`. +- **Ranking ops (`nearest`/`bm25`/`rrf`) require a trailing `limit N`** — omitting it is a compile error. They live in `order { }`, not as filters. Scope with `match`/filters first, then rank (`order { nearest($d.embedding, $q) } limit 10`). + +### Mutation (`.gq`) + +There is **no top-level `mutation { }`** — every block is a named `query`; the verb (`insert`/`update`/`delete`) makes it a write. Dispatch with `omnigraph mutate` (not `query`). + +```gq +query add_signal($slug: String, $name: String, $brief: String, $createdAt: DateTime) { + insert Signal { slug: $slug, name: $name, brief: $brief, + stagingTimestamp: $createdAt, createdAt: $createdAt, updatedAt: $createdAt } +} +query link($from: String, $to: String) { insert FormsPattern { from: $from, to: $to } } +query retitle($slug: String, $t: String) { update Signal set { name: $t } where slug = $slug } +query remove($slug: String) { delete Signal where slug = $slug } +``` + +- **Every non-nullable property must be supplied** or lint fails (`T12: insert for 'Signal' must provide non-nullable property 'X'`). +- A single mutation is insert/update-only **or** delete-only — never both (parse-time D₂ rule); split them. +- Edges have no `@key`: give `from`/`to` slugs; the property block is `{}` when the edge has none. + +### Bulk load (JSONL) + +```jsonl +{"type":"Signal","data":{"slug":"sig-foo","name":"Foo","brief":"…","stagingTimestamp":"2026-04-14T00:00:00Z","createdAt":"2026-04-14T00:00:00Z","updatedAt":"2026-04-14T00:00:00Z"}} +{"edge":"FormsPattern","from":"sig-foo","to":"pat-bar","data":{}} +``` + +```bash +omnigraph load --data seed.jsonl --mode merge $GRAPH # --mode is REQUIRED (no default) +omnigraph load --data delta.jsonl --from main --branch review --mode merge $GRAPH # fork a review branch in one shot +``` + +- `--mode`: `merge` (upsert by `@key`) · `append` (fails on collision) · `overwrite` (destructive, staged). `--from ` forks a missing `--branch`; bare `load` needs an existing branch. Works local **and** remote. +- **Date footgun**: `mutate --params` takes ISO strings (`Date` `"2026-04-29"`, `DateTime` `"…T00:00:00Z"`); `load` JSONL takes **integer days since epoch** for `Date` (`20572`) but ISO for `DateTime`. + +### Dispatching + +```bash +omnigraph alias signal sig-foo # operator alias → its bound stored query (read or write) +omnigraph query get_signal --params '{"slug":"sig-foo"}' # served stored query by name (verb asserts read vs write) +omnigraph query -e 'query q() { match { $s: Signal } return { $s.slug } limit 5 }' # ad-hoc/inline (or: --query f.gq ) +omnigraph mutate add_signal --query mutations.gq --params '{"slug":"sig-foo", ...}' # name positional; ad-hoc file source +omnigraph lint --schema schema.pg --query queries/foo.gq # after EVERY .gq/.pg edit (no server needed) +``` + +### `.gq` grammar + +The non-obvious facts that bite, then the full grammar: + +- **Scalar param types**: `String Bool I32 I64 U32 U64 F32 F64 DateTime Date Blob`. Modifiers: `T?` (optional), `[T]` (list), `Vector(N)`. There is **no `Int`** — use `I64`. +- **A read query needs `match` *and* `return`** (`order`/`limit` optional); a mutation has neither — only `insert`/`update`/`delete`. +- **`limit` takes an integer literal, not a param** — `limit 50`, never `limit $n`. +- **Variable-hop traversal**: `$p knows{1,3} $f` (`{1,}` = unbounded). +- **Literals & calls**: `now()`, `date("2026-04-29")`, `datetime("…T00:00:00Z")`, list `[…]`. +- **Filters** `= != > < >= <= contains`; **aggregates** `count/sum/avg/min/max` (`count($f) as n`). +- **Stored-query metadata**: `@description("…")` / `@instruction("…")` may follow the param list. +- **Casing**: type names uppercase-initial (`Signal`); idents/edges lowercase-initial (`formsPattern`); variables `$`-prefixed. `//` and `/* */` comments only. + +Authoritative PEG grammar (pest) for `.gq` files ("NanoGraph" is the legacy engine name): + +```pest +// NanoGraph Query Grammar (.gq files) + +WHITESPACE = _{ " " | "\t" | "\r" | "\n" } +COMMENT = _{ LINE_COMMENT | BLOCK_COMMENT } +LINE_COMMENT = _{ "//" ~ (!"\n" ~ ANY)* } +BLOCK_COMMENT = _{ "/*" ~ (!"*/" ~ ANY)* ~ "*/" } + +query_file = { SOI ~ query_decl* ~ EOI } + +query_decl = { + "query" ~ ident ~ "(" ~ param_list? ~ ")" ~ query_annotation* ~ "{" + ~ query_body + ~ "}" +} +query_annotation = { description_annotation | instruction_annotation } +description_annotation = { "@description" ~ "(" ~ string_lit ~ ")" } +instruction_annotation = { "@instruction" ~ "(" ~ string_lit ~ ")" } + +query_body = { read_query_body | mutation_body } +mutation_body = { mutation_stmt+ } +read_query_body = { + match_clause + ~ return_clause + ~ order_clause? + ~ limit_clause? +} + +mutation_stmt = { insert_stmt | update_stmt | delete_stmt } +insert_stmt = { "insert" ~ type_name ~ "{" ~ mutation_assignment+ ~ "}" } +update_stmt = { "update" ~ type_name ~ "set" ~ "{" ~ mutation_assignment+ ~ "}" ~ "where" ~ mutation_predicate } +delete_stmt = { "delete" ~ type_name ~ "where" ~ mutation_predicate } +mutation_assignment = { ident ~ ":" ~ match_value ~ ","? } +mutation_predicate = { ident ~ comp_op ~ match_value } + +param_list = { param ~ ("," ~ param)* } +param = { variable ~ ":" ~ type_ref } + +type_ref = { (list_type | base_type | vector_type) ~ "?"? } +list_type = { "[" ~ base_type ~ "]" } +vector_type = { "Vector" ~ "(" ~ integer ~ ")" } +base_type = { "String" | "Blob" | "Bool" | "I32" | "I64" | "U32" | "U64" | "F32" | "F64" | "DateTime" | "Date" } + +match_clause = { "match" ~ "{" ~ clause+ ~ "}" } + +clause = { negation | binding | traversal | filter | text_search_clause } +text_search_clause = { search_call | fuzzy_call | match_text_call } + +// Binding: $p: Person { name: "Alice" } +binding = { variable ~ ":" ~ type_name ~ ("{" ~ prop_match_list ~ "}")? } + +prop_match_list = { prop_match ~ ("," ~ prop_match)* ~ ","? } +prop_match = { ident ~ ":" ~ match_value } +match_value = { literal | variable | now_call } + +// Traversal: $p knows $f +traversal = { variable ~ edge_ident ~ traversal_bounds? ~ variable } +traversal_bounds = { "{" ~ integer ~ "," ~ integer? ~ "}" } + +// Filter: $f.age > 25 +filter = { expr ~ filter_op ~ expr } + +// Negation: not { ... } +negation = { "not" ~ "{" ~ clause+ ~ "}" } + +// Return clause — projections separated by commas or newlines +return_clause = { "return" ~ "{" ~ projection+ ~ "}" } +projection = { expr ~ ("as" ~ ident)? ~ ","? } + +// Order clause +order_clause = { "order" ~ "{" ~ ordering ~ ("," ~ ordering)* ~ "}" } +ordering = { nearest_ordering | (expr ~ order_dir?) } +nearest_ordering = { "nearest" ~ "(" ~ prop_access ~ "," ~ expr ~ ")" } +order_dir = { "asc" | "desc" } + +// Limit clause +limit_clause = { "limit" ~ integer } + +// Expressions +expr = { now_call | nearest_ordering | search_call | fuzzy_call | match_text_call | bm25_call | rrf_call | agg_call | prop_access | variable | literal | ident } +now_call = { "now" ~ "(" ~ ")" } +search_call = { "search" ~ "(" ~ expr ~ "," ~ expr ~ ")" } +fuzzy_call = { "fuzzy" ~ "(" ~ expr ~ "," ~ expr ~ ("," ~ expr)? ~ ")" } +match_text_call = { "match_text" ~ "(" ~ expr ~ "," ~ expr ~ ")" } +bm25_call = { "bm25" ~ "(" ~ expr ~ "," ~ expr ~ ")" } +rank_expr = { nearest_ordering | bm25_call } +rrf_call = { "rrf" ~ "(" ~ rank_expr ~ "," ~ rank_expr ~ ("," ~ expr)? ~ ")" } + +prop_access = { variable ~ "." ~ ident } + +agg_call = { agg_func ~ "(" ~ expr ~ ")" } +agg_func = { "count" | "sum" | "avg" | "min" | "max" } + +comp_op = { ">=" | "<=" | "!=" | ">" | "<" | "=" } +filter_op = { "contains" | comp_op } + +// Terminals +variable = @{ "$" ~ (ident_chars | "_") } +ident_chars = @{ (ASCII_ALPHA_LOWER | "_") ~ (ASCII_ALPHANUMERIC | "_")* } + +// Edge identifier — lowercase start, same as ident but used in traversal context +// Must not match keywords +edge_ident = @{ !("not" ~ !ASCII_ALPHANUMERIC) ~ (ASCII_ALPHA_LOWER | "_") ~ (ASCII_ALPHANUMERIC | "_")* } + +type_name = @{ ASCII_ALPHA_UPPER ~ (ASCII_ALPHANUMERIC | "_")* } +ident = @{ (ASCII_ALPHA_LOWER | "_") ~ (ASCII_ALPHANUMERIC | "_")* } + +literal = { list_lit | datetime_lit | date_lit | string_lit | float_lit | integer | bool_lit } +date_lit = { "date" ~ "(" ~ string_lit ~ ")" } +datetime_lit = { "datetime" ~ "(" ~ string_lit ~ ")" } +list_lit = { "[" ~ (literal ~ ("," ~ literal)*)? ~ "]" } +string_lit = @{ "\"" ~ string_char* ~ "\"" } +string_char = @{ !("\"" | "\\") ~ ANY | "\\" ~ ANY } +float_lit = @{ ASCII_DIGIT+ ~ "." ~ ASCII_DIGIT+ } +integer = @{ ASCII_DIGIT+ } +bool_lit = { "true" | "false" } +``` + +## CLI Reference (condensed) + +Notation: `` required · `[x]` optional · `` choice · `…` repeatable. + +**Global addressing flags**: `--as ` (direct/`--store` writes only — a server resolves the actor from its token), `--server `, `--cluster ` (cluster-managed storage, for maintenance), `--graph ` (selects the graph within a `--server` or `--cluster` scope), `--profile ` (`$OMNIGRAPH_PROFILE`), `--store `. Data commands also take a positional `file://`/`s3://` URI (`--config ` is for `cluster` commands only). Output: `--json`, or reads take `--format `. **Write guards:** `--yes` skips the confirm prompt for a destructive write (`cleanup`, overwrite `load`, `branch delete`) against a non-local scope (it *refuses* without it when non-TTY or `--json`); `--quiet` suppresses the resolved-target echo. + +**Data plane** — `any` (served via `--server`/`--profile`, or direct via `--store`/URI): +- `query` (alias `read`) `` — a **served stored query** by name (via `--server`/`--profile`); or ad-hoc `[] (--query | -e '')` where `` picks which query in the source. `[--params | --params-file

] [--branch | --snapshot ] [--format | --json]`. No positional URI — address via `--server`/`--store`/`--profile`. +- `mutate` (alias `change`) — same shape (served stored mutation by ``, or ad-hoc `--query`/`-e`); `[--params …] [--branch ] [--json]`. The verb asserts kind: `query`→read, `mutate`→write (400 on mismatch). +- `alias [args…]` — invoke an operator alias's bound stored query (read or write); `[--params … | --params-file

] [--format | --json]` (server/graph/query come from the binding) +- `load --data --mode [--branch ] [--from ] [--json]` — `--mode` required; `--from` forks a missing `--branch` +- `snapshot [--branch ] [--json]` +- `export [--branch ] [--type …] [--table …]` (streams JSONL) +- `branch [--from ] | list | delete | merge --into > [--json]` +- `commit ] | show > [--json]` +- `schema --schema [--allow-data-loss] [--json]` · `schema show` (alias `get`) — `apply` **refuses a cluster-managed graph** (evolve those via `cluster apply`) + +**Served only** (needs `--server`/`--profile`): `graphs list [--json]` + +**Direct / storage** — reject `--server`; address by positional URI or `--cluster --graph `: +- `init --schema [--force]` +- `lint --query [--schema ] [] [--json]` — offline with `--schema`, graph-backed with a URI +- `optimize [--json]` · `repair [--confirm] [--force] [--json]` · `cleanup (--keep | --older-than <7d>) --confirm [--json]` +- `queries ] | list> [--json]` + +**Control plane** — cluster (`--config

`, default `.`): +- `cluster [--config ] [--json]` +- `cluster approve --as [--config ] [--json]` · `cluster force-unlock [--config ] [--json]` + +**Local** (no graph): +- `policy | explain --actor --action [--branch | --target-branch ]> --cluster [--graph ]` +- `embed --seed [--reembed_all | --clean | --select ":="]` +- `login [--token ]` (prefer piping the token on stdin) · `logout ` · `profile ]>` · `version` + +Pre-0.7.0 spellings (`read`/`change`/`ingest`, `--target`, positional `http://`) → [`references/migrations.md`](references/migrations.md). + +## Five Ontology Design Criteria (Gruber 1993) + +Omnigraph schemas are ontologies. The canonical design criteria from Gruber's *Toward Principles for the Design of Ontologies Used for Knowledge Sharing* (Int. J. Human-Computer Studies 43:907–928) apply directly when authoring `.pg` files. + +1. **Clarity** — definitions should communicate intended meaning unambiguously and be independent of social or computational context. In Omnigraph: precise type names, narrow enums over `String`, `@check`/`@range` for stated invariants. A reviewer should understand the domain from the schema alone. +2. **Coherence** — inferences sanctioned by the schema must be consistent with the domain modeled. Gruber's trap: defining quantity as a `(magnitude, unit)` pair makes `6 feet ≠ 2 yards` even though they describe the same length. In Omnigraph: watch for `@card`, `@unique`, and edge directionality that let the schema distinguish things the domain treats as equal. +3. **Extendibility** — the schema should support specialization without revising existing definitions. In Omnigraph: prefer interfaces for shared shape, leave enums open where the domain genuinely admits more, model identifiers via mapping functions rather than baking units/formats into the entity. +4. **Minimal encoding bias** — representation choices made for notation or implementation convenience leak into the model. In Omnigraph: don't type dates as `String` because the source API returns strings; separate conceptual entities (a publication date, a person) from their surface encoding (a year integer, a name string) when both matter. +5. **Minimal ontological commitment** — make as few claims about the world as the use case requires. In Omnigraph: don't add required properties, closed enums, or `@card(1..1)` "in case"; tighten later via `schema plan`/`apply` when a real constraint emerges. Weaker schemas leave consumers room to specialize. + +The criteria trade off against each other — Clarity wants tight definitions while Minimal Commitment wants weak ones. Gruber's resolution: *having decided a distinction is worth making, give it the tightest possible definition*. Decide what to model conservatively; once modeled, constrain precisely. + +## Schema Authoring Principles + +Twelve practical rules for `.pg` authoring — full text and examples in [`docs/omni-schema.md`](../../docs/omni-schema.md). In short: schema-is-the-contract · explicit identity via `@key` · model meaning not tables · strong intentional types · deliberate optionality · shared shape in interfaces · schema-level constraints (`@unique`/`@index`/`@range`/`@check`/`@card`) · search as a schema decision · edge semantics matter · reviewable schemas · intentional migrations (`@rename_from`) · domain clarity over ORM habits. + +Design flow: entities → stable keys → relationships worth their own edge → enum candidates → uniqueness/bounds/cardinality → search needs → shared shape into interfaces → evolution plan. + +## Provenance Is Structural (Multi-Agent Source of Truth) + +When Omnigraph serves as canonical truth across multiple agents, every assertion must answer *who said it, when, based on what evidence*. This is the runtime guarantee Gruber's criteria don't cover — his agents shared vocabulary; ours additionally must share attribution. Provenance belongs in the schema, not in logs. + +Without structural provenance, agents cannot reconcile contradictory assertions, retract facts when a source is discredited, replay graph state at a past timestamp, or distinguish high-evidence facts from speculation. + +**In Omnigraph:** model provenance as a `Claim`-style interface (or a separate `Claim` node linked to each sourced fact) with required fields — `asserted_by: Actor`, `asserted_at: DateTime`, `evidence_source: Source`, optionally `confidence: F64`. Don't stash provenance into a free-text `source: String` or a `metadata: JSON` dump — structured provenance is queryable, indexable, and migratable; free-form is none of these. + +## Storage & Credentials + +A graph's bytes live in one of two backends: + +- **Local filesystem** — a path or `file://` URI. In cluster mode `storage:` defaults to the config directory, so local dev needs no object store. +- **S3-compatible object storage** — AWS, Railway, Tigris, etc. (`s3://bucket/prefix`). Authenticate with the standard `AWS_*` environment contract; keep dev creds in a git-ignored `.env.omni` and source it before CLI calls: + +```bash +set -a && source .env.omni && set +a +``` + +`init` and `load` write storage directly (bypassing the server); the server reads from it. Validate with `curl http://127.0.0.1:8080/healthz`, then `omnigraph snapshot --json`. + +## Project Layout + +### Deployment & access (omnigraph >= 0.7.0) + +- **Cluster deployment — the only way to serve.** A `cluster.yaml` declares the + whole deployment (graphs, schemas, stored queries, policies, optional S3 + `storage:` root); `omnigraph cluster apply` converges it and + `omnigraph-server --cluster .` (or `--cluster s3://bucket/prefix`, + config-free) serves it. See `references/cluster.md`. +- **Direct / embedded access — no server.** Address a graph's storage directly + with `--store ` or a positional URI for one-off CLI ops. + There is **no single-graph server mode** — the server is cluster-only. + +### The two config surfaces (omnigraph >= 0.7.0) + +Configuration has two single-owner homes (RFC-007/008), plus an +everything-explicit flag/env tier: + +| Surface | Owner | Location | Declares | +|---|---|---|---| +| **Cluster config** | the team, in the repo | `cluster.yaml` + the `.pg`/`.gq`/policy files it references | what the system **is**: graphs, schemas, queries, policies, storage | +| **Operator config** | one person | `~/.omnigraph/config.yaml` (`$OMNIGRAPH_HOME` relocates it) | who **I** am: identity, named servers, output defaults, personal aliases | +| Flags / env | per invocation | — | everything, explicitly | + +```yaml +# ~/.omnigraph/config.yaml — per operator, never committed +operator: + actor: act-andrew # default --as identity +servers: + intel-dev: + url: https://graph.example.com # no tokens here, ever +defaults: + output: table # read-format default + server: intel-dev # default served scope (or `store: file://…/g.omni` for a local default — mutually exclusive) + default_graph: spike # graph within a server/cluster scope +profiles: # optional named scope bundles — pick with --profile + staging: { server: intel-staging, default_graph: spike } +aliases: # personal bindings to TEAM stored queries (see references/aliases.md) + triage: { server: intel-dev, graph: spike, query: weekly_triage, args: [since] } +``` + +The operator config and credentials are **auto-discovered — no flag points at them**: the CLI reads `$OMNIGRAPH_HOME/config.yaml` (default `~/.omnigraph/config.yaml`), and an absent file is just an empty layer (zero-config). `$OMNIGRAPH_HOME` relocates the *directory* only, not a specific file. (`--config`/`$OMNIGRAPH_CONFIG` is a separate flag for the cluster / server config — not this.) + +Credentials live outside config: `echo $TOKEN | omnigraph login intel-dev` +writes `~/.omnigraph/credentials` (`0600`); the matching token resolves via +`OMNIGRAPH_TOKEN_INTEL_DEV` or that file. + +**Addressing a graph**: `--store ` or a positional URI for +direct storage; `--server ` (+ `--graph `) for a served remote; +`--profile ` for a named bundle; else the operator `defaults`. A remote is +addressed with `--server` (a bare `http(s)://` URL is not a graph address). Run +data-plane commands from a graph's project folder so relative `queries/`, +`schema.pg`, and `.env.omni` paths resolve. + +### What to commit + +**Commit:** `schema.pg`, `queries/*.gq`, `cluster.yaml`, `seed.md`, `seed.jsonl`, and the project's `README.md` and `CLAUDE.md`. + +**Ignore:** `.env.omni` (credentials), `.claude/` (local agent state), `*.omni/` (local graph artifacts), `__cluster/` and `graphs/` (cluster state + derived graph roots). + +### Give agents a `CLAUDE.md` + +A per-project `CLAUDE.md` tells coding agents where files live and what conventions matter. Without it, agents re-discover the same things every session. + +## Common Gotchas + +These are the traps most likely to bite. Scan this table before debugging any parse or runtime error. + +| Trap | Symptom | Fix | +|------|---------|-----| +| `#` comments in `.pg` | `parse error: expected schema_file` | Use `//` | +| Standalone `enum Foo { ... }` block | `parse error: expected EOI or schema_decl` | Inline: `kind: enum(a, b)` | +| `[Category]` (list of enum) | compile error | Use `[String]`; lists must contain scalars | +| `@embed(text)` without quotes | `unexpected constraint_name` | `@embed("text")` | +| `@unique(src)` on edge without body block | parse error | `@card(1..1) { @unique(src) }` | +| `load --mode merge` after `@embed` source change | stale embeddings | `omnigraph embed --reembed_all` or `load --mode overwrite` | +| `schema apply` with feature branches open | rejected | Merge or delete branches first | +| `nearest(...)` / `bm25(...)` / `rrf(...)` without `limit` | compile error | Add `limit N` | +| Adding non-nullable property without backfill | unsupported migration | Make optional → backfill → tighten in follow-up apply | +| `omnigraph init --json` | `unexpected argument --json` | `init` doesn't support `--json`; drop the flag | +| `omnigraph init` on an already-initialized URI | `AlreadyInitialized` error (v0.6.0+) | `--force` to re-init (skips the schema preflight; does **not** purge data) | +| `schema apply` dropping a property/type | soft-dropped or rejected (no data loss) | add `--allow-data-loss` to actually drop the column | +| Committing `.env.omni` | credential leak | Add `.env*` to `.gitignore` | +| Non-parameterized query values | typecheck surprise, injection risk | Declare `$param: Type` and pass via `--params` | +| Missing required field in `insert` | `T12: insert for 'X' must provide non-nullable property 'Y'` | Accept the param in the mutation signature | +| Long-lived feature branches | merge conflicts, schema apply blocked | Merge promptly; delete when done | +| `mutation { ... }` wrapper in `.gq` | `parse error: expected query_file` at line 1 | Use `query (...) { insert T { ... } }`; there is no top-level `mutation` keyword | +| `--config` placed before subcommand | `unexpected argument --config` | Put `--config` **after** the subcommand (e.g. `omnigraph schema show --config X`) | +| Reading a large schema via stdout-capped tool | Truncated, garbled, or duplicated output | `omnigraph schema show > /tmp/schema.pg` first; then read the file with offset/limit | +| `omnigraph load` without `--mode` | error: `--mode` is required | Pass `--mode merge\|append\|overwrite` — there is no default (overwrite is destructive, so it is never implicit). `load` works against local and remote URIs | +| Blind retry after 504 | Duplicate Signal/Decision/Claim (append-only types lack `@key` dedup) | `commit list --branch main --json` first; head advanced means it landed; only retry if unchanged | +| `sync_branch()` mentioned in version-drift error | Searching for nonexistent CLI command | Server-internal directive in error text; just retry — the next call re-pins to the new head | +| Stale empty branches at `main`'s head | 504-orphaned forks from a timed-out `load --from`; eventually block writes | List branches, find ones at `main`'s `graph_commit_id`, `omnigraph branch delete --config X ` | +| `omnigraph schema apply` / `init` on a cluster-managed graph | refused — bypasses the cluster ledger | Evolve cluster graphs via `omnigraph cluster apply --config .`; `schema apply`/`init` are for a non-cluster store | +| `omnigraph optimize` against a table with a `Blob` property | table is **skipped**, not failed (Lance blob-v2 compaction bug) | Expected — `--json` reports it under `skipped`; non-blob tables still compact | +| `@unique` on a `[List]`/`Blob` column | `load` now errors loudly (was silently un-enforced before #160) | Use `@unique` only on scalar columns (and composite `@unique(a, b)`, now keyed as a true tuple) — uniqueness needs a type that reduces to a scalar key | + +## Deep Dives + +- `references/cluster.md` — cluster-mode declarative deployments: cluster.yaml, the validate/import/plan/apply loop, approval-gated deletes, `--cluster` serving, the two-file contract, recovery + +For anything beyond the basics, load the relevant reference file. Each is self-contained — load only what you need. + +| Reference | When to load | +|-----------|--------------| +| [`references/schema.md`](references/schema.md) | Editing `.pg` files, running `schema plan`/`apply`, renaming types, backfilling required fields | +| [`references/queries.md`](references/queries.md) | Writing or linting `.gq` files, search functions, aggregations, multi-hop patterns | +| [`references/data.md`](references/data.md) | Choosing between `mutate` and `load` (required `--mode`, `--from` to fork a review branch); branch review workflow; destructive ops | +| [`references/remote-ops.md`](references/remote-ops.md) | Operating against a remote/CloudFront-fronted graph: 504 verification ritual, version drift, fork-branch 504 fingerprints, append-only retry safety, operator `--server`/`login` targeting | +| [`references/search.md`](references/search.md) | Embeddings, `@embed`, vector/text ranking, scope-then-rank pattern | +| [`references/aliases.md`](references/aliases.md) | Defining aliases for agents, structured output, JSON args | +| [`references/stored-queries.md`](references/stored-queries.md) | Server-side stored-query registry: declared in `cluster.yaml`, `omnigraph queries validate/list`, `GET /graphs/{id}/queries` + `POST /graphs/{id}/queries/{name}`, `invoke_query` Cedar gating | +| [`references/server-policy.md`](references/server-policy.md) | Starting the HTTP server, routes, bearer auth, Cedar policy gating, multi-graph mode | +| [`references/commands.md`](references/commands.md) | `snapshot`, `export`, `commit list/show`, addressing & resolution | +| [`references/migrations.md`](references/migrations.md) | Migrating a pre-0.7.0 setup, or you hit an old config/command/flag/route/error and need its current form | diff --git a/skills/omnigraph/references/aliases.md b/skills/omnigraph/references/aliases.md new file mode 100644 index 0000000..85dba93 --- /dev/null +++ b/skills/omnigraph/references/aliases.md @@ -0,0 +1,141 @@ +# Aliases & Agent Automation + +## Contents +- What an alias is +- Operator alias schema +- Args binding & JSON-first parsing +- Default to structured output +- Alias naming convention +- Secrets don't belong in aliases +- Example alias set +- Invocation patterns + +How to wire Omnigraph operations for agents and scripts. + +## What an alias is + +An **operator alias** decouples a stable **operation name** from its implementation, so an agent calling `omnigraph alias signal …` keeps working as the query evolves. Aliases live in `~/.omnigraph/config.yaml` and are personal *bindings* to a **stored query on a named server** — they carry no query content; the stored query in the cluster catalog is the team's contract. + +```yaml +# ~/.omnigraph/config.yaml +aliases: + triage: + server: intel-dev # an entry under servers: + graph: spike # optional (multi-graph servers) + query: weekly_triage # the STORED query's name — never a file + args: [since] # positional args → params, in order + params: { limit: 20 } # fixed defaults; positionals/--params win + format: table +``` + +```bash +omnigraph alias triage 2026-06-01 +# → POST /graphs/spike/queries/weekly_triage with the keyed credential +``` + +> **Alias vs stored query.** The alias is *yours* (a personal name + defaults); the **stored query** it points at is the *team's* — declared in `cluster.yaml`, type-checked and served by the cluster (`GET /graphs//queries`, `POST /graphs//queries/`, gated by `invoke_query`). See [`stored-queries.md`](stored-queries.md). +## Operator Alias Schema + +```yaml +aliases: + : + server: # an entry under servers: in ~/.omnigraph/config.yaml + graph: # optional: for multi-graph servers + query: # the stored query's NAME (never a file path) + args: [, ] # positional CLI args → named params, in order + params: { : } # fixed default params; positionals / --params win + format: table|kv|csv|jsonl|json # optional: output format +``` + +Dispatch with `omnigraph alias [args]` — one subcommand for read **and** write stored queries (a mutation alias is double-gated by `invoke_query` + `change`). Aliases live in their own namespace, so one can never shadow or be shadowed by a built-in verb. + +### `args` bind to query parameters + +If `args: [slug, name, age]`, then: + +```bash +omnigraph alias foo sig-bar "Some Name" 29 +``` + +...maps to `{"slug":"sig-bar","name":"Some Name","age":29}`. + +### Args are JSON-first + +Each arg is parsed as JSON first, then falls back to string: +- `29` → integer +- `"29"` → string +- `true` → boolean +- `Alice` → string (JSON parse fails, falls back) +- `{"x":1}` → object + +Explicit `--params '{...}'` wins on key conflict. + +## Default to Structured Output + +For scripts and agents, prefer `jsonl` or `json`; `table` is for humans. Set a default in `~/.omnigraph/config.yaml`: + +```yaml +defaults: + output: jsonl +``` + +Or per-alias (`format: jsonl`), or per-call (`--format jsonl`). + +### When to use which + +- **`jsonl`** — one JSON object per line, first line is metadata; streams; ideal for agents +- **`json`** — pretty-printed JSON array; smaller results; human-readable +- **`kv`** — `key: value` per line; good for single-row lookups +- **`csv`** — for spreadsheets or line-count-heavy analysis +- **`table`** — default human view; don't use in automation + +## Alias Naming Convention + +Short, hyphenated, matches the conceptual operation: + +- `signal`, `pattern`, `element` — single lookup (typical pair with `format: kv`) +- `signals`, `patterns`, `elements` — list +- `signal-patterns`, `pattern-signals` — traversals +- `add-signal`, `link-forms-pattern` — mutations + +## Secrets Don't Belong in Aliases + +Credentials never live in an alias or any config file. For remote servers, `omnigraph login ` stores the bearer token in `~/.omnigraph/credentials` (`0600`); for S3-backed storage, AWS creds go in `.env.omni`. Aliases should only contain query names and parameter bindings — never tokens, passwords, or API keys. + +## Example Alias Set + +```yaml +# ~/.omnigraph/config.yaml +servers: + intel-dev: { url: https://graph.example.com } +aliases: + # Lookups (kv format for single-row readability) + signal: { server: intel-dev, graph: spike, query: get_signal, args: [slug], format: kv } + pattern: { server: intel-dev, graph: spike, query: get_pattern, args: [slug], format: kv } + # Lists + signals: { server: intel-dev, graph: spike, query: recent_signals } + # Traversals + pattern-signals: { server: intel-dev, graph: spike, query: pattern_signals, args: [slug] } + # Mutations (stored mutation; invoke_query + change) + add-signal: { server: intel-dev, graph: spike, query: add_signal, args: [slug, name, brief, stagingTimestamp, createdAt, updatedAt] } + link-forms-pattern: { server: intel-dev, graph: spike, query: link_signal_forms_pattern, args: [signal, pattern] } +``` + +Each `query:` names a stored query the cluster serves — declare them in `cluster.yaml` and `cluster apply` first (see [`stored-queries.md`](stored-queries.md)). + +## Invocation Patterns + +```bash +# Invoke an alias (read or write — the bound stored query decides) +omnigraph alias signal sig-kimi-k25 +omnigraph alias add-signal sig-new "Name" "Brief" \ + 2026-04-14T00:00:00Z 2026-04-14T00:00:00Z 2026-04-14T00:00:00Z + +# Override output format +omnigraph alias signals --format jsonl + +# Explicit --params (wins over positional args on key conflict) +omnigraph alias signal --params '{"slug":"sig-override"}' +``` + +The `alias` subcommand carries `--params`/`--params-file`, `--format`/`--json`, and `--config`; the server, graph, and stored-query name come from the binding. For a different server/graph or a branch read, call `query`/`mutate` directly. diff --git a/skills/omnigraph/references/cluster.md b/skills/omnigraph/references/cluster.md new file mode 100644 index 0000000..3e9f6e9 --- /dev/null +++ b/skills/omnigraph/references/cluster.md @@ -0,0 +1,128 @@ +# Cluster Mode — Declarative Deployments + +## Contents +- The model +- The loop (validate → import → plan → apply → serve) +- The config contract (`cluster.yaml` vs `~/.omnigraph/config.yaml`) +- Serving (`--cluster`, config-free bucket boot) +- Recovery cheat-sheet + +The cluster control plane (omnigraph >= 0.7.0) manages a whole deployment — +graphs, schemas, stored queries, Cedar policies — as **declared files in one +directory**, converged Terraform-style. It is the **only way to serve** a +graph (the server is cluster-only); the data-plane operations in the other +references work against the cluster's graphs unchanged. + +## The model + +``` +company-brain/ +├── cluster.yaml # the deployment: graphs, schemas, queries, policies +├── schema.pg +├── queries/*.gq +├── *.policy.yaml +├── graphs/.omni # DERIVED — created by apply, never by hand (gitignore) +└── __cluster/ # ledger + catalog + approvals — local state (gitignore) +``` + +```yaml +# cluster.yaml +version: 1 +# storage: s3://my-bucket/clusters/company-brain # optional — put ledger, +# catalog, and graph roots on S3 object storage (default: this folder) +state: { backend: cluster, lock: true } +graphs: + knowledge: + schema: schema.pg + queries: queries/ # the .gq files ARE the declaration — every `query ` registers +policies: + base: { file: base.policy.yaml, applies_to: [knowledge] } # or [cluster] for server-level +``` + +`queries` also accepts a file list (`[a.gq, b.gq]`) or a fine-grained +`name: { file: ... }` map. Discovery is loud: unparseable files and duplicate +names across files fail validation. + +## The loop (memorize this) + +```bash +omnigraph cluster validate --config . # parse + typecheck everything +omnigraph cluster import --config . # one-time: create the state ledger +omnigraph cluster plan --config . # preview — REQUIRED reading before apply +omnigraph cluster apply --config . --as # converge (idempotent) +omnigraph-server --cluster . --bind 127.0.0.1:8080 --unauthenticated # serve (local dev) +``` + +- **`apply` creates graphs** at `graphs/.omni` — there is no separate + `omnigraph init` in cluster mode. +- **Schema changes**: edit the `.pg`, `plan` shows the engine's real migration + steps (`add_property`, `drop_property [soft]`, `unsupported: …`), `apply` + migrates the live graph. **Soft drops only** — data-loss migrations are not + reachable from cluster apply (prior versions retain dropped columns). +- **Applied = serving on the next server restart.** No hot reload. +- **`storage: s3://bucket/prefix`** (optional) puts the entire cluster — state + ledger, lock, content-addressed catalog, recovery sidecars, approval + artifacts, and the derived graph roots (`/graphs/.omni`) — on + S3-compatible object storage. The ledger CAS uses S3 conditional writes and + the lock becomes genuinely cross-machine. Absent, everything defaults to the + config directory (byte-compatible with pre-existing clusters). Credentials + come from the standard `AWS_*` env contract, never `cluster.yaml`. +- **`--as ` attributes every run** (sidecars, audit, engine commits). + Defaults from your operator config's `operator.actor`; required for `approve`. +- **Destructive changes are gated**: removing a graph from `cluster.yaml` + blocks with `approval_required` until + `omnigraph cluster approve graph. --config . --as ` records a + digest-bound approval. Any config/state drift after approving invalidates it. +- **Drift**: `cluster refresh` re-observes live graphs and marks out-of-band + changes `drifted`; the next `apply` converges them back to the declaration. +- **Data is NOT cluster's job**: rows flow through `omnigraph load / mutate` + against the derived roots, with branches as usual. + +## The config contract (do not blur this) + +| File | Owns | Read by | +|---|---|---| +| `cluster.yaml` | the deployment: graph set, schemas, stored queries, policy bindings, storage | `cluster` commands; the `--cluster` server | +| `~/.omnigraph/config.yaml` | per-operator: identity (`operator.actor`), named `servers:`, output defaults, personal aliases | data-plane CLI commands (tokens live in `~/.omnigraph/credentials` via `omnigraph login`) | + +Cluster commands read the operator config for **exactly one thing**: the actor +default when `--as` is omitted (`--as` > `operator.actor`). A `--cluster` server +reads it for **nothing** — boot from cluster state XOR the operator file, never +a merge. +Address a cluster-managed graph's data directly with `--store /graphs/.omni`, +or via `--server`/aliases against a serving instance — that is ergonomics, not +coupling. + +## Serving + +`omnigraph-server --cluster ` is exclusive (cannot combine with a URI, +`--target`, or `--config`), always multi-graph (`/graphs/{id}/...`), and +fail-fast: missing/pending/tampered state refuses boot with a remedy. Every +declared query is exposed (`GET /graphs//queries`, `POST +/graphs//queries/`); Cedar bundles attach via `applies_to` +(`cluster` → server-level gate incl. `graph_list`; `graph.` → that +graph's gate incl. `invoke_query`). Bearer tokens and bind stay process-level +(env/flags). + +**Config-free serving.** `--cluster` also accepts the storage-root URI +directly — `omnigraph-server --cluster s3://bucket/prefix` boots from the +applied revision on the bucket with **no checkout of the config repo**. The +ledger and catalog on the bucket are the whole deployment artifact; policy +bundles serve as digest-verified content from the catalog. The preferred +container shape is **bucket, no volume** (AWS ECS / Railway recipes in the +omnigraph repo's `docs/user/deployment.md`). For a mounted config directory +instead, `OMNIGRAPH_CLUSTER=` works and the image ships the CLI for +in-container `cluster apply`. + +## Recovery cheat-sheet + +| Symptom | Fix | +|---|---| +| Apply crashed mid-run | run `cluster apply` again — sidecars + sweep reconcile | +| Held lock | `cluster status` (shows lock id) → `cluster force-unlock --config .` | +| Lost/corrupt `state.json` | `cluster import` rebuilds from config + live graphs, then `apply` | +| Server refuses to boot | the error names its remedy (usually `cluster refresh` + `apply`, restart) | +| `approval_stale` warning | re-run `cluster approve` — the plan changed since you approved | + +Full reference: the omnigraph repo's `docs/user/clusters/index.md` (operator guide) +and `docs/user/clusters/config.md` (every key, flag, and diagnostic). diff --git a/skills/omnigraph/references/commands.md b/skills/omnigraph/references/commands.md new file mode 100644 index 0000000..a76844b --- /dev/null +++ b/skills/omnigraph/references/commands.md @@ -0,0 +1,237 @@ +# Reference Commands + +## Contents +- Inspect state (snapshot, export) +- Branches · commits · graphs +- Schema · lint · embed · init +- Load (bulk JSONL) +- Query / mutate +- Maintenance (optimize, cleanup) +- Stored queries +- Operator config & credentials +- Config resolution order +- Output formats · health check +- Cluster control plane + +Commands you'll reach for but don't need best-practice rules around. Quick syntax reference. + +## Inspect State + +### `snapshot` — tables + row counts + +```bash +omnigraph snapshot $REPO --branch main --json +``` + +Returns the manifest: all node/edge tables with row counts and versions. Use this to verify a load succeeded or to see what types exist. + +### `export` — full JSONL dump + +```bash +omnigraph export $REPO --branch main > graph.jsonl +``` + +Streams all nodes and edges as JSONL. The right tool for large-snapshot inspection. Don't try to page through the whole graph with read queries. + +Filter by type: + +```bash +omnigraph export $REPO --branch main --type Signal > signals.jsonl +``` + +## Branches + +```bash +omnigraph branch create --from main --store $REPO +omnigraph branch list --store $REPO +omnigraph branch merge --into main --store $REPO +omnigraph branch delete --store $REPO +``` + +All support `--json`. + +## Commits (History) + +```bash +omnigraph commit list $REPO --branch main +omnigraph commit show $REPO +``` + +Inspect graph history. Useful for "what changed between these two points" investigation. + +## Graphs (multi-graph servers) + +```bash +omnigraph graphs list --config X --json +``` + +Lists the graphs a multi-graph server serves. Remote servers only (rejects local URIs); the server must expose `GET /graphs` via `server.policy.file`. See `references/server-policy.md`. + +## Schema + +```bash +omnigraph schema plan --schema next.pg $REPO --json +omnigraph schema apply --schema next.pg $REPO +``` + +See `references/schema.md` for the full workflow. + +## Lint + +```bash +omnigraph lint --schema schema.pg --query queries/foo.gq --json +# or against a live repo: +omnigraph lint --query queries/foo.gq $REPO --json +``` + +`lint` is the single query-validation command. See `references/queries.md`. + +## Embed + +```bash +omnigraph embed --seed embed-config.yaml # fill missing +omnigraph embed --seed embed-config.yaml --reembed_all # regenerate all +omnigraph embed --seed embed-config.yaml --clean # delete +omnigraph embed --seed embed-config.yaml --select "Type:field=value" +``` + +See `references/search.md`. + +## Init + +```bash +omnigraph init --schema schema.pg $REPO +``` + +Creates a new graph at `$REPO` with the given schema. Declare the deployment in a `cluster.yaml` (see `references/cluster.md`). + +**Strict by default (v0.6.0+):** `init` against a URI that already holds schema files errors with `AlreadyInitialized` instead of silently overwriting. Use `omnigraph init --force` to re-init deliberately. `--force` only skips the schema-file preflight — it does **not** purge existing Lance datasets. + +**Note:** `init` does not accept `--json`. Drop the flag if you see `unexpected argument --json`. + +## Load (bulk JSONL) + +```bash +# bare load: operates on an existing branch (default main); --mode is required +omnigraph load --data seed.jsonl --mode merge $REPO + +# --from forks a missing branch from , then loads onto it (one-shot review branch) +omnigraph load --data delta.jsonl --branch feature-x --from main --mode merge $REPO +``` + +`--mode` is **required** (no default): `merge`, `append`, or `overwrite`. `load` works against local **and** remote URIs. See `references/data.md`. + +## Query / Mutate + +```bash +omnigraph query get_signal --query queries/signals.gq --params '{"slug":"sig-foo"}' # ad-hoc file; is positional +omnigraph query get_signal --server intel-dev --params '{"slug":"sig-foo"}' # served stored query by name +omnigraph mutate add_signal --query queries/mutations.gq --params '{"slug":"sig-foo",...}' +``` + +With aliases: + +```bash +omnigraph alias signal sig-foo +omnigraph alias add-signal sig-foo "Name" "Brief" 2026-04-14T00:00:00Z 2026-04-14T00:00:00Z 2026-04-14T00:00:00Z +``` + +> `query` and `mutate` also accept inline source via `-e/--query-string ''` instead of `--query `. + +## Maintenance: Optimize & Cleanup (v0.6.1) + +### `optimize` — non-destructive Lance compaction + +```bash +omnigraph optimize $REPO --json +``` + +Compacts fragments and reclaims deleted-row space. Non-destructive — safe to run any time. **Skips tables with a `Blob` property** (Lance blob-v2 compaction decode bug); skipped tables are reported in the `skipped` field of `--json` output and in logs. Non-blob tables compact normally. Blob-table fragment count won't shrink until the upstream Lance fix lands — reads/writes are unaffected. + +### `cleanup` — destructive version GC + +```bash +omnigraph cleanup $REPO --keep 5 --older-than 7d --confirm +``` + +Garbage-collects old table versions, dropping time-travel reachability for anything pruned. **Destructive** — requires `--confirm`. Duration units for `--older-than`: `s`, `m`, `h`, `d`, `w`. Also reconciles orphaned per-table forks left by an interrupted `branch delete`. + +## Stored Queries (v0.6.1) + +```bash +omnigraph queries validate # type-check the stored-query registry vs the live schema (offline; exits non-zero on drift) +omnigraph queries list # list registry query names, MCP exposure, and typed params +``` + +`validate` opens the addressed graph and type-checks every applied stored query against the live schema — catches drift without restarting the server. `list` prints that graph's registry. Address the graph with `--store ` or a positional URI. Distinct from `lint` (which validates a single `.gq` file). See `references/stored-queries.md`. + +## Operator Config & Credentials + +```bash +echo "$TOKEN" | omnigraph login # store a bearer token in ~/.omnigraph/credentials (0600) +omnigraph logout # remove it (idempotent) +``` + +The operator config and `~/.omnigraph/credentials` are **auto-discovered — there is no flag to point at them.** `$OMNIGRAPH_HOME` relocates the `~/.omnigraph` *directory* (mainly for test isolation), and an absent file is just an empty layer (zero-config). Separately, `$OMNIGRAPH_CONFIG` stands in for the `--config` flag — which targets the **cluster directory / server config**, never the operator config. See SKILL.md → *The two config surfaces*. + +## Addressing a Graph + +How the CLI resolves which graph a data command (`query`, `mutate`, `load`, `branch`, …) runs against. A remote is addressed with `--server` (a bare `http(s)://` URL is not a graph address). + +Precedence (highest first): + +1. **`--store `** or a **positional `file://`/`s3://` URI** — direct storage access (bypasses any server; no catalog, so stored-query *names* don't resolve). `--store` is exclusive with a positional URI and with `--server`. +2. **`--server `** (+ `--graph ` for a multi-graph server) — served/remote. A name resolves from `servers:` in `~/.omnigraph/config.yaml`; a literal `http(s)://` URL also works. +3. **`--profile `** (or `$OMNIGRAPH_PROFILE`) — a named scope bundle from `profiles:` in the operator config (binds one of server/cluster/store + a default graph). +4. **Operator defaults** — `defaults.server` + `defaults.default_graph`, or `defaults.store` for a zero-flag local scope (mutually exclusive with `defaults.server`). + +Control-plane commands use `--config ` (cluster); maintenance against a cluster-managed graph uses `--cluster --graph `. Each command declares a **capability** — `any` / `served` / `direct` / `control` / `local` — shown in `omnigraph --help`; mis-addressing (e.g. `--server` on a `direct` verb, or a remote URI to `optimize`) fails loudly. + +For query source (`query`/`mutate`): + +1. **`--query `** or **`-e/--query-string ''`** — exactly one (operator aliases are invoked via the separate `alias` subcommand) +2. Relative `--query` paths resolve through **`query.roots`** in config + +For params: + +1. **Explicit `--params '{...}'`** wins on key conflict +2. **Positional alias args** map to alias `args` list + +## Output Formats + +`--format ` on query/mutate: + +- `table` (default) — human-readable +- `kv` — `key: value` per line; good for single rows +- `csv` — comma-separated +- `jsonl` — NDJSON, one per line, with metadata line first +- `json` — pretty JSON array + +For admin commands (branch, commit, schema, policy): use `--json` for structured output, otherwise human text. + +## Health Check + +```bash +curl http://127.0.0.1:8080/healthz +``` + +Returns `200 OK` if the server is up. + +## Cluster Control Plane (omnigraph >= 0.7.0) + +```bash +omnigraph cluster validate --config # parse + typecheck the declaration +omnigraph cluster import --config # one-time: create the state ledger +omnigraph cluster plan --config [--json] # preview (schema changes show migration steps) +omnigraph cluster apply --config --as # converge; idempotent +omnigraph cluster approve --config --as # gate destructive changes (graph deletes) +omnigraph cluster status --config [--json] # read the ledger (read-only) +omnigraph cluster refresh --config # re-observe live graphs; flags drift +omnigraph cluster force-unlock --config # clear a crashed run's lock (exact id from status) +``` + +Topology rule: `omnigraph schema apply` and `omnigraph init` **refuse a +cluster-managed graph** — in a cluster their jobs belong to `cluster apply`. +Data commands (`load`, `mutate`, branches) work either way — point them at the +derived root (`/graphs/.omni`, or `/graphs/.omni` for an +S3-backed cluster). See `references/cluster.md`. diff --git a/skills/omnigraph/references/data.md b/skills/omnigraph/references/data.md new file mode 100644 index 0000000..f553270 --- /dev/null +++ b/skills/omnigraph/references/data.md @@ -0,0 +1,175 @@ +# Data Changes & Branches + +## Contents +- Choose the right write command +- `mutate` — single edits +- `load` — bulk JSONL (`--mode`, `--from`) +- Branches: review before merge +- Destructive ops go through a branch +- Branch commands +- Inspecting state after changes + +How to modify data safely in Omnigraph. + +## Choose the Right Write Command + +`load` is the one bulk-JSONL command — local **or** remote, against any +existing branch, with a **required** `--mode`. `mutate` is for single typed +edits. + +| Task | Command | Why | +|------|---------|-----| +| Add/update a single entity | `mutate` with a named mutation | typechecked, parameterized, auditable | +| Bulk upsert by `@key` | `load --mode merge` | preserves rows not in the file | +| Additive-only bulk | `load --mode append` | fails on key collision | +| Clean-slate reseed | `load --mode overwrite` | **destructive** — wipes the branch | +| Bulk load onto a fresh review branch | `load --from main --mode merge --branch ` | forks `` from `main`, loads onto it, leaves it for review | + +> **`--mode` is required** — there is no default. Overwrite is destructive, so +> the CLI never picks a mode for you. +> +> **Local and remote are one command.** `load` works against a local repo URI +> (writing storage directly) *and* a remote `omnigraph-server` endpoint (the +> server orchestrates the write and publishes one atomic commit). See +> [`references/remote-ops.md`](remote-ops.md) for remote-specific concerns +> (504 handling, write-verification ritual). + +## `mutate` — Single Edits + +Goes through the running server (the configured default graph, or an alias): + +```bash +omnigraph mutate add_signal \ + --query mutations.gq \ + --params '{"slug":"sig-foo","name":"Foo","brief":"...","stagingTimestamp":"2026-04-14T00:00:00Z","createdAt":"2026-04-14T00:00:00Z","updatedAt":"2026-04-14T00:00:00Z"}' +``` + +Or via an alias: + +```bash +omnigraph alias add-signal sig-foo "Foo" "..." 2026-04-14T00:00:00Z 2026-04-14T00:00:00Z 2026-04-14T00:00:00Z +``` + +Prefer `mutate` for interactive edits, mutations called from agents, and anything you want typechecked at call time. + +## `load` — Bulk JSONL + +JSONL format: + +```jsonl +{"type":"Signal","data":{"id":"sig-foo","slug":"sig-foo","name":"Foo","brief":"...","stagingTimestamp":"2026-04-14T00:00:00Z","createdAt":"2026-04-14T00:00:00Z","updatedAt":"2026-04-14T00:00:00Z"}} +{"edge":"FormsPattern","from":"sig-foo","to":"pat-bar","data":{}} +``` + +- Nodes: `{"type":"","data":{...props...}}` — `id` equals `slug` +- Edges: `{"edge":"","from":"","to":"","data":{...edge_props...}}` + +Load command: + +```bash +omnigraph load --data seed.jsonl --mode merge s3://my-bucket/repos/spike-intel +``` + +`--from ` forks a missing `--branch` from `` before loading (the +one-shot review-branch flow below). Without `--from`, the target `--branch` +(default `main`) must already exist. + +### `--mode` semantics + +- **`overwrite`** (destructive) — replaces every node/edge table on the branch with the file's contents. **Staged**: the loader validates node/edge constraints, referential integrity, and edge cardinality *before* any data moves, so a bad file fails before touching the branch. Safe on a **first** load; risky afterward. Don't run it against `main` in production without a branch backup path. +- **`merge`** (upsert) — for each row, insert if `@key` is new, update if it exists. Rows not in the file are preserved. The safe default for incremental bulk updates. +- **`append`** (strict insert) — fails on key collision. Use when you're certain every row is new. + +### `merge` does NOT recompute embeddings + +If you change seed rows that feed into `@embed("source")` via `load --mode merge`, the source field updates but the embedding stays stale. + +**Fix:** run `omnigraph embed --reembed_all` after, or use `load --mode overwrite` once (which re-triggers embedding on load). + +### `overwrite` is destructive + +Wipes the entire branch's data for every node and edge type. Use only for: +- First-time seed +- Intentional full reseed on a feature branch +- Recovery scenarios + +Never on `main` without a branch backup. + +## Branches: Review Before Merge + +Branches exist for **data review**, not schema changes. Schema goes straight to `main` via `plan` + `apply`. + +### The review loop + +```bash +REPO=s3://my-bucket/repos/spike-intel + +# 1. Create feature branch from main +omnigraph branch create --from main staging-2026-04-14 --store $REPO + +# 2. Load delta onto the branch (merge mode is typical for review) +omnigraph load --data delta.jsonl --branch staging-2026-04-14 --mode merge $REPO + +# 3. Verify on the branch (reads can target --branch or --snapshot) +omnigraph query recent_signals --query queries/signals.gq --branch staging-2026-04-14 --store $REPO + +# 4. Merge to main when happy +omnigraph branch merge staging-2026-04-14 --into main --store $REPO + +# 5. Optionally delete the branch +omnigraph branch delete staging-2026-04-14 --store $REPO +``` + +### Fork a branch in one shot with `--from` + +- Bare `load` operates on an existing branch (default `main`). +- `load --from main --branch ` forks `` from `main`, loads onto it, and leaves it for review — the whole review-branch flow in one command. + +Use `--from` for anything you want reviewed before it touches `main`. + +### Keep branches short-lived + +Long-lived branches compound merge risk. The usual flow is: create → load → verify → merge → delete, all in the same session. A week-old feature branch is a yellow flag. + +### Schema apply blocks non-main branches + +`omnigraph schema apply` rejects the request if any non-main branches exist. Merge or delete them first. This is enforced — it's not just a guideline. + +## Destructive Ops Go Through a Branch + +For any bulk load that could disrupt downstream queries (overwriting a heavily-referenced node type, removing edges en masse, reseeding a core table), use a feature branch: + +```bash +omnigraph load --data risky.jsonl --branch recovery-2026-04-14 \ + --from main --mode overwrite $REPO +# inspect, diff, verify reads +omnigraph branch merge recovery-2026-04-14 --into main --store $REPO +``` + +## Branch Commands (quick reference) + +```bash +omnigraph branch create --from main --store $REPO +omnigraph branch list --store $REPO +omnigraph branch merge --into main --store $REPO +omnigraph branch delete --store $REPO +``` + +All support `--json` for automation-friendly output. Address the graph with a +positional `file://`/`s3://` URI (shown), `--store `, or `--server `. + +## Inspecting State After Changes + +```bash +omnigraph snapshot $REPO --branch main --json # tables + row counts +omnigraph export $REPO --branch main > graph.jsonl # full JSONL dump +omnigraph commit list $REPO --branch main --json # history +``` + +`export` is the right tool for large-snapshot inspection — don't try to page through the whole graph with read queries. + +> **Cluster note:** everything in this file applies unchanged in cluster +> deployments — the control plane owns schema/queries/policies; rows, loads, +> and branches stay on the data plane against the derived graph roots +> (`/graphs/.omni`, or `/graphs/.omni` for an S3-backed +> cluster). diff --git a/skills/omnigraph/references/migrations.md b/skills/omnigraph/references/migrations.md new file mode 100644 index 0000000..9aca605 --- /dev/null +++ b/skills/omnigraph/references/migrations.md @@ -0,0 +1,65 @@ +# Migration & Deprecations (pre-0.7.0 → 0.7.0) + +The rest of this skill teaches the **current 0.7.0 surface only**. Consult this page solely when you meet an old config file, command, flag, route, or error and need its current form. Pre-0.7.0 spellings keep working as deprecated aliases (they print a warning) unless marked **removed**. + +## Config files + +| Before (pre-0.7.0) | Now (0.7.0) | +|---|---| +| `omnigraph.yaml` (one combined file) | **`cluster.yaml`** (team deployment) + **`~/.omnigraph/config.yaml`** (operator) | +| `cli.actor` | `operator.actor` | +| `cli.graph` / `server.graph` | `defaults.default_graph` (+ `defaults.server`) | +| `targets:` / `target:` | `graphs:` / `graph:` | +| `omnigraph init` scaffolds `omnigraph.yaml` | `init` scaffolds nothing — start a `cluster.yaml` from [`cluster.md`](cluster.md) | + +- **`omnigraph.yaml` is fully removed in 0.7.0** — no CLI command or server reads it, and there is **no `config migrate`**. Move team settings to `cluster.yaml` and personal settings (identity, `servers:`, `defaults:`, `aliases:`) to `~/.omnigraph/config.yaml` by hand. + +## CLI addressing (RFC-011) + +| Before | Now | +|---|---| +| `--target ` | **removed** — use `--server `, `--store `, or `--profile ` (SKILL.md → *Addressing a graph*) | +| positional `http(s)://` URL → a server | **removed** — address a remote with `--server ` | +| `--as` on a served (remote) write | no-op — the server resolves the actor from the bearer token (`--as` applies to direct `--store` writes) | +| `--cluster-graph ` | **removed** — `--cluster ` is a global scope; pick the graph with `--graph `. `--graph` now selects within a `--server` *or* `--cluster` scope | +| `query`/`mutate` `--name ` + positional graph URI / `--uri` | **removed** — the query name is the **positional** (`omnigraph query `): a bare `` invokes a served stored query (kind-asserted), `--query`/`-e` is the ad-hoc lane. Address the graph via `--server`/`--store`/`--profile` (not a positional URI on query/mutate) | + +## Server boot & schema (RFC-011) + +| Before | Now | +|---|---| +| `omnigraph-server ` / `--config omnigraph.yaml` / `--target` / single-graph flat routes | **removed** — the server is **cluster-only**: `omnigraph-server --cluster `; all HTTP is nested under `/graphs//...` (flat routes → 404) | +| `omnigraph schema apply` on a cluster-managed graph | **refused** — evolve cluster graphs via `cluster apply` (the ledger). `schema apply` still works on a non-cluster store or via `--server` | +| `policy …` / `queries validate` via `--config omnigraph.yaml` | `policy validate\|test\|explain` reads `--cluster ` (+ `--graph`); `queries validate` takes the store URI | + +## CLI verbs + +| Before | Now | +|---|---| +| `omnigraph ingest …` | `omnigraph load --from main --mode merge …` | +| `omnigraph read` | `omnigraph query` | +| `omnigraph change` | `omnigraph mutate` | +| `omnigraph query lint` / `query check` | `omnigraph lint` | +| `omnigraph query --alias ` / `mutate --alias ` | `omnigraph alias ` (dedicated subcommand; the `--alias` flag was removed) | + +## HTTP routes + +| Before | Now | +|---|---| +| `POST /ingest` | `POST /load` | +| `POST /read` | `POST /query` | +| `POST /change` | `POST /mutate` | + +The old routes remain as **deprecated aliases** (retained indefinitely), carrying `Deprecation: true` + `Link: ` response headers. + +## Server token resolution + +| Before | Now | +|---|---| +| `graphs..bearer_token_env` in `omnigraph.yaml` | `omnigraph login ` → `~/.omnigraph/credentials`, or `OMNIGRAPH_TOKEN_` | + +The client bearer token now comes only from `OMNIGRAPH_TOKEN_` or the credentials file — the `omnigraph.yaml` `bearer_token_env` chain is gone with the file. + +## Older removals (still worth knowing) + +- The transactional **Run** state machine, its `/runs` routes, and the `run_publish` / `run_abort` Cedar actions were **removed in v0.4.0**. Writes publish directly — use `GET /commits` for history and the `change` action for write gating; `/runs` returns 404. diff --git a/skills/omnigraph/references/queries.md b/skills/omnigraph/references/queries.md new file mode 100644 index 0000000..f9f84e0 --- /dev/null +++ b/skills/omnigraph/references/queries.md @@ -0,0 +1,302 @@ +# Query Authoring & Linting + +## Contents +- File organization +- Linting +- Parameterization +- Query structure +- Search functions +- Aggregations +- Filter operators +- Mutations +- Naming convention +- Aliases over raw queries + +Writing `.gq` query files in Omnigraph. + +## File Organization + +- One `.gq` file per primary node type (`signals.gq`, `patterns.gq`, `elements.gq`) +- One `mutations.gq` file for all insert/update/delete queries +- Put query files in `queries/` — cluster mode discovers `queries/*.gq` automatically + +## Linting + +```bash +omnigraph lint --schema schema.pg --query queries/signals.gq +``` + +Or (lint against a live repo): + +```bash +omnigraph lint --query queries/signals.gq s3://bucket/repo +``` + +Lint returns: +- `"status": "ok"` — all queries passed +- `"errors": N` — count of type errors (exit 1 when nonzero) +- `"warnings": N` — count of drift warnings + +Run lint after every `.gq` or `.pg` edit. Wire into precommit. + +## Parameterization + +### Always declare typed parameters + +```gq +query get_signal($slug: String) { + match { $s: Signal { slug: $slug } } + return { $s.slug, $s.name } +} +``` + +Never string-interpolate values into query bodies. Pass them via `--params`: + +```bash +omnigraph query get_signal --query signals.gq --params '{"slug":"sig-foo"}' +``` + +The compiler typechecks parameter values against declared types. + +> For one-off/ad-hoc execution, pass the query inline instead of a file with `-e/--query-string` (v0.6.0+): `omnigraph query -e 'query q($slug: String){ match { $s: Signal { slug: $slug } } return { $s.name } }' --params '{"slug":"sig-foo"}'` (and `omnigraph mutate -e '...'`). `-e` is mutually exclusive with `--query ` — exactly one of the two is required. (Operator aliases are invoked via the separate `omnigraph alias ` subcommand.) + +## Query Structure + +### Match → Return → Order → Limit + +```gq +query recent_signals() { + match { + $s: Signal + } + return { $s.slug, $s.name, $s.stagingTimestamp } + order { $s.stagingTimestamp desc } + limit 50 +} +``` + +### Edge traversal (lowerCamelCase) + +Schema edges are PascalCase; traversal uses lowerCamelCase: + +```gq +match { + $s: Signal { slug: $slug } + $s formsPattern $p // edge FormsPattern: Signal -> Pattern +} +``` + +### Multi-hop + +Chain traversal clauses: + +```gq +query friends_of_friends($name: String) { + match { + $p: Person { name: $name } + $p knows $mid + $mid knows $fof + } + return { $fof.name } +} +``` + +### Reverse traversal + +Flip the subject/object: + +```gq +query employees_of($company: String) { + match { + $c: Company { name: $company } + $p worksAt $c + } + return { $p.name } +} +``` + +### Negation + +```gq +query orphan_signals() { + match { + $s: Signal + not { $s formsPattern $_ } + } + return { $s.slug } +} +``` + +## Search Functions + +### Text search + +```gq +match { + $d: Doc + search($d.title, $q) // full-text on @index'd String +} +``` + +```gq +match { + $d: Doc + fuzzy($d.title, $q, 2) // fuzzy match, max 2 edits +} +``` + +```gq +match { + $d: Doc + match_text($d.body, $q) // phrase match +} +``` + +### Vector/ranking (require `limit`) + +```gq +query vector_search($q: Vector(3072)) { + match { $d: Doc } + return { $d.slug, $d.title } + order { nearest($d.embedding, $q) } + limit 10 +} +``` + +`nearest`, `bm25`, and `rrf` are ranking operators, not filters. Every query using them **must** end with `limit N` — omitting it is a compile error. + +### Hybrid (reciprocal rank fusion) + +```gq +query hybrid_search($vq: Vector(3072), $tq: String) { + match { $d: Doc } + return { $d.slug, $d.title } + order { rrf(nearest($d.embedding, $vq), bm25($d.title, $tq)) } + limit 10 +} +``` + +## Aggregations + +```gq +query friend_counts() { + match { + $p: Person + $p knows $f + } + return { + $p.name + count($f) as friends + } + order { friends desc } + limit 20 +} +``` + +Supported: `count`, `sum`, `avg`, `min`, `max`. Grouping is implicit on non-aggregated return fields. + +## Filter Operators + +`=`, `!=`, `>`, `<`, `>=`, `<=`, `contains` + +```gq +match { + $p: Person + $p.age > 30 + $p.name contains "Al" +} +``` + +## Mutations + +> **No top-level `mutation { ... }` wrapper.** Agents trained on GraphQL reflexively write `mutation { insert T { ... } }` — that fails the parser at character 1 with `parse error: expected query_file`. Every executable block in a `.gq` file is a named `query`; the body's verb (`insert` / `update` / `delete`) determines whether it's a write. Dispatch via `omnigraph mutate` (not `query`). + +### Insert + +```gq +query add_signal($slug: String, $name: String, $brief: String, + $stagingTimestamp: DateTime, $createdAt: DateTime, $updatedAt: DateTime) { + insert Signal { + slug: $slug, + name: $name, + brief: $brief, + stagingTimestamp: $stagingTimestamp, + createdAt: $createdAt, + updatedAt: $updatedAt + } +} +``` + +**Every non-nullable property must be provided.** Lint catches missing ones as: + +``` +error: T12: insert for 'Signal' must provide non-nullable property 'brief' +``` + +### Insert edge + +```gq +query link_signal_forms_pattern($signal: String, $pattern: String) { + insert FormsPattern { from: $signal, to: $pattern } +} +``` + +Edge `data` block is `{}` if the edge has no properties — just specify `from` and `to` slugs. + +### Update + +```gq +query retitle_signal($slug: String, $new_title: String) { + update Signal set { name: $new_title } where slug = $slug +} +``` + +### Delete + +```gq +query remove_signal($slug: String) { + delete Signal where slug = $slug +} +``` + +### Multi-statement + +```gq +query add_and_link($slug: String, $pattern: String, $createdAt: DateTime, $updatedAt: DateTime) { + insert Signal { slug: $slug, name: $slug, brief: $slug, + stagingTimestamp: $createdAt, createdAt: $createdAt, updatedAt: $updatedAt } + insert FormsPattern { from: $slug, to: $pattern } +} +``` + +There's no `upsert` keyword at the query level — use `load --mode merge` for bulk upsert. + +> **Insert/update-only OR delete-only (the D₂ rule).** A single mutation query may contain inserts and updates, **or** deletes — never both. Mixing a `delete` with an `insert`/`update` in the same query is rejected at parse time. (Inserts/updates go through a staged two-phase publish; deletes inline-commit — omnigraph doesn't yet use Lance's two-phase delete API (it shipped in Lance 7.0.0 but isn't wired in) — so they can't share one atomic statement.) Split a delete-then-insert into two separate mutations. + +### Date and DateTime values + +Date format is asymmetric between `mutate` (parameter values) and `load` (JSONL): + +| Path | Date | DateTime | +|---|---|---| +| `mutate --params` | ISO string `"2026-04-29"` | ISO string `"2026-04-29T10:00:00Z"` | +| `load` JSONL | Integer days since epoch `20572` | ISO string `"2026-04-29T10:00:00Z"` | + +Compute integer days form for a given date `d`: + +```python +(d - datetime.date(1970, 1, 1)).days # d is the date you're loading, not today() +``` + +This asymmetry is one of the most common silent type errors when bulk-loading data prepared for one path through the other. + +## Naming Convention + +`verb_object`: +- `get_signal`, `recent_signals`, `search_signals` +- `signal_patterns`, `signal_elements` (traversal queries) +- `add_signal`, `link_signal_forms_pattern` (mutations) + +## Aliases Over Raw Queries + +For anything an agent or script will call repeatedly, define an operator alias. See `references/aliases.md`. diff --git a/skills/omnigraph/references/remote-ops.md b/skills/omnigraph/references/remote-ops.md new file mode 100644 index 0000000..e956dd7 --- /dev/null +++ b/skills/omnigraph/references/remote-ops.md @@ -0,0 +1,142 @@ +# Remote Graph Operations + +## Contents +- What's different about remote +- Verify after every write +- 504 Gateway Timeout +- Fork-branch 504 fingerprint +- Targeting a remote graph (`--server`, `login`) +- Version drift / `sync_branch()` +- `manifest_conflict` 409 +- 429 Too Many Requests +- Duplicate risk on blind retry +- Reading large schemas safely +- Prevention checklist + +When the graph URI is a remote endpoint (`omnigraph-server` behind ALB / CloudFront, bearer-authenticated) instead of a local S3 path, several CLI behaviors change in ways the local-storage workflow never exposes. This reference covers the failures and operational rituals specific to remote graphs. + +## What's different about remote + +A remote graph runs server-side. Every write executes on the server — staged per touched table, then published atomically as a **single manifest commit** guarded by a compare-and-swap on expected table versions — and is gated by a connection-level idle timeout (CloudFront defaults to ~30s). There is no separate "run" object to poll — write status is implied by the HTTP response (and verifiable via `commit list`). The local CLI is a thin client; it never sees the commit happen, only the HTTP response. That asymmetry is the root of every gotcha below. + +| Local repo | Remote repo | +|---|---| +| CLI writes S3 directly | Server executes the write, publishes one atomic manifest commit | +| No connection timeout | ~30s idle timeout (CloudFront) | +| No admission control | Per-actor `429` + `Retry-After` on writes | +| `load` writes S3-backed storage directly | `load` is server-orchestrated — same command, one atomic commit | +| CLI exit code is authoritative | CLI exit code can lie — verify via `commit list` | + +## Verify after every write + +The CLI's exit code is **not authoritative on remote graphs**. The proxy can drop a response after the server has already committed. Always verify by comparing `main`'s head: + +```bash +HEAD_BEFORE=$(omnigraph commit list --config X --branch main --json | jq -r '.commits[0].graph_commit_id') + +# … run your load / mutate … + +HEAD_AFTER=$(omnigraph commit list --config X --branch main --json | jq -r '.commits[0].graph_commit_id') + +if [[ "$HEAD_BEFORE" != "$HEAD_AFTER" ]]; then + echo "landed" +else + echo "did NOT land — safe to retry" +fi +``` + +For a `load --from` that forks a review branch, also compare the new branch head's `graph_commit_id` against `main`'s. **Identical means the load didn't land — empty fork left behind.** + +For pointed verification of a single record: + +```bash +omnigraph export --config X --type | grep +omnigraph export --config X --type | grep +``` + +## 504 Gateway Timeout: response lost, write status unknown + +A 504 from the proxy means the server didn't respond within the idle timeout. Two server-side outcomes are possible — **the 504 alone cannot distinguish them**: + +1. **Write completed and published** — landed, `main`'s head advanced. Common for small mutations finishing just past the 30s edge. +2. **Write still in progress** — will publish or fail soon. Re-check after a minute. + +Always verify via `commit list` before retrying. Blind retry on append-only types creates duplicates. + +## Fork-branch 504 fingerprint + +`load --from ` creates the branch **before** loading data. A timed-out fork-load where the data didn't land leaves an empty branch at ``'s head. Stale numbered branches (`feature-v2`, `-v3`, `-v4` …) all sitting at the same `graph_commit_id` as `main` are the fingerprint of prior 504-blocked attempts. + +Find them by comparing each branch's head against `main`'s in `omnigraph branch list --config X --json`, then delete the empty ones. + +## Targeting a remote graph: `--server` and `login` + +`load`, `query`, and `mutate` all run against a remote `omnigraph-server` endpoint — there is no local-only restriction as of 0.7.0. Address an operator-defined server by name instead of pasting URLs and juggling tokens: + +```bash +echo "$TOKEN" | omnigraph login intel-dev # stores it in ~/.omnigraph/credentials (0600) +omnigraph load --server intel-dev --graph spike \ + --data delta.jsonl --from main --mode merge --branch staging +``` + +`--server ` resolves the URL from `~/.omnigraph/config.yaml` and the token via `OMNIGRAPH_TOKEN_` or the credentials file. A token is only ever sent to the server it is keyed to. `--graph ` selects the graph on a multi-graph server. + +## Version drift / `sync_branch()` + +``` +version drift on node:: snapshot pinned vN but dataset is at vM — call sync_branch() and retry +``` + +- `sync_branch()` is **not a CLI command** — it's a server-internal directive that leaked into the error text. Don't go looking for it. +- Cause: another actor committed to `main` between your CLI's snapshot pin and your `mutate` attempt. +- Usually self-resolves on retry — the next call re-pins. +- Calling `omnigraph snapshot` does **not** reliably re-pin for subsequent `mutate`s in the same session. +- If persistent, fall back to `load --from main` onto a fresh branch — a forked branch doesn't suffer from concurrent-commit drift on `main`. +- The cleaner, modern form of this conflict is a structured `manifest_conflict` **409** — see below. + +## `manifest_conflict` 409 — stale snapshot, retry + +When another actor commits to the same branch between your query's snapshot pin and your write, the server returns a structured **`manifest_conflict` 409** carrying `table_key` / `expected` / `actual`, rather than silently overwriting. Since v0.4.2 this is the form most concurrent update/delete/merge races take. + +- **Retry it.** A 409 means your write was computed against a stale view and was rejected *before* committing — there is no partial state and no duplicate risk. Re-issue the same call; it re-pins to the new head. +- Concurrent `mutate` × branch-merge on the same target branch resolves to either success or a clean 409 depending on who wins the server's per-table queue — both outcomes are safe. + +## 429 Too Many Requests — back off, then retry + +The server applies **per-actor admission control** to every mutating endpoint (`mutate` / `load` / `schema apply` / branch create·delete·merge). An actor that exceeds its in-flight-request or estimated-byte budget gets a structured **HTTP 429** (`code: too_many_requests`) with a `Retry-After` header — instead of blocking unrelated actors behind a global lock. + +- This is **not** a failed write — the write never started. Honor `Retry-After` and retry; it is always safe (no partial write, no duplicate risk). +- It's per-actor, so one noisy automation can't starve others. If you hit it constantly, batch less aggressively or space your calls out. +- Read-only endpoints are not admission-gated. + +## Duplicate risk on blind retry + +After a 504, never retry without verifying first. Different node kinds have different retry semantics: + +| Kind | Retry safety | +|---|---| +| Pointer nodes (`Org`, `Person`, `Opportunity`, `Channel`, `Actor`, `ActionItem`, `Artifact`, `Meeting`, `Technology`, `Campaign`, `UseCase`) | ✓ Idempotent — `@key` upserts dedupe | +| Append-only nodes (`Signal`, `Claim`, `Decision`, `Event`, `Interaction`, `MarketingElement`, `Policy`, `Outcome`) | ✗ Duplicates on retry — verify before retrying | +| Edges | ⚠ No `@key`. Verify via `export --type ` + grep. Some simple edges dedupe server-side; don't rely on it. | + +## Reading large schemas safely + +Remote schemas can be large (tens of KB). Tools that cap stdout (~50KB is common) will truncate or duplicate the output silently — leading to memory-based answers from agents that look correct but reference nonexistent fields. + +Always redirect to a file before reading: + +```bash +omnigraph schema show --config X > /tmp/schema.pg +wc -l /tmp/schema.pg +``` + +Then read the file with offset/limit, not via piped stdout. + +## Prevention checklist + +- Keep mutations small. Single-node inserts finish well under the timeout. +- Prefer `mutate` over `load` for ≤ a handful of records. +- Always run `commit list` after a 504 before deciding to retry. +- For destructive or large-batch work, use `load --from main` onto a feature branch and verify the branch head before merging. +- Read large schemas via file redirect, not piped stdout. +- A `429` (throttle) or a `manifest_conflict` `409` (stale snapshot) is always safe to retry — the write never committed. Honor `Retry-After` on a 429. diff --git a/skills/omnigraph/references/schema.md b/skills/omnigraph/references/schema.md new file mode 100644 index 0000000..b30745b --- /dev/null +++ b/skills/omnigraph/references/schema.md @@ -0,0 +1,192 @@ +# Schema Authoring & Evolution + +## Contents +- Authoring (.pg files) +- Evolution (schema plan/apply) +- Supported types +- Decorators (quick reference) +- Interfaces +- Design principles +- Schema evolution in cluster mode + +How to write and evolve `.pg` schemas in Omnigraph. + +## Authoring (.pg files) + +### Use `//` for comments + +Not `#`. The compiler rejects `#` with a parse error that looks like: + +``` +parse error: expected schema_file +``` + +### Enums are inline, not standalone + +The compiler does **not** accept top-level `enum Foo { ... }` blocks. Put the values inline on the property: + +```pg +kind: enum(product, technology, framework, concept, ops) @index +``` + +If the same enum appears on multiple nodes, duplicate it inline — there's no shared enum type. + +### Lists contain scalars only + +`[String]` and `[I32]` are fine. `[Category]` (a list of enum values) is **not** supported. Use `[String]` with query-side filtering, or use a single-valued enum property if one value is enough. + +### `@embed` takes a quoted string + +```pg +embedding: Vector(3072) @embed("text") @index +``` + +Not `@embed(text)`. The source property name is a string literal. + +### Edge constraints go inside a body block + +`@unique(src, dst)` on an edge goes inside `{ }`, after `@card(...)`: + +```pg +edge PartOfArtifact: Chunk -> InformationArtifact @card(1..1) { + @unique(src) +} +``` + +### Lint after every edit + +```bash +omnigraph lint --schema schema.pg --query queries/signals.gq +``` + +This validates the schema **and** the queries against it. No running repo required. Wire it into a precommit hook. + +## Evolution (schema plan/apply) + +### Plan before apply — always + +```bash +omnigraph schema plan --schema next.pg s3://bucket/repo --json +# inspect "supported": true|false and the step list +omnigraph schema apply --schema next.pg s3://bucket/repo +``` + +If `supported: false`, fix the source before applying. Plan is free; run it as often as needed. + +Plan/apply diagnostics carry stable codes of the form **`OG-XXX-NNN`** (since v0.5.0) — match on the code, not the free-form message text. + +**Destructive drops are gated (since v0.5.0).** Dropping a property or type is a soft drop by default (or rejected); to actually lose data you must opt in: + +```bash +omnigraph schema apply --schema next.pg s3://bucket/repo --allow-data-loss +``` + +Over HTTP the equivalent is `{"allow_data_loss": true}` in the schema-apply body. Without the flag, a destructive drop returns a structured diagnostic instead of silently deleting columns. + +### Apply is main-only + +`omnigraph schema apply` rejects any non-`main` branches. Delete or merge feature branches first. This is deliberate: schema changes don't go through review branches. They go straight to main via `plan` + `apply`. + +### Rename, don't replace + +Use `@rename_from(...)` on renames so the planner emits a rename step (preserves data), not a drop+add pair (loses data): + +```pg +node Account @rename_from("User") { + full_name: String @rename_from("name") +} +``` + +Works on node types, edge types, and properties. + +### Required properties need a backfill plan + +Adding a non-nullable property to an existing node is rejected as unsupported. Pattern: + +1. Add as optional: `new_prop: String?` +2. Apply +3. Backfill via a `mutate` or `load --mode merge` +4. Tighten to required in a follow-up apply: `new_prop: String` + +### Keep `@key` stable + +Changing the key field is effectively a replace — it invalidates every external reference to the node. Treat identity changes as deliberate, multi-step migrations, not casual field renames. + +### `schema apply` blocks writes while running + +No concurrent mutations during an apply. Plan for a short read-only window. + +## Supported Types + +- **Scalars:** `String`, `Bool`, `I32`, `I64`, `U32`, `U64`, `F32`, `F64`, `Date`, `DateTime`, `Blob` +- **Collections:** `Vector(N)` (fixed-size float vector), `[ScalarType]` (list of scalar) +- **Enums:** `enum(value1, value2, ...)` — inline only, values can contain alphanumerics, underscores, hyphens +- **Optional:** any type + `?` suffix (`String?`, `[I32]?`, `Vector(4)?`) + +## Decorators (quick reference) + +**Property-level:** +- `@key` — primary key (implies index; usually one per node) +- `@unique` — uniqueness constraint +- `@index` — query optimization +- `@range(min, max)` — numeric bounds (open ranges allowed) +- `@check(prop, "regex")` — regex pattern validation on a String property +- `@embed("source_prop")` — embed from a String source into a Vector property +- `@description("...")` — metadata (no migration impact) +- `@instruction("...")` — semantic hint for LLMs/operators + +**Edge-level:** +- `@card(min..max)` — edge cardinality (default: `0..*`) + +**Type-level (nodes/edges/properties):** +- `@rename_from("OldName")` — migration-aware rename + +**Group-level (inside body block):** +- `@unique(prop1, prop2)` — composite uniqueness, enforced as a true tuple key at both intake and merge (works on edges too: `@unique(src, dst)`). Columns must reduce to a scalar key: `@unique` on a `[List]`/`Blob` column is rejected loudly at `load` (it used to be silently un-enforced — fixed in #160). +- `@index(prop1, prop2)` — composite index + +## Interfaces + +Supported but rarely used. Declare shared property contracts and node types implement them: + +```pg +interface Searchable { + title: String @index + embedding: Vector(3072) @embed("title") +} + +node Doc implements Searchable { + slug: String @key + body: String +} +``` + +Most schemas are fine without interfaces. Reach for them only when 3+ node types need to share a property contract. + +## Design Principles (brief) + +- **Identity is explicit** — use `@key` on a semantic slug, not internal row IDs +- **Narrow types** — `Date` over `String` for dates, `enum` over `String` for lifecycle states +- **Edge semantics matter** — prefer `AuthoredBy` over `RelatedTo` +- **Constraints live in the schema** — `@unique`, `@range`, `@card` keep invariants out of application code +- **Schemas are reviewable** — clear names, explicit enums, obvious keys + +## Schema Evolution in Cluster Mode + +In a cluster deployment there is **no direct `omnigraph schema apply`** — the +schema is declared (`graphs..schema:` in `cluster.yaml`) and converged: + +```bash +$EDITOR schema.pg +omnigraph cluster plan --config . # shows the engine's migration steps +omnigraph cluster apply --config . --as +# restart the --cluster server to serve the new shape +``` + +Differences from direct `schema apply` (on a non-cluster store): **soft drops +only** (`--allow-data-loss` is not reachable from cluster apply — prior versions +retain dropped columns), +and out-of-band schema changes on the live graph are *drift* — `cluster +refresh` flags them and the next `apply` converges the graph back to the +declared schema. Everything else in this file (`@rename_from`, backfills, +linting, enum discipline) applies unchanged to the `.pg` you edit. diff --git a/skills/omnigraph/references/search.md b/skills/omnigraph/references/search.md new file mode 100644 index 0000000..53397ab --- /dev/null +++ b/skills/omnigraph/references/search.md @@ -0,0 +1,150 @@ +# Search & Embeddings + +## Contents +- Embeddings are schema-declared +- Generating embeddings +- Embeddings + `load --mode merge` interaction +- Search functions in queries +- The key pattern: scope first, rank second +- Model / config + +Vector embeddings and text search in Omnigraph. + +## Embeddings are Schema-Declared + +```pg +node Chunk { + text: String + chunk_index: I32 + embedding: Vector(3072) @embed("text") @index + createdAt: DateTime +} +``` + +- `Vector(N)` — fixed-size float vector +- `@embed("source_prop")` — what text field to embed from (quoted string) +- `@index` — enables vector search on this field + +The schema says **where** embeddings live and **what** they come from. Queries don't recompute; they read. + +## Generating Embeddings + +### First time / refresh missing + +```bash +omnigraph embed --seed embed-config.yaml +``` + +Default mode is `fill_missing` — only generates embeddings for rows without one. + +### Re-embed everything + +```bash +omnigraph embed --seed embed-config.yaml --reembed_all +``` + +Use when: +- You changed the source field: `@embed("body")` → `@embed("title")` +- You mutated text at scale and need fresh embeddings +- You switched embedding models (rare) + +### Selective refresh + +```bash +omnigraph embed --seed embed-config.yaml --select "Chunk:chunk_index=42" +``` + +Regenerate only rows matching the selector. + +### Clean (delete) embeddings + +```bash +omnigraph embed --seed embed-config.yaml --clean +``` + +## Embeddings + `load --mode merge` Interaction + +**`load --mode merge` does NOT recompute embeddings.** + +If you update rows whose source fields feed into `@embed(...)`, the source updates but the embedding stays stale. + +Two fixes: +1. Run `omnigraph embed --reembed_all` after the merge +2. Use `load --mode overwrite` instead, which re-triggers embedding on load + +## Search Functions in Queries + +All ranking functions require `limit N` — they're order operators, not filters. + +### Vector similarity + +```gq +query nearest_chunks($q: Vector(3072)) { + match { $c: Chunk } + return { $c.text } + order { nearest($c.embedding, $q) } + limit 10 +} +``` + +### BM25 text ranking + +```gq +query top_titles($q: String) { + match { $d: Doc } + return { $d.slug, $d.title } + order { bm25($d.title, $q) } + limit 10 +} +``` + +### Hybrid (Reciprocal Rank Fusion) + +```gq +query hybrid($vq: Vector(3072), $tq: String) { + match { $d: Doc } + return { $d.slug, $d.title } + order { rrf(nearest($d.embedding, $vq), bm25($d.title, $tq)) } + limit 10 +} +``` + +### Text filter (not ranking — no `limit` required) + +```gq +match { + $d: Doc + search($d.title, $q) // full-text filter + fuzzy($d.title, $q, 2) // fuzzy filter, max 2 edits + match_text($d.body, $q) // phrase filter +} +``` + +## The Key Pattern: Scope First, Rank Second + +Filter with graph traversal before invoking vector or text ranking. Ranking over a narrow set is both cheaper and more relevant. + +```gq +query related_chunks($artifact_slug: String, $q: Vector(3072)) { + match { + $a: InformationArtifact { slug: $artifact_slug } + $c partOfArtifact $a // scope: only this artifact's chunks + } + return { $c.text } + order { nearest($c.embedding, $q) } // rank: vector similarity within scope + limit 10 +} +``` + +Don't rank over the entire chunk set if you know a traversal can narrow it first. + +## Model / Config + +Omnigraph uses **two distinct embedding clients** — don't conflate them: + +| Client | When it runs | Default model | Configured via | +|--------|--------------|---------------|----------------| +| **Engine / load-time** | At load, when an `@embed("source")` field is populated (and `omnigraph embed`) | `gemini-embedding-2-preview` (3072-dim) | `GEMINI_API_KEY`, `OMNIGRAPH_GEMINI_BASE_URL`, `OMNIGRAPH_EMBED_*`, `OMNIGRAPH_EMBEDDINGS_MOCK` | +| **Compiler / query-time** | When a query passes a *string* to a ranking op (e.g. `nearest($c.embedding, "some text")`) and the server auto-embeds it | `text-embedding-3-small` (OpenAI-style) | `NANOGRAPH_EMBED_MODEL`, `OPENAI_API_KEY`, `OPENAI_BASE_URL`, `NANOGRAPH_EMBEDDINGS_MOCK` | + +The vector stored in the schema is produced by the **load-time (engine)** client, so `Vector(N)` must match that model's output dimension — `Vector(3072)` for `gemini-embedding-2-preview`. If you point the query-time client at a model with a different dimension than your stored vectors, similarity search returns garbage or errors — keep both sides on the same dimension. Vectors are stored L2-normalized. diff --git a/skills/omnigraph/references/server-policy.md b/skills/omnigraph/references/server-policy.md new file mode 100644 index 0000000..225c708 --- /dev/null +++ b/skills/omnigraph/references/server-policy.md @@ -0,0 +1,224 @@ +# HTTP Server & Cedar Policy + +## Contents +- Starting the server (boot sources) +- HTTP routes +- Auth +- Setup operations bypass the server +- Cedar policy +- Multi-graph mode +- Server + policy together +- Cluster-booted servers + +How to run `omnigraph-server` and gate operations with Cedar policies. + +## Starting the Server + +The server is the canonical runtime entry point — all CLI queries, mutations, and admin ops go through it. **Boot is cluster-only** (RFC-011): the server boots from a cluster and serves N graphs (N ≥ 1) under nested routes. There is **no** single-graph / bare-URI / `omnigraph.yaml` boot. + +```bash +omnigraph-server --cluster ./company-brain --bind 127.0.0.1:8080 # a config directory … +omnigraph-server --cluster s3://bucket/prefix --bind 0.0.0.0:8080 # … or a storage-root URI (config-free) +``` + +`--cluster` boots from the cluster's applied revision (see *Cluster-Booted Servers* below). Run it in a separate terminal or background process. + +## HTTP Routes + +All per-graph routes are nested under `/graphs/{id}/...` (`{id}` = a graph id from the applied cluster); bare flat paths (`/query`, `/snapshot`, …) return **404**. `/healthz` and `/graphs` stay flat. + +| Route | Purpose | +|-------|---------| +| `GET /healthz` | liveness probe (flat) | +| `GET /graphs` | enumerate served graphs (flat; `graph_list`-gated) | +| `GET /graphs/{id}/snapshot?branch=` | table state + row counts | +| `POST /graphs/{id}/query` | read query (canonical; `/read` = deprecated alias) | +| `POST /graphs/{id}/mutate` | mutation (`/change` = deprecated alias) | +| `POST /graphs/{id}/load` | bulk JSONL load, 32 MB; branch creation opt-in via `from` (`/ingest` = deprecated alias) | +| `POST /graphs/{id}/export` | NDJSON stream of a branch | +| `GET /graphs/{id}/queries` · `POST /graphs/{id}/queries/{name}` | stored-query catalog (`read`) + invocation (`invoke_query`, +`change` for a stored mutation; deny == 404) | +| `GET /graphs/{id}/schema` · `POST /graphs/{id}/schema/apply` | read `.pg` · migrate (`schema_apply`) | +| `GET/POST /graphs/{id}/branches` · `DELETE …/branches/{b}` · `POST …/branches/merge` | branch ops | +| `GET /graphs/{id}/commits?branch=` · `…/commits/{commit_id}` | history | + +Read routes take `?branch=main` or `?snapshot=`. Writes publish directly and commit atomically via `__manifest`; use the commits route for write/audit history. + +## Auth + +Set bearer tokens on the server process. Three sources, in precedence: `OMNIGRAPH_SERVER_BEARER_TOKENS_AWS_SECRET` (AWS Secrets Manager) → `OMNIGRAPH_SERVER_BEARER_TOKENS_JSON`/`_FILE` (JSON `{actor_id: token}`) → `OMNIGRAPH_SERVER_BEARER_TOKEN` (single token, actor `default`): + +```bash +OMNIGRAPH_SERVER_BEARER_TOKENS_JSON='{"act-reader":"s3cret"}' \ + omnigraph-server --cluster ./company-brain --bind 0.0.0.0:8080 +``` + +On the client side (0.7.0), register the server once and store its token out of band: + +```bash +echo "s3cret" | omnigraph login remote # → ~/.omnigraph/credentials (0600) +omnigraph query get_signal --server remote --graph spike --params '{"slug":"sig-foo"}' +``` + +`--server remote` resolves the URL from `~/.omnigraph/config.yaml`'s `servers:` and the token via `OMNIGRAPH_TOKEN_REMOTE` or the credentials file. A token is only ever sent to the server it is keyed to. + +### Running without auth requires an explicit opt-in + +You can no longer just "leave auth off." Since v0.6.0 the server **refuses to start** when it has neither bearer tokens nor a policy file, unless you explicitly opt in: + +```bash +omnigraph-server --cluster . --unauthenticated +# or: OMNIGRAPH_UNAUTHENTICATED=1 omnigraph-server --cluster . +``` + +This is a guardrail against accidentally shipping an open server. For pure local dev, pass `--unauthenticated` deliberately. + +## Setup Operations Bypass the Server + +`init` and **local** `load` write storage directly — they don't go through the server (a **remote** `load` is server-orchestrated, POSTing `/load`). Pass the repo URI: + +```bash +omnigraph init --schema schema.pg s3://my-bucket/repos/ +omnigraph load --data seed.jsonl --mode overwrite s3://my-bucket/repos/ +``` + +Everything else — `query`, `mutate`, `snapshot`, `schema plan/apply`, `branch`, `commit` — goes through the running server. + +## Cedar Policy + +Omnigraph can gate sensitive actions with [Cedar](https://www.cedarpolicy.com/) policies. + +### Default-deny posture + +Policy is enforced engine-wide (every authoring path calls the same gate), and the default is **closed**, not open: + +| Server state | Bearer tokens | Policy file | Behavior | +|---|---|---|---| +| **Open** | no | no | Every request permitted — but the server refuses to start without `--unauthenticated` / `OMNIGRAPH_UNAUTHENTICATED=1`. | +| **DefaultDeny** | yes | no | Every authenticated request for an action other than `read` is rejected (HTTP 403). "Tokens but forgot the policy file" no longer ships the illusion of protection. | +| **PolicyEnabled** | yes | yes | Requests are evaluated against your Cedar rules. | + +So configuring a policy file is what *enables* writes — there is no "permit everything by default" mode once tokens are set. + +### Gated actions + +Per-graph actions (evaluated against the graph being addressed): + +| Action | Protects | +|--------|----------| +| `read` | query execution | +| `export` | data export | +| `change` | mutations | +| `invoke_query` | stored-query invocation via `POST /graphs/{id}/queries/{name}` (graph-scoped, not branch-scoped). A stored **mutation** is double-gated — it also passes `change`. For a caller without the grant, a denial and an unknown query name both return the same **404** so the catalog can't be probed. | +| `schema_apply` | schema migrations | +| `branch_create` | branch creation | +| `branch_delete` | branch deletion | +| `branch_merge` | merges (especially into protected branches) | + +`admin` exists but is reserved (no call site yet — don't write rules for it). A server-scoped `graph_list` action gates `GET /graphs`; declare it in a `[cluster]`-scoped bundle. + +For any shared repo, gate at least `schema_apply` and `branch_merge`. + +### Where policy is declared + +Cedar bundles are declared in `cluster.yaml` and attach via `applies_to`: `[cluster]` is the server-level engine (gates `graph_list` / `GET /graphs`); `[]` is that graph's engine (gates `invoke_query`, `read`, `change`, `branch_*`, `schema_apply`). `cluster apply` publishes them and the `--cluster` server enforces the applied revision. The `policy.yaml` rule format (below) is the bundle content. + +### `policy.yaml` shape + +The policy model is **allow-only**: every rule is a `permit`. You grant capabilities to groups; anything ungranted is denied by default. There is **no `deny` / `effect` key** — to forbid something, simply don't grant it. + +```yaml +version: 1 # required; must be 1 + +groups: + admins: [act-alice, act-bob] + team: [act-carol, act-dan] + +protected_branches: + - main + +rules: + - id: admins-can-apply-schema # rules use `id`, not `name` + allow: # required `allow:` block + actors: { group: admins } # references a group by name + actions: [schema_apply] + target_branch_scope: protected + + - id: team-can-merge-to-protected + allow: + actors: { group: team } + actions: [branch_merge] + target_branch_scope: protected + + - id: team-can-read-write-unprotected + allow: + actors: { group: team } + actions: [read, change] + branch_scope: unprotected +``` + +To "block unreviewed schema applies," you don't write a deny rule — you just don't grant `schema_apply` to that group. Default-deny does the rest. + +Scope rules (a rule's `allow` block may use **at most one**): + +- `branch_scope: any | protected | unprotected` — for `read`, `export`, `change` (matches the source branch). +- `target_branch_scope: any | protected | unprotected` — for `schema_apply`, `branch_create`, `branch_delete`, `branch_merge` (matches the destination branch). + +### Validate, test, explain + +```bash +# Compile Cedar + check the cluster's applied policies +omnigraph policy validate --cluster . + +# Run declarative test cases +omnigraph policy test --cluster . --tests policy.tests.yaml + +# Debug a single decision +omnigraph policy explain \ + --actor act-alice \ + --action schema_apply \ + --target-branch main \ + --cluster . +``` + +### Test cases (`policy.tests.yaml`) + +```yaml +version: 1 # required; must be 1 +cases: + - id: alice-can-apply-schema # cases use `id`, not `name` + actor: act-alice + action: schema_apply + target_branch: main # schema_apply is target-branch scoped + expect: allow # `allow` / `deny` (not `permit`) + + - id: random-user-cannot-merge-to-main + actor: act-random + action: branch_merge + target_branch: main + expect: deny +``` + +Run `policy test` after every policy edit. Tests are cheap. + +## Multi-graph serving + +A `--cluster` server serves every graph in the applied cluster, each under `/graphs/{id}/...`. `GET /graphs` enumerates them (sorted by id), gated by the cluster-level `graph_list` action — even under `--unauthenticated`, topology stays closed until a `[cluster]` policy grants it. `omnigraph graphs list` mirrors it (remote servers only). + +Policy attaches at two levels via `cluster.yaml` `applies_to`: +- `[]` — per-graph rules (`read`, `change`, `branch_*`, `schema_apply`, `invoke_query`). +- `[cluster]` — server-level rules (`graph_list`). + +There is no runtime add/remove of graphs — edit `cluster.yaml`, `cluster apply`, restart. + +## Server + Policy Together + +When the server is running with a policy file: +1. Every request resolves the actor from the bearer token (the client cannot set actor identity) and checks it against Cedar rules. +2. Unauthorized requests return `403 Forbidden`. +3. The CLI doesn't bypass policy when it connects over HTTP — it's enforced at the server. Enforcement is also engine-wide, so CLI direct-engine writes and embedded SDK consumers hit the same gate. + +Setup ops (`init`, `load`) write storage directly. With a policy configured they still flow through the engine-layer enforce gate for the actor you pass via `--as` (or `operator.actor` in `~/.omnigraph/config.yaml`); gate the raw storage layer too (S3 bucket ACLs, object locks) if the bucket is shared. + +## Cluster-Booted Servers + +`omnigraph-server --cluster ` is the only boot source (covered above). It serves the cluster's **applied revision**: `cluster apply` changes take effect on the next restart (no hot reload), and boot is fail-fast with named remedies for missing/pending/tampered state. Bearer tokens and bind stay process-level (env/flags). See `references/cluster.md`. diff --git a/skills/omnigraph/references/stored-queries.md b/skills/omnigraph/references/stored-queries.md new file mode 100644 index 0000000..02aaf75 --- /dev/null +++ b/skills/omnigraph/references/stored-queries.md @@ -0,0 +1,54 @@ +# Stored-Query Registries + +A **stored query** is a `.gq` query that the *server* loads, type-checks at startup, and exposes by name — without ever accepting ad-hoc query source from the client. It's how you publish a vetted, typed query surface to remote callers and MCP tools. + +This is a server-side feature introduced in **v0.6.1**. It is distinct from CLI `aliases:` (see [`aliases.md`](aliases.md)): an alias is local client ergonomics; a stored query is a server-published, policy-gated endpoint. + +## Declaring stored queries (`cluster.yaml`) + +Stored queries are declared in the cluster's `cluster.yaml` — every `query ` in the listed `.gq` files registers: + +```yaml +graphs: + : + schema: schema.pg + queries: queries/ # discover every `query ` in queries/*.gq +``` + +`queries` also accepts an explicit file list (`[a.gq, b.gq]`) or a fine-grained `name: { file: … }` map; an unparseable `.gq` or a duplicate query name across files fails `cluster validate`. `cluster apply` publishes them to the content-addressed catalog, and the `--cluster` server type-checks and serves every applied query. Every applied query is listed (per-query `mcp:`/expose flags are a planned phase). + +## CLI + +```bash +omnigraph queries validate # type-check every stored query against the live schema (offline; opens the graph; exits non-zero on drift) +omnigraph queries list # print the addressed graph's registry: query names and typed params +``` + +- `validate` catches schema drift **without restarting the server** — run it after a `schema apply` or before deploying a config change. The server also runs this check at startup and **refuses to boot** on drift or on a duplicate MCP tool name. +- `validate` opens the graph (address with `--store ` or a positional URI); `list` reads the addressed graph's catalog. +- `queries` is distinct from `lint` — `lint` validates a single `.gq` file you point it at; `queries validate` validates the registry the server will actually serve. + +## HTTP surface + +| Route | Gate | Purpose | +|-------|------|---------| +| `GET /graphs/{id}/queries` | `read` | Typed tool catalog of the served queries. Graph-wide (branch-independent; `read` authorized against `main`). | +| `POST /graphs/{id}/queries/{name}` | `invoke_query` (+ `change` for a stored mutation) | Invoke a named query. Body carries params only — **never** `.gq` source. A stored mutation cannot target a `snapshot` (`400`); a param type error is a structured `400` naming the param. | + +`?branch=` / `?snapshot=` query params apply to `POST /graphs/{id}/queries/{name}` reads; branch/snapshot access stays enforced by the inner `read`/`change` gate (`invoke_query` itself is graph-scoped, not branch-scoped). + +## Policy gating (`invoke_query`) + +- **`invoke_query`** is a per-graph Cedar action gating the whole stored-query invocation surface. Grant it like any other action (see [`server-policy.md`](server-policy.md)). +- **Stored mutations are double-gated:** the caller needs `invoke_query` to reach the query **and** `change` for the write. An actor with `invoke_query` but not `change` gets `403` on a stored mutation. +- **Deny == unknown:** for a caller *lacking* `invoke_query`, a denial and an unknown query name return the **same 404** (identical body) — the catalog can't be probed. A caller who *holds* `invoke_query` may still get a `403` from the inner gate for a query it can't `read`/`change`, so existence is visible to grant-holders by design. +- **Default-deny mode** (bearer tokens, no `policy.file`) permits only `read`, so *every* `/graphs/{id}/queries/{name}` call returns `404` until an `invoke_query` rule is configured. + +## MCP exposure + +Every applied query is listed in `GET /graphs/{id}/queries` as a typed MCP tool. Per-query exposure controls (`mcp.expose`, `tool_name`) are a planned phase — there is no per-query `mcp:` flag in cluster mode today. + +## Note on per-query authorization + +The catalog is **not** Cedar-filtered per query yet: a caller with `read` but not `invoke_query` can *list* a query it cannot *invoke* (invocation would 404). Per-query authorization is future work; for now the catalog is a discovery surface and `invoke_query` is the invocation gate. +