From 0d273c5641f31b9cd5c15bfe10f2281617145296 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Tue, 21 Apr 2026 20:29:40 +0200 Subject: [PATCH 01/38] docs: ADR 0001 + Phase 1-4 implementation plans Pluggable storage backend, network access, and emergent domain classification. Introduces MemoryStore + Embedder traits, PgMemoryStore alongside SqliteMemoryStore, HTTP MCP + API key auth, and HDBSCAN-based domain clustering. Phase 5 federation deferred to a follow-up ADR. - docs/adr/0001-pluggable-storage-and-network-access.md -- Accepted - docs/plans/0001-phase-1-storage-trait-extraction.md - docs/plans/0002-phase-2-postgres-backend.md - docs/plans/0003-phase-3-network-access.md - docs/plans/0004-phase-4-emergent-domain-classification.md - docs/prd/001-getting-centralized-vestige.md -- source RFC --- ...01-pluggable-storage-and-network-access.md | 303 ++++ .../0001-phase-1-storage-trait-extraction.md | 1026 ++++++++++++ docs/plans/0002-phase-2-postgres-backend.md | 1269 +++++++++++++++ docs/plans/0003-phase-3-network-access.md | 1435 +++++++++++++++++ ...-phase-4-emergent-domain-classification.md | 883 ++++++++++ docs/prd/001-getting-centralized-vestige.md | 751 +++++++++ 6 files changed, 5667 insertions(+) create mode 100644 docs/adr/0001-pluggable-storage-and-network-access.md create mode 100644 docs/plans/0001-phase-1-storage-trait-extraction.md create mode 100644 docs/plans/0002-phase-2-postgres-backend.md create mode 100644 docs/plans/0003-phase-3-network-access.md create mode 100644 docs/plans/0004-phase-4-emergent-domain-classification.md create mode 100644 docs/prd/001-getting-centralized-vestige.md diff --git a/docs/adr/0001-pluggable-storage-and-network-access.md b/docs/adr/0001-pluggable-storage-and-network-access.md new file mode 100644 index 0000000..c150c70 --- /dev/null +++ b/docs/adr/0001-pluggable-storage-and-network-access.md @@ -0,0 +1,303 @@ +# ADR 0001: Pluggable Storage Backend, Network Access, and Emergent Domains + +**Status**: Accepted +**Date**: 2026-04-21 +**Related**: [docs/prd/001-getting-centralized-vestige.md](../prd/001-getting-centralized-vestige.md) + +--- + +## Context + +Vestige v2.x runs as a per-machine local process: stdio MCP transport, SQLite + +FTS5 + USearch HNSW in `~/.vestige/`, fastembed locally for embeddings. This is +ideal for single-machine single-agent use but blocks three real needs: + +- **Multi-machine access** -- same memory brain from laptop, desktop, server +- **Multi-agent access** -- multiple AI clients against one store concurrently +- **Future federation** -- syncing memory between decentralized nodes (MOS / + Threefold grid) + +SQLite's single-writer model and lack of a native network protocol make it +unsuitable as a centralized server. PostgreSQL + pgvector collapses our three +storage layers (SQLite, FTS5, USearch) into one engine with MVCC concurrency, +auth, and replication. + +Separately, Vestige today has no notion of domain or project scope -- all memories +share one namespace. For a multi-machine brain, users want soft topical +boundaries ("dev", "infra", "home") without manual tenanting. HDBSCAN clustering +on embeddings produces these boundaries from the data itself. + +The PRD at `docs/prd/001-getting-centralized-vestige.md` sketches the full design. +This ADR records the architectural decisions and resolves the open questions from +that document. + +--- + +## Decision + +Introduce two new trait boundaries, a network transport layer, and a domain +classification module. All four changes ship in parallel phases. + +**Trait boundaries:** + +1. `MemoryStore` -- single trait covering CRUD, hybrid search, FSRS scheduling, + graph edges, and domains. One big trait, not four. +2. `Embedder` -- separate trait for text-to-vector encoding. Storage never calls + fastembed directly. Callers (cognitive engine locally, HTTP server remotely) + compute embeddings and pass them into the store. + +**Backends:** + +- `SqliteMemoryStore` -- existing code refactored behind the trait, no behavior + change. +- `PgMemoryStore` -- new, using sqlx + pgvector + tsvector. Selectable at runtime + via `vestige.toml`. + +**Network:** + +- MCP over Streamable HTTP on the existing Axum server. +- API key auth middleware (blake3-hashed, stored in `api_keys` table). +- Dashboard uses the same API keys for login, then signed session cookies for + subsequent requests. + +**Domain classification:** + +- HDBSCAN clustering over embeddings to discover domains automatically. +- Soft multi-domain assignment -- raw similarity scores stored per memory, every + domain above a threshold is assigned. +- Conservative drift handling -- propose splits/merges, never auto-apply. + +--- + +## Architecture Overview + +### Component Breakdown + +1. **`Embedder` trait** (new module `crates/vestige-core/src/embedder/`) + - `async fn embed(&self, text: &str) -> Result>` + - `fn model_name(&self) -> &str` + - `fn dimension(&self) -> usize` + - Impls: `FastembedEmbedder` (local ONNX, today), future `JinaEmbedder`, + `OpenAiEmbedder`, etc. + - Stays pluggable forever -- no lock-in to fastembed or to nomic-embed-text. + +2. **`MemoryStore` trait** (new module `crates/vestige-core/src/storage/trait.rs`) + - One trait, ~25 methods across CRUD, search, FSRS, graph, domain sections. + - Uses `trait_variant::make` to generate a `Send`-bound variant for + `Arc` in Axum/tokio contexts. + - The 29 cognitive modules operate exclusively through this trait. No direct + SQLite or Postgres access from the modules. + +3. **`SqliteMemoryStore`** (refactor of existing `crates/vestige-core/src/storage/sqlite.rs`) + - Existing rusqlite + FTS5 + USearch code, wrapped behind the trait. + - Add `domains TEXT[]` equivalent (JSON-encoded array column in SQLite). + - Add `domain_scores` JSON column. + - No behavioral change for current users. + +4. **`PgMemoryStore`** (new `crates/vestige-core/src/storage/postgres.rs`) + - `sqlx::PgPool` with compile-time checked queries. + - pgvector HNSW index for vector search, tsvector + GIN for FTS. + - Native array columns for `domains`, JSONB for `domain_scores` and `metadata`. + - Hybrid search via RRF (Reciprocal Rank Fusion) in a single SQL query. + +5. **Model registry** + - Per-database table `embedding_model` with `(name, dimension, hash, created_at)`. + - Both backends refuse writes from an embedder whose signature doesn't match + the registered row. + - Model swap = `vestige migrate --reembed --model=`, O(n) cost, explicit. + +6. **`DomainClassifier` cognitive module** (new `crates/vestige-core/src/neuroscience/domain_classifier.rs`) + - Owns the HDBSCAN discovery pass (using the `hdbscan` crate). + - Computes soft-assignment scores for every memory against every centroid. + - Stores raw `domain_scores: HashMap` per memory; thresholds into + the `domains` array using `assign_threshold` (default 0.65). + - Runs discovery on demand (`vestige domains discover`) or during dream + consolidation passes. + +7. **HTTP MCP transport** (extension of existing Axum server in `crates/vestige-mcp/src/`) + - New route `POST /mcp` for Streamable HTTP JSON-RPC. + - New route `GET /mcp` for SSE (for long-running operations). + - REST API under `/api/v1/` for direct HTTP clients (non-MCP integrations). + - Auth middleware validates `Authorization: Bearer ...` or `X-API-Key`, plus + signed session cookies for dashboard. + +8. **Key management** (new `crates/vestige-mcp/src/auth/`) + - `api_keys` table -- blake3-hashed keys, scopes, optional domain filter, + last-used timestamp. + - CLI: `vestige keys create|list|revoke`. + +9. **FSRS review event log** (future-proofing for federation) + - New table `review_events` -- append-only `(memory_id, timestamp, rating, + prior_state, new_state)`. + - Current `scheduling` table becomes a materialized view over the event log + (reconstructible from events). + - Phase 5 federation merges event logs, not derived state. Zero cost today, + avoids lock-in tomorrow. + +### Data Flow + +**Local mode (stdio MCP, unchanged UX):** +``` +stdio client -> McpServer -> CognitiveEngine -> FastembedEmbedder -> MemoryStore (SQLite) +``` + +**Server mode (HTTP MCP, new):** +``` +Remote client -> Axum HTTP -> auth middleware -> CognitiveEngine + -> FastembedEmbedder (server-side) -> MemoryStore (Postgres) +``` + +The cognitive engine is backend-agnostic. The embedder and the store are both +swappable. The 7-stage search pipeline (overfetch -> cross-encoder rerank -> +temporal -> accessibility -> context match -> competition -> spreading activation) +sits *above* the `MemoryStore` trait and works identically against either backend. + +### Orthogonality of HDBSCAN and Reranking + +HDBSCAN and the cross-encoder reranker solve different problems and both stay: + +- **HDBSCAN** discovers domains by clustering embeddings. Runs once per discovery + pass. Produces centroids. Used to *filter* search candidates, not to rank them. +- **Cross-encoder reranker** (Jina Reranker v1 Turbo) scores query-document pairs + at search time. Runs on every search. Produces ranked results. + +Domain membership is a filter applied before or during overfetch; reranking runs +on whatever candidate set survives the filter. + +--- + +## Alternatives Considered + +| Alternative | Pros | Cons | Why Not | +|-------------|------|------|---------| +| Split into 4 traits (`MemoryStore + SchedulingStore + GraphStore + DomainStore`) | Cleaner interface segregation | Every module holds 4 trait objects, coordinates transactions across them | One trait is fine in Rust; extract sub-traits later if a genuine need appears | +| Embedding computed inside the backend | Simpler call sites for callers | Backend becomes aware of embedding models; can't support remote clients without local fastembed | Keep storage pure; separate `Embedder` trait handles pluggability | +| Unconstrained pgvector `vector` (no dimension) | Flexible for model swaps | HNSW still needs fixed dims at index creation; hides a meaningful migration as "silent" | Fixed dimension per install, explicit `--reembed` migration | +| Dashboard separate auth (cookies only, no keys) | Simpler dashboard UX | Two auth systems to maintain | Shared API keys with session cookie layer on top | +| Auto-tuned `assign_threshold` targeting an unclassified ratio | Adapts to corpus | Hard to debug ("why did this memory change domain?"); magical | Static 0.65 default, config-tunable, dashboard shows `domain_scores` for manual retuning | +| Aggressive drift (auto-reassign memories whose scores drifted) | Always up-to-date domains | Breaks user muscle memory; silent reshuffling | Conservative: always propose, user accepts | +| CRDTs for federation state | Mathematically clean merges | Massive complexity, performance cost, overkill | Defer; design FSRS as event log now so any future sync model works | + +--- + +## Consequences + +### Positive + +- Single memory brain accessible from every machine. +- Multi-agent concurrent access via Postgres MVCC. +- Natural topical scoping emerges from data, not manual tenants. +- Future embedding model swaps are a config + migration, not a rewrite. +- Federation has a clean on-ramp (event log merge) without committing now. +- The `Embedder` / `MemoryStore` split unlocks other storage backends later + (Redis, Qdrant, Iroh-backed blob store, etc.) with minimal work. + +### Negative + +- Operating a Postgres instance is more work than managing a SQLite file. +- Users who stay on SQLite gain nothing from this ADR (but lose nothing either). +- Migration (`vestige migrate --from sqlite --to postgres`) is a sensitive + operation for users with months of memories -- needs strong testing. +- HDBSCAN + re-soft-assignment runs in O(n) over all embeddings. At 100k+ + memories this starts to matter; manageable but not free. + +### Risks + +- **Trait abstraction leaks**: a cognitive module might need backend-specific + behavior (e.g., Postgres triggers for tsvector). Mitigation: keep such logic + inside the backend impl; the trait stays pure. + Escalation: if a module genuinely cannot express what it needs through the + trait, the trait grows, not the module bypasses. +- **Embedding model drift**: users on older fastembed versions silently + producing slightly different vectors after a fastembed upgrade. Mitigation: + model hash in the registry, refuse mismatched writes, surface a clear error. +- **Auth misconfiguration**: a user binds to `0.0.0.0` without setting + `auth.enabled = true`. Mitigation: refuse to start with non-localhost bind + and auth disabled. Hard error, not a warning. +- **Re-clustering feedback loop**: dream consolidation proposes re-clusters, + which the user accepts, which changes classifications, which affects future + retrievals, which affect future dreams. Mitigation: cap re-cluster frequency + (every 5th dream by default), require explicit user acceptance of proposals. +- **Cross-domain spreading activation weight (0.5 default)**: arbitrary choice; + could be too aggressive or too lax. Mitigation: config-tunable; instrument + retrieval quality metrics in the dashboard so the user sees impact. + +--- + +## Resolved Decisions (from Q&A) + +| # | Question | Resolution | +|---|----------|------------| +| 1 | Trait granularity | Single `MemoryStore` trait | +| 2 | Embedding on insert | Caller provides; separate `Embedder` trait for pluggability | +| 3 | pgvector dimension | Fixed per install, derived from `Embedder::dimension()` at schema init | +| 4 | Federation sync | Defer algorithm; store FSRS reviews as append-only event log now | +| 5 | Dashboard auth | Shared API keys + signed session cookie | +| 6 | HDBSCAN `min_cluster_size` | Default 10; user reruns with `--min-cluster-size N`; no auto-sweep | +| 7 | Domain drift | Conservative -- always propose splits/merges, never auto-apply | +| 8 | Cross-domain spreading activation | Follow with decay factor 0.5 (tunable) | +| 9 | Assignment threshold | Static 0.65 default, config-tunable, raw `domain_scores` stored for introspection | + +--- + +## Implementation Plan + +Five phases, each independently shippable. + +### Phase 1: Storage trait extraction +- Define `MemoryStore` and `Embedder` traits in `vestige-core`. +- Refactor `SqliteMemoryStore` to implement `MemoryStore`; no behavior change. +- Refactor `FastembedEmbedder` to implement `Embedder`. +- Add `embedding_model` registry table; enforce consistency on write. +- Add `domains TEXT[]`-equivalent and `domain_scores` JSON columns to SQLite + (empty for all existing rows). +- Convert all 29 cognitive modules to operate via the traits. +- **Acceptance**: existing test suite passes unchanged. Zero warnings. + +### Phase 2: PostgreSQL backend +- `PgMemoryStore` with sqlx, pgvector, tsvector. +- sqlx migrations (`crates/vestige-core/migrations/postgres/`). +- Backend selection via `vestige.toml` `[storage]` section. +- `vestige migrate --from sqlite --to postgres` command. +- `vestige migrate --reembed` command for model swaps. +- **Acceptance**: full test suite runs green against Postgres with a testcontainer. + +### Phase 3: Network access +- Streamable HTTP MCP route on Axum (`POST /mcp`, `GET /mcp` for SSE). +- REST API under `/api/v1/`. +- API key table + blake3 hashing + `vestige keys create|list|revoke`. +- Auth middleware (Bearer, X-API-Key, session cookie). +- Refuse non-localhost bind without auth enabled. +- **Acceptance**: MCP client over HTTP works from a second machine; dashboard + login flow works; unauth requests return 401. + +### Phase 4: Emergent domain classification +- `DomainClassifier` module using the `hdbscan` crate. +- `vestige domains discover|list|rename|merge` CLI. +- Automatic soft-assignment pipeline (compute `domain_scores` on ingest, threshold + into `domains`). +- Re-cluster every Nth dream consolidation (default 5); surface proposals in the + dashboard. +- Context signals (git repo, IDE) as soft priors on classification. +- Cross-domain spreading activation with 0.5 decay. +- **Acceptance**: on a corpus of 500+ mixed memories, discover produces sensible + clusters; search scoped to a domain returns tightly relevant results. + +### Phase 5: Federation (future, explicitly out of scope for this ADR's +acceptance) +- Node discovery (Mycelium / mDNS). +- Memory sync protocol over append-only review events and LWW-per-UUID for + memory records. +- Explicit follow-up ADR before any code. + +--- + +## Open Questions + +None at ADR acceptance time. Follow-up items that are *implementation choices*, +not architectural: + +- Precise cross-domain decay weight (start at 0.5, instrument, tune) +- Dashboard histogram of `domain_scores` (UX design detail) +- Whether to gate Postgres behind a Cargo feature flag (`postgres-backend`) or + always compile it in (lean toward feature flag to keep SQLite-only builds small) diff --git a/docs/plans/0001-phase-1-storage-trait-extraction.md b/docs/plans/0001-phase-1-storage-trait-extraction.md new file mode 100644 index 0000000..9960462 --- /dev/null +++ b/docs/plans/0001-phase-1-storage-trait-extraction.md @@ -0,0 +1,1026 @@ +# Phase 1 Plan: Storage Trait Extraction + +**Status**: Draft +**Depends on**: none +**Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 1) + +--- + +## Scope + +### In scope + +- Introduce a new module `crates/vestige-core/src/storage/memory_store.rs` defining: + - `LocalMemoryStore` base trait (Sync + 'static) + - `MemoryStore` Send-bound alias generated via `#[trait_variant::make(MemoryStore: Send)]` + - Supporting data types referenced by the trait: `MemoryRecord`, `SchedulingState`, `SearchQuery`, `SearchResult`, `MemoryEdge`, `Domain`, `ClassificationResult`, `StoreStats`, `HealthStatus`, `MemoryStoreError`. +- Introduce a new module `crates/vestige-core/src/embedder/` defining: + - `Embedder` async trait with `embed`, `model_name`, `dimension` plus `model_hash` (for the registry) and optional `embed_batch` with a default implementation. + - Move/adapt the existing `EmbeddingService` impl into a new struct `FastembedEmbedder` that implements `Embedder`. +- Refactor `Storage` (existing `crates/vestige-core/src/storage/sqlite.rs`) into `SqliteMemoryStore`: + - Keep the struct, the `writer`/`reader` `Mutex` pair, the `FSRSScheduler`, and the USearch `VectorIndex`. + - Rename the type alias `Storage` to `SqliteMemoryStore` with a `pub type Storage = SqliteMemoryStore;` alias for backward source compatibility during the transition. (The trait method surface is the new public contract.) + - Implement `LocalMemoryStore` by wrapping existing synchronous `rusqlite` methods inside `async fn` bodies that call a small `spawn_blocking`-or-inline adapter. Bodies MAY block; the `async fn` signature exists because `LocalMemoryStore` is async. +- Add a `schema_version = 12` migration that introduces two schema additions: + 1. `embedding_model` registry table (one-row constraint enforced in code). + 2. Two new TEXT columns on `knowledge_nodes`: `domains TEXT NOT NULL DEFAULT '[]'` and `domain_scores TEXT NOT NULL DEFAULT '{}'` (both JSON-encoded). +- Enforce model registry on every write path: on the first non-empty embedding write the model signature is recorded; subsequent writes whose `Embedder::model_name()` / `dimension()` / `model_hash()` disagree must fail with `MemoryStoreError::ModelMismatch` before touching the DB. +- Audit all 29 cognitive modules under `crates/vestige-core/src/neuroscience/` and `crates/vestige-core/src/advanced/` to confirm they hold no direct `rusqlite::Connection` references, no `Storage` struct field, and no SQL strings. Any that do get refactored to take `&dyn LocalMemoryStore` (local-only modules) or `&Arc` (modules crossing `await` points). +- Add unit tests alongside each new trait method and integration tests in `tests/phase_1/`. + +### Out of scope + +- Implementing `PgMemoryStore` on sqlx + pgvector -- that is Phase 2. +- `vestige migrate --from sqlite --to postgres` and `vestige migrate --reembed` -- Phase 2. +- MCP over Streamable HTTP, API key middleware, `api_keys` table, `vestige keys create|list|revoke` -- Phase 3. +- `DomainClassifier` module, HDBSCAN clustering, `vestige domains discover|list|rename|merge` CLI, incremental soft-assignment, cross-domain spreading activation decay -- Phase 4. +- Federation, mycelium/mDNS node discovery, review event log table -- Phase 5. +- Removing the `pub type Storage = SqliteMemoryStore;` compatibility alias -- that cleanup happens at the end of Phase 4 when no consumers still spell the old name. + +## Prerequisites + +### Current code state + +- Single concrete type `Storage` in `crates/vestige-core/src/storage/sqlite.rs` (4592 lines, 216 public symbols on the impl blocks, approximately 85 public methods) is the only storage surface the crate exposes. +- `EmbeddingService` in `crates/vestige-core/src/embeddings/local.rs` holds the fastembed singleton. No trait exists; callers type-erase via `&EmbeddingService`. +- Migrations live in `crates/vestige-core/src/storage/migrations.rs`; the current head is v11. +- All cognitive modules in `neuroscience/` and `advanced/` are pure (verified by `grep rusqlite|Connection::|execute\(|prepare\(` returning no matches in those trees). They operate on `KnowledgeNode`, `Vec`, `ConnectionRecord`, etc. passed in by the caller. +- `vestige-mcp` consumes `Arc` in `crates/vestige-mcp/src/server.rs` and every tool under `crates/vestige-mcp/src/tools/`. These call sites will type-check unchanged after the alias is introduced because the trait methods preserve the exact signatures of the existing `pub fn` on `Storage`. +- Test count reported in `CLAUDE.md`: 758 tests (406 mcp + 352 core). This is the no-regression target. + +### Required crates (add via `cargo add` under `crates/vestige-core`) + +| Crate | Version | Why | +|-------|---------|-----| +| `trait-variant` | `0.1` | Generates the `Send`-bound `MemoryStore` alias from `LocalMemoryStore` so `Arc` works under tokio/axum without hand-writing two traits. Listed in PRD section "Crate Dependencies (new)" under Phase 1. | +| `blake3` | `1` | `Embedder::model_hash() -> [u8; 32]` uses blake3 to stabilise the "model signature" stored in the `embedding_model` registry. Already slated for Phase 3 auth; pulling it forward costs nothing and avoids a second migration to add a hash column. | +| `async-trait` | `0.1` | Not strictly required with `trait-variant` on MSRV 1.91 (RPITIT is stable), but used for one utility trait (`EmbedderExt`) that carries a default `embed_batch` body. OPTIONAL; see Open Implementation Questions below. | + +No changes to `vestige-mcp/Cargo.toml` are required for Phase 1 -- the new trait lives in `vestige-core` and the mcp crate continues to depend on the `SqliteMemoryStore` concrete type (via the `Storage` alias) until Phase 2 introduces backend selection. + +## Deliverables + +1. `crates/vestige-core/src/storage/memory_store.rs` -- `LocalMemoryStore` + `MemoryStore` traits and supporting types. +2. `crates/vestige-core/src/storage/mod.rs` -- updated exports and module wiring. +3. `crates/vestige-core/src/storage/sqlite.rs` -- `Storage` renamed to `SqliteMemoryStore`, `impl LocalMemoryStore for SqliteMemoryStore` block, enforcement hooks for the model registry, serde of `domains` / `domain_scores` columns. +4. `crates/vestige-core/src/storage/migrations.rs` -- `MIGRATION_V12_UP` adding `embedding_model` table and `domains`, `domain_scores` columns. +5. `crates/vestige-core/src/embedder/mod.rs` -- `Embedder` trait and re-exports. +6. `crates/vestige-core/src/embedder/fastembed.rs` -- `FastembedEmbedder` implementation. +7. `crates/vestige-core/src/embeddings/local.rs` -- retained; `EmbeddingService` kept as the underlying fastembed holder; `FastembedEmbedder` wraps it. +8. `crates/vestige-core/src/lib.rs` -- new `pub mod embedder;` + re-exports for `MemoryStore`, `LocalMemoryStore`, `Embedder`, `FastembedEmbedder`, and the data types. +9. `tests/phase_1/trait_round_trip.rs` -- integration test: round-trip of every trait method through `SqliteMemoryStore`. +10. `tests/phase_1/embedding_model_registry.rs` -- integration test: first-write registers, mismatch refuses, dimension mismatch refuses. +11. `tests/phase_1/domain_column_migration.rs` -- integration test: a v11 DB upgraded to v12 reads `domains=[]` and `domain_scores={}` for all existing rows. +12. `tests/phase_1/cognitive_module_isolation.rs` -- integration test: every cognitive module compiles and executes against an `Arc` without touching `SqliteMemoryStore` concretely. +13. `tests/phase_1/send_bound_variant.rs` -- integration test: an `Arc` can be moved across `tokio::spawn`. +14. Updated `tests/phase_1/mod.rs` (if the dir already uses a module layout) or individual `[[test]]` entries in `tests/e2e/Cargo.toml` as needed -- see "Test Plan" for the exact layout. + +## Detailed Task Breakdown + +### D1. Trait + supporting types (`memory_store.rs`) + +- **File**: `crates/vestige-core/src/storage/memory_store.rs` (new). +- **Depends on**: `trait-variant` crate added under vestige-core, `chrono`, `serde_json`, `uuid`, `thiserror` (all already in Cargo.toml). +- **Signatures**: + +```rust +//! Backend-agnostic memory store trait. +//! +//! This is the single abstraction every cognitive module sits above. It is +//! intentionally flat: one trait, ~25 methods, no sub-traits. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +// ---------------------------------------------------------------------------- +// ERROR +// ---------------------------------------------------------------------------- + +/// Error returned by every `LocalMemoryStore` / `MemoryStore` method. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum MemoryStoreError { + #[error("not found: {0}")] + NotFound(String), + + #[error("backend error: {0}")] + Backend(String), + + #[error( + "embedding model mismatch: store registered {registered_name} (dim {registered_dim}, \ + hash {registered_hash}), embedder is {actual_name} (dim {actual_dim}, hash {actual_hash})" + )] + ModelMismatch { + registered_name: String, + registered_dim: usize, + registered_hash: String, + actual_name: String, + actual_dim: usize, + actual_hash: String, + }, + + #[error("invalid input: {0}")] + InvalidInput(String), + + #[error("initialization error: {0}")] + Init(String), +} + +impl From for MemoryStoreError { + fn from(e: crate::storage::StorageError) -> Self { + use crate::storage::StorageError as S; + match e { + S::NotFound(s) => MemoryStoreError::NotFound(s), + S::Database(e) => MemoryStoreError::Backend(e.to_string()), + S::Io(e) => MemoryStoreError::Backend(e.to_string()), + S::InvalidTimestamp(s) => MemoryStoreError::Backend(format!("invalid timestamp: {s}")), + S::Init(s) => MemoryStoreError::Init(s), + } + } +} + +pub type MemoryStoreResult = std::result::Result; + +// ---------------------------------------------------------------------------- +// DATA TYPES +// ---------------------------------------------------------------------------- + +/// Backend-agnostic memory record. +/// +/// Phase 1 intentionally keeps this type independent of `KnowledgeNode` to +/// avoid dragging 30+ legacy fields through the trait surface. The SQLite +/// backend converts between `MemoryRecord` and `KnowledgeNode` at the +/// boundary. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRecord { + pub id: Uuid, + /// Empty = unclassified. Populated in Phase 4. + pub domains: Vec, + /// Raw similarity per domain centroid. Empty until Phase 4 runs clustering. + pub domain_scores: HashMap, + pub content: String, + pub node_type: String, + pub tags: Vec, + pub embedding: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, + pub metadata: serde_json::Value, +} + +/// FSRS-6 scheduling state, one row per memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SchedulingState { + pub memory_id: Uuid, + pub stability: f64, + pub difficulty: f64, + pub retrievability: f64, + pub last_review: Option>, + pub next_review: Option>, + pub reps: u32, + pub lapses: u32, +} + +/// Hybrid search request. +#[derive(Debug, Clone, Default)] +pub struct SearchQuery { + pub domains: Option>, + pub text: Option, + pub embedding: Option>, + pub tags: Option>, + pub node_types: Option>, + pub limit: usize, + pub min_retrievability: Option, +} + +#[derive(Debug, Clone)] +pub struct SearchResult { + pub record: MemoryRecord, + pub score: f64, + pub fts_score: Option, + pub vector_score: Option, +} + +/// Edge in the spreading-activation graph. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryEdge { + pub source_id: Uuid, + pub target_id: Uuid, + pub edge_type: String, + pub weight: f64, + pub created_at: DateTime, +} + +/// A topical domain (populated in Phase 4). Phase 1 only needs the type to +/// shape the trait surface; discover/classify are Phase 4 work. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Domain { + pub id: String, + pub label: String, + pub centroid: Vec, + pub top_terms: Vec, + pub memory_count: usize, + pub created_at: DateTime, +} + +/// Result of classifying one vector against all known domains. +#[derive(Debug, Clone)] +pub struct ClassificationResult { + pub scores: HashMap, + pub domains: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StoreStats { + pub total_memories: usize, + pub memories_with_embeddings: usize, + pub total_edges: usize, + pub total_domains: usize, + pub registered_model_name: Option, + pub registered_model_dim: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum HealthStatus { + Healthy, + Degraded { reason: String }, + Unavailable { reason: String }, +} + +// ---------------------------------------------------------------------------- +// EMBEDDING MODEL SIGNATURE +// ---------------------------------------------------------------------------- + +/// Snapshot of the embedding model that was used to write vectors into the +/// store. Persisted in the `embedding_model` table; compared on every write +/// before the vector is accepted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModelSignature { + pub name: String, + pub dimension: usize, + /// Lowercase hex-encoded blake3 hash, 64 chars. + pub hash: String, +} + +// ---------------------------------------------------------------------------- +// TRAIT +// ---------------------------------------------------------------------------- + +/// The single storage abstraction. `trait_variant::make` auto-generates a +/// `MemoryStore` alias with `Send`-bound return futures so `Arc` +/// works in tokio/axum contexts. +#[trait_variant::make(MemoryStore: Send)] +pub trait LocalMemoryStore: Sync + 'static { + // --- Lifecycle --- + async fn init(&self) -> MemoryStoreResult<()>; + async fn health_check(&self) -> MemoryStoreResult; + + // --- Embedding model registry --- + async fn registered_model(&self) -> MemoryStoreResult>; + async fn register_model(&self, sig: &ModelSignature) -> MemoryStoreResult<()>; + + // --- CRUD --- + async fn insert(&self, record: &MemoryRecord) -> MemoryStoreResult; + async fn get(&self, id: Uuid) -> MemoryStoreResult>; + async fn update(&self, record: &MemoryRecord) -> MemoryStoreResult<()>; + async fn delete(&self, id: Uuid) -> MemoryStoreResult<()>; + + // --- Search --- + async fn search(&self, query: &SearchQuery) -> MemoryStoreResult>; + async fn fts_search(&self, text: &str, limit: usize) -> MemoryStoreResult>; + async fn vector_search( + &self, + embedding: &[f32], + limit: usize, + ) -> MemoryStoreResult>; + + // --- FSRS Scheduling --- + async fn get_scheduling( + &self, + memory_id: Uuid, + ) -> MemoryStoreResult>; + async fn update_scheduling(&self, state: &SchedulingState) -> MemoryStoreResult<()>; + async fn get_due_memories( + &self, + before: DateTime, + limit: usize, + ) -> MemoryStoreResult>; + + // --- Graph (spreading activation) --- + async fn add_edge(&self, edge: &MemoryEdge) -> MemoryStoreResult<()>; + async fn get_edges( + &self, + node_id: Uuid, + edge_type: Option<&str>, + ) -> MemoryStoreResult>; + async fn remove_edge(&self, source: Uuid, target: Uuid) -> MemoryStoreResult<()>; + async fn get_neighbors( + &self, + node_id: Uuid, + depth: usize, + ) -> MemoryStoreResult>; + + // --- Domains (Phase 1: stubs return empty; full impl in Phase 4) --- + async fn list_domains(&self) -> MemoryStoreResult>; + async fn get_domain(&self, id: &str) -> MemoryStoreResult>; + async fn upsert_domain(&self, domain: &Domain) -> MemoryStoreResult<()>; + async fn delete_domain(&self, id: &str) -> MemoryStoreResult<()>; + /// Phase 1: returns `Ok(vec![])` since no centroids exist. Phase 4 wires + /// the full soft-assignment pass. + async fn classify(&self, embedding: &[f32]) -> MemoryStoreResult>; + + // --- Bulk / Maintenance --- + async fn count(&self) -> MemoryStoreResult; + async fn get_stats(&self) -> MemoryStoreResult; + async fn vacuum(&self) -> MemoryStoreResult<()>; +} +``` + +- **Behavior notes**: + - Every method returns `MemoryStoreResult`; the trait never exposes `rusqlite::Error`. + - `LocalMemoryStore` requires `Sync + 'static` so `Arc` is usable. The auto-generated `MemoryStore` alias adds `Send` bounds on the returned `impl Future`. + - `register_model` is idempotent: writing the same signature twice is `Ok(())`. Writing a different signature after one is registered returns `MemoryStoreError::ModelMismatch`. + - `classify` on Phase 1 returns `Ok(vec![])` and MUST NOT error; cognitive modules call it and Phase 4 will flesh it out without changing the signature. + - `upsert_domain` / `delete_domain` / `list_domains` / `get_domain` operate against a `domains` table that is empty until Phase 4 populates it. Phase 1 still exposes the methods so Phase 2 can implement them against Postgres in one shot. + - `get_neighbors(node_id, depth)` with `depth == 0` returns just `(node, 1.0)` if the node exists, otherwise `NotFound`. `depth > 0` performs breadth-first expansion over edges, weight = product of edge weights along the shortest path discovered, capped at `max_neighbors = 256` to prevent runaway expansion. + +--- + +### D2. Storage module wiring (`storage/mod.rs`) + +- **File**: `crates/vestige-core/src/storage/mod.rs`. +- **Depends on**: D1. +- **Signatures / diff**: + +```rust +//! Storage Module +//! +//! Backend-agnostic memory store abstraction plus SQLite reference impl. + +mod memory_store; +mod migrations; +mod sqlite; + +pub use memory_store::{ + ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, + MemoryStore, MemoryStoreError, MemoryStoreResult, ModelSignature, SchedulingState, + SearchQuery, SearchResult, StoreStats, +}; +pub use migrations::MIGRATIONS; +pub use sqlite::{ + ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, InsightRecord, + IntentionRecord, Result, SmartIngestResult, SqliteMemoryStore, StateTransitionRecord, + StorageError, +}; + +/// Backwards-compatibility alias. Retained until Phase 4 completes so every +/// existing `Arc` call site keeps compiling. Scheduled for removal +/// once no downstream source file references it. +pub type Storage = SqliteMemoryStore; +``` + +- **Behavior notes**: + - The alias MUST be a `pub type` (not a re-export), because several tool files pattern on `vestige_core::Storage` through `use` statements and we want to keep them compiling verbatim. This has zero runtime cost. + - `StorageError` stays exported for the 29 existing inherent-method callers; the trait exposes `MemoryStoreError` and provides `From`. + +--- + +### D3. Rename + trait impl in `sqlite.rs` + +- **File**: `crates/vestige-core/src/storage/sqlite.rs`. +- **Depends on**: D1, D2, D4 (for schema columns), D5/D6 (to have `Embedder` to accept on `insert`). +- **Signatures (key excerpts)**: + +```rust +pub struct SqliteMemoryStore { + writer: Mutex, + reader: Mutex, + scheduler: Mutex, + #[cfg(feature = "embeddings")] + embedding_service: EmbeddingService, + #[cfg(feature = "vector-search")] + vector_index: Mutex, + #[cfg(feature = "embeddings")] + query_cache: Mutex>>, + /// Cached model signature. `None` until the first embedding is written. + registered_model: std::sync::RwLock>, +} + +impl SqliteMemoryStore { + pub fn new(db_path: Option) -> MemoryStoreResult { /* existing body, Result converted */ } + + /// Internal: convert a row into a `MemoryRecord` (new mapping reading + /// `domains` / `domain_scores` JSON columns). + fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result { /* ... */ } + + /// Internal: given a `MemoryRecord` plus an optional embedding, enforce + /// the registered model signature and return a `MemoryStoreError` if + /// the embedder would produce a mismatched vector. + fn enforce_model( + &self, + incoming: Option<&ModelSignature>, + ) -> MemoryStoreResult<()> { /* ... */ } +} + +impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { + async fn init(&self) -> MemoryStoreResult<()> { /* no-op; migrations run in `new` */ Ok(()) } + + async fn health_check(&self) -> MemoryStoreResult { + // SELECT 1; check vector index loaded; check embedding_model presence. + } + + async fn registered_model(&self) -> MemoryStoreResult> { + let cached = self.registered_model.read().map_err(|_| MemoryStoreError::Init("registered_model rwlock poisoned".into()))?.clone(); + if cached.is_some() { + return Ok(cached); + } + // Fall through to DB read... + } + + async fn register_model(&self, sig: &ModelSignature) -> MemoryStoreResult<()> { + // INSERT OR IGNORE; if a row exists and differs, return ModelMismatch. + } + + async fn insert(&self, record: &MemoryRecord) -> MemoryStoreResult { + if let Some(vec) = &record.embedding { + // Caller is REQUIRED to have called register_model first (or the + // store auto-registers on the first embedded write -- see + // "embedding_model_registry.rs" test). + let derived = ModelSignature { /* from cache or from record.metadata */ }; + self.enforce_model(Some(&derived))?; + if vec.len() != derived.dimension { + return Err(MemoryStoreError::InvalidInput( + format!("embedding length {} != registered dimension {}", vec.len(), derived.dimension), + )); + } + } + // Delegate to a private `insert_record_blocking` helper that is the + // current `ingest`/`update_node_content` body, rewritten to accept a + // `MemoryRecord` and to also write `domains` / `domain_scores` JSON. + } + + // ... remaining ~24 methods follow the same pattern: convert inputs, + // call the existing synchronous body, convert outputs. +} +``` + +- **SQL** (covered in full in D4 below). +- **Behavior notes**: + - The `async fn` bodies are allowed to be synchronous under the hood (rusqlite is blocking). We do NOT wrap in `spawn_blocking` for Phase 1 -- the current `Storage` is already used from synchronous code paths (CLI, MCP stdio handler) and forcing the tokio runtime is a Phase 2 concern when we also add sqlx. The trait simply lifts the synchronous body into an `async fn` so the signatures match the trait. MSRV 1.91 supports async fn in trait via `trait_variant::make`. + - `insert` preserves the current FSRS initialization logic (stability, difficulty, next_review, etc.) -- the new code path converts `MemoryRecord.metadata` back into `IngestInput`-equivalent fields when needed. All existing inherent methods (`ingest`, `smart_ingest`, `mark_reviewed`, ...) remain on `SqliteMemoryStore` untouched; the trait impl calls into them. + - `registered_model` cache is an `RwLock>`. Invalidated on schema reset. Never mutated after first population until an explicit `--reembed` migration (Phase 2) takes the RwLock exclusively and writes a new row. + - `enforce_model` returns `Ok(())` if no model is registered yet AND `incoming.is_none()` (no-embedding write). Returns `Ok(())` if no model is registered and `incoming.is_some()` after calling `register_model`. Returns `Err(ModelMismatch)` if registered and they disagree. + - `domains` / `domain_scores` serialization uses `serde_json::to_string` on write and `serde_json::from_str` on read. Empty vec -> `"[]"`, empty map -> `"{}"`. `NULL` in the DB is treated as the empty value for pre-migration rows. + - Every existing inherent method is kept verbatim. The trait impl dispatches to them. This is the "no behavior change" guarantee. + +--- + +### D4. Schema migration V12 + +- **File**: `crates/vestige-core/src/storage/migrations.rs`. +- **Depends on**: D2. +- **SQL**: + +```sql +-- Migration V12: embedding model registry + per-memory domain columns. + +-- 1. Embedding model registry. Single logical row; the (id = 1) constraint is +-- enforced in code via `register_model` (SQLite CHECK on a single-row +-- table is uglier than a constraint we already enforce in Rust). +CREATE TABLE IF NOT EXISTS embedding_model ( + id INTEGER PRIMARY KEY CHECK (id = 1), + name TEXT NOT NULL, + dimension INTEGER NOT NULL, + hash TEXT NOT NULL, -- lowercase hex blake3 + created_at TEXT NOT NULL +); + +-- 2. Per-memory domain columns (JSON TEXT; SQLite has no native arrays). +ALTER TABLE knowledge_nodes ADD COLUMN domains TEXT NOT NULL DEFAULT '[]'; +ALTER TABLE knowledge_nodes ADD COLUMN domain_scores TEXT NOT NULL DEFAULT '{}'; + +-- 3. Index on the domains JSON column to enable `LIKE '%"dev"%'`-style +-- filter in Phase 4. Kept lightweight here; Postgres will use GIN. +CREATE INDEX IF NOT EXISTS idx_nodes_domains ON knowledge_nodes(domains); +CREATE INDEX IF NOT EXISTS idx_nodes_domain_scores ON knowledge_nodes(domain_scores); + +-- 4. Domains catalogue (empty until Phase 4 populates). +CREATE TABLE IF NOT EXISTS domains ( + id TEXT PRIMARY KEY, + label TEXT NOT NULL, + centroid BLOB, -- f32 vector, raw bytes + top_terms TEXT NOT NULL DEFAULT '[]', + memory_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_domains_created_at ON domains(created_at); + +UPDATE schema_version SET version = 12, applied_at = datetime('now'); +``` + +- **Rust changes** to `migrations.rs`: + +```rust +pub const MIGRATIONS: &[Migration] = &[ + // ... V1..V11 unchanged ... + Migration { + version: 12, + description: "Phase 1: embedding_model registry, domains/domain_scores columns, domains table", + up: MIGRATION_V12_UP, + }, +]; + +const MIGRATION_V12_UP: &str = r#"...SQL above..."#; +``` + +- **Behavior notes**: + - Idempotent: `ALTER TABLE ... ADD COLUMN` on SQLite is not idempotent by default, but the `apply_migrations` driver only applies migrations whose version > current. A user who has already applied V12 never sees the SQL again. + - The `CHECK (id = 1)` on `embedding_model` is the only one-row guardrail -- all inserts go through `register_model` which uses `INSERT OR IGNORE INTO embedding_model (id, ...) VALUES (1, ...)` followed by a `SELECT` to detect mismatch. + - `centroid BLOB` stores the f32 vector using the same `Embedding::to_bytes()` format used in `node_embeddings`, for consistency. + +--- + +### D5. Embedder trait (`embedder/mod.rs`) + +- **File**: `crates/vestige-core/src/embedder/mod.rs` (new). +- **Depends on**: `blake3` crate added to vestige-core. +- **Signatures**: + +```rust +//! Text-to-vector encoding trait. Pluggable per-install. + +use std::fmt::Debug; + +mod fastembed; + +pub use fastembed::FastembedEmbedder; + +/// Error returned by every `Embedder` method. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum EmbedderError { + #[error("embedder initialization failed: {0}")] + Init(String), + #[error("embedding generation failed: {0}")] + EmbedFailed(String), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +pub type EmbedderResult = std::result::Result; + +/// Pluggable embedder. The storage layer NEVER calls fastembed directly; +/// callers compute vectors via this trait and pass them into `MemoryStore`. +#[trait_variant::make(Embedder: Send)] +pub trait LocalEmbedder: Sync + 'static { + async fn embed(&self, text: &str) -> EmbedderResult>; + + fn model_name(&self) -> &str; + + fn dimension(&self) -> usize; + + /// Stable blake3 hash of (model_name || dimension || optional weights + /// digest if available). Lowercase hex, 64 chars. + /// + /// Used by `MemoryStore::register_model` to detect silent model drift + /// (e.g. a fastembed minor upgrade that changes vector output). + fn model_hash(&self) -> String; + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>> { + // Default: sequential. Backends with native batching override this. + let mut out = Vec::with_capacity(texts.len()); + for t in texts { + out.push(self.embed(t).await?); + } + Ok(out) + } + + /// Returns the `ModelSignature` describing this embedder. Convenience + /// wrapper over the three accessors above. + fn signature(&self) -> crate::storage::ModelSignature { + crate::storage::ModelSignature { + name: self.model_name().to_string(), + dimension: self.dimension(), + hash: self.model_hash(), + } + } +} +``` + +- **Behavior notes**: + - The `embed_batch` default implementation is non-trivial only in that backends with genuine batching override it. The `FastembedEmbedder` overrides to call `EmbeddingService::embed_batch`. + - `model_hash()` is intentionally a function, not a constant, so backends with configurable weights (a future `OnnxEmbedder` that loads an arbitrary file) can hash the file bytes into the signature. + - `Embedder` (the `Send` variant) is what cognitive modules bind against when they hold `Arc`. `LocalEmbedder` is available for single-threaded callers (CLI, tests). + +--- + +### D6. FastembedEmbedder impl (`embedder/fastembed.rs`) + +- **File**: `crates/vestige-core/src/embedder/fastembed.rs` (new). +- **Depends on**: D5, existing `crate::embeddings::local::EmbeddingService`. +- **Signatures**: + +```rust +use super::{EmbedderError, EmbedderResult, LocalEmbedder}; +use crate::embeddings::{EMBEDDING_DIMENSIONS, EmbeddingService, matryoshka_truncate}; + +pub struct FastembedEmbedder { + inner: EmbeddingService, + cached_hash: std::sync::OnceLock, +} + +impl FastembedEmbedder { + pub fn new() -> Self { + Self { + inner: EmbeddingService::new(), + cached_hash: std::sync::OnceLock::new(), + } + } + + fn compute_hash(name: &str, dim: usize) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(name.as_bytes()); + hasher.update(&(dim as u64).to_le_bytes()); + // fastembed's ONNX bytes are not directly accessible at runtime; we + // use `(name, dim, static fastembed crate version)` as the + // signature. If fastembed ever changes its output deterministically + // between minor versions, bumping the crate version triggers a + // mismatch -- which is exactly the drift we want to detect. + hasher.update(env!("CARGO_PKG_VERSION").as_bytes()); + hasher.finalize().to_hex().to_string() + } +} + +impl Default for FastembedEmbedder { + fn default() -> Self { Self::new() } +} + +impl LocalEmbedder for FastembedEmbedder { + async fn embed(&self, text: &str) -> EmbedderResult> { + let emb = self + .inner + .embed(text) + .map_err(|e| EmbedderError::EmbedFailed(e.to_string()))?; + Ok(emb.vector) + } + + fn model_name(&self) -> &str { self.inner.model_name() } + + fn dimension(&self) -> usize { EMBEDDING_DIMENSIONS } + + fn model_hash(&self) -> String { + self.cached_hash + .get_or_init(|| Self::compute_hash(self.inner.model_name(), EMBEDDING_DIMENSIONS)) + .clone() + } + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>> { + let embs = self + .inner + .embed_batch(texts) + .map_err(|e| EmbedderError::EmbedFailed(e.to_string()))?; + Ok(embs.into_iter().map(|e| e.vector).collect()) + } +} +``` + +- **Behavior notes**: + - `EmbeddingService` is kept as the fastembed singleton holder; `FastembedEmbedder` is a thin trait adapter. Existing callers of `EmbeddingService` continue to work during the transition. + - `model_hash` is deterministic for a given `(model_name, EMBEDDING_DIMENSIONS, vestige-core version)` triple. This is the drift detector the ADR calls out under "Risks: Embedding model drift". + - `matryoshka_truncate` is already applied inside `EmbeddingService::embed`, so the vectors returned here are the 256-dim Matryoshka-truncated L2-normalized vectors that the rest of the stack expects. + +--- + +### D7. `lib.rs` re-exports + +- **File**: `crates/vestige-core/src/lib.rs`. +- **Depends on**: D1, D2, D5, D6. +- **Diff** (inserted alongside the existing `pub mod storage;` re-exports): + +```rust +pub mod embedder; + +pub use embedder::{Embedder, EmbedderError, EmbedderResult, FastembedEmbedder, LocalEmbedder}; + +pub use storage::{ + ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, + MemoryStore, MemoryStoreError, MemoryStoreResult, ModelSignature, SchedulingState, + SearchQuery, SearchResult, SqliteMemoryStore, Storage, StoreStats, + // Existing re-exports retained: + ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, InsightRecord, + IntentionRecord, Result, SmartIngestResult, StateTransitionRecord, StorageError, +}; +``` + +- **Behavior notes**: + - `Storage` remains a top-level re-export so `use vestige_core::Storage;` keeps working in `vestige-mcp` without changes. Post-Phase-4 cleanup will grep the downstream crates and replace. + +--- + +### D8. Cognitive module audit + +- **Files**: all under `crates/vestige-core/src/neuroscience/*.rs` and `crates/vestige-core/src/advanced/*.rs` -- 21 source files. +- **Depends on**: D1..D7. +- **Work**: perform the following grep-gate BEFORE and AFTER the refactor: + +``` +Grep pattern: "rusqlite|Connection::|execute\\(|prepare\\(|&Storage|SqliteMemoryStore" +Expected in neuroscience/ and advanced/ BEFORE: only a single comment-only hit in `neuroscience/active_forgetting.rs:54` referencing `Storage::suppress_memory` in a doc comment. +Expected AFTER: zero hits that reference `SqliteMemoryStore` concretely. References through `&dyn LocalMemoryStore` or `&Arc` are acceptable. +``` + +- **Behavior notes**: + - Current state: the 29 cognitive modules are already pure (they take nodes/vectors/connections as arguments, not a `&Storage`). No refactor is required for their bodies. + - The only work is the `consolidation/sleep.rs` and `consolidation/phases.rs` path, which in the current codebase accepts `&Storage`. These get rewritten to accept `&dyn LocalMemoryStore` (callable from sync contexts) or `&Arc` (callable from async contexts). See file inventory below. + - Actual rewrites (expected number): 3-5 functions across `consolidation/sleep.rs` and `consolidation/mod.rs`. All trait-object refactors; no logic changes. + - `cognitive.rs` in `vestige-mcp` uses `storage.get_all_connections()`. Because `SqliteMemoryStore` keeps `get_all_connections` as an inherent method AND implements `MemoryStore::get_edges`, both call styles keep compiling. `cognitive.rs` does not need to change in Phase 1. + +--- + +### D9. Backwards-compatible inherent methods on `SqliteMemoryStore` + +- **File**: `crates/vestige-core/src/storage/sqlite.rs`. +- **Depends on**: D3. +- **Behavior notes**: + - Every one of the 85 existing `pub fn` on `Storage` (e.g. `ingest`, `smart_ingest`, `mark_reviewed`, `hybrid_search_filtered`, `save_intention`, `save_insight`, `save_connection`, `apply_rac1_cascade`, ...) stays as an inherent method on `SqliteMemoryStore`. The Phase 1 refactor ONLY adds the trait impl; it does NOT remove any method, rename any field, or change any SQL. + - Internal writes that previously embedded `INSERT INTO knowledge_nodes (...)` statements gain two more columns (`domains = '[]'`, `domain_scores = '{}'`) in the INSERT list. These are non-optional columns after migration V12, and their DEFAULT is `'[]'`/`'{}'` respectively, so ALTER behaves correctly for pre-existing rows but INSERT statements need to either list the defaults explicitly or rely on the DB default. Plan: explicitly write `'[]'` and `'{}'` in every `INSERT INTO knowledge_nodes` statement to avoid surprises if a future migration drops the DEFAULT. + +--- + +## Test Plan + +### Unit tests (colocated, `#[cfg(test)] mod tests` at end of each source file) + +Every public trait method on `LocalMemoryStore` gets at least one unit test, exercised through the `SqliteMemoryStore` impl. The unit test file is `crates/vestige-core/src/storage/sqlite.rs` (inside the existing `mod tests`). + +- `vestige_core::storage::sqlite::tests::trait_init_is_idempotent` -- calling `LocalMemoryStore::init` twice returns `Ok(())` both times. +- `vestige_core::storage::sqlite::tests::trait_health_check_reports_healthy_on_fresh_db` -- asserts `HealthStatus::Healthy` on a fresh in-memory DB. +- `vestige_core::storage::sqlite::tests::trait_register_model_first_write_succeeds` -- after registering a signature, `registered_model()` returns it. +- `vestige_core::storage::sqlite::tests::trait_register_model_mismatched_write_refused` -- registering a second, different signature returns `MemoryStoreError::ModelMismatch`. +- `vestige_core::storage::sqlite::tests::trait_register_model_same_signature_idempotent` -- registering the same signature twice returns `Ok(())` both times. +- `vestige_core::storage::sqlite::tests::trait_insert_returns_uuid` -- `insert(record)` returns the UUID from the record. +- `vestige_core::storage::sqlite::tests::trait_insert_refuses_dimension_mismatch` -- inserting a record with a 512-dim vector into a store registered for 256 dims returns `MemoryStoreError::InvalidInput`. +- `vestige_core::storage::sqlite::tests::trait_get_missing_returns_none` -- `get(non_existent_uuid)` returns `Ok(None)`. +- `vestige_core::storage::sqlite::tests::trait_get_after_insert_round_trip` -- insert then get returns a record equal (by content/tags/type) to the input; `domains == []`, `domain_scores == {}`. +- `vestige_core::storage::sqlite::tests::trait_update_modifies_content` -- update with new content reflects in subsequent `get`. +- `vestige_core::storage::sqlite::tests::trait_delete_removes_record` -- `delete` then `get` returns `Ok(None)`. +- `vestige_core::storage::sqlite::tests::trait_search_combines_fts_and_vector` -- with one memory whose content matches by FTS and another by vector, `search` returns both, higher score for the exact content match. +- `vestige_core::storage::sqlite::tests::trait_fts_search_returns_tokens_match` -- verifies FTS path. +- `vestige_core::storage::sqlite::tests::trait_vector_search_returns_cosine_order` -- verifies ordering. +- `vestige_core::storage::sqlite::tests::trait_scheduling_round_trip` -- `update_scheduling` then `get_scheduling` returns equivalent state. +- `vestige_core::storage::sqlite::tests::trait_get_scheduling_missing_returns_none`. +- `vestige_core::storage::sqlite::tests::trait_get_due_memories_returns_in_order` -- inserts 3 records with different `next_review`, asserts older-due listed first. +- `vestige_core::storage::sqlite::tests::trait_add_edge_is_idempotent` -- adding the same edge twice does not duplicate. +- `vestige_core::storage::sqlite::tests::trait_get_edges_filters_by_type`. +- `vestige_core::storage::sqlite::tests::trait_remove_edge_deletes_single`. +- `vestige_core::storage::sqlite::tests::trait_get_neighbors_bfs_depth_zero_returns_self_only`. +- `vestige_core::storage::sqlite::tests::trait_get_neighbors_bfs_depth_two_expands` -- build A->B->C, get_neighbors(A, 2) returns {A, B, C}. +- `vestige_core::storage::sqlite::tests::trait_list_domains_empty_in_phase_1` -- Phase 1 has no clustering, so `list_domains()` returns `[]`. +- `vestige_core::storage::sqlite::tests::trait_upsert_then_get_domain_round_trip`. +- `vestige_core::storage::sqlite::tests::trait_delete_domain_idempotent`. +- `vestige_core::storage::sqlite::tests::trait_classify_with_no_domains_returns_empty` -- verifies Phase 1 stub behavior. +- `vestige_core::storage::sqlite::tests::trait_count_matches_insert_count`. +- `vestige_core::storage::sqlite::tests::trait_get_stats_reports_registered_model`. +- `vestige_core::storage::sqlite::tests::trait_vacuum_succeeds` -- runs and asserts no error. + +Every public method on `LocalEmbedder` gets at least one unit test under `crates/vestige-core/src/embedder/fastembed.rs`: + +- `vestige_core::embedder::fastembed::tests::embedder_reports_correct_name` -- `model_name()` contains "nomic". +- `vestige_core::embedder::fastembed::tests::embedder_reports_256_dimension`. +- `vestige_core::embedder::fastembed::tests::embedder_hash_is_stable` -- `model_hash()` called twice returns identical string. +- `vestige_core::embedder::fastembed::tests::embedder_hash_includes_crate_version` -- a synthetic test that asserts the hash contains the blake3 of `(name, 256, VERSION)`. +- `vestige_core::embedder::fastembed::tests::embedder_embed_smoke` -- gated on `#[cfg(feature = "embeddings")]`; asserts output length == 256. +- `vestige_core::embedder::fastembed::tests::embedder_embed_batch_matches_sequential` -- gated; assert batch result equals sequential result. +- `vestige_core::embedder::fastembed::tests::embedder_signature_matches_accessors`. + +Migration V12 unit tests under `crates/vestige-core/src/storage/migrations.rs`: + +- `vestige_core::storage::migrations::tests::v12_adds_embedding_model_table` -- apply V12 then assert `SELECT count(*) FROM sqlite_master WHERE name='embedding_model'` == 1. +- `vestige_core::storage::migrations::tests::v12_adds_domains_columns` -- assert `PRAGMA table_info(knowledge_nodes)` includes `domains` and `domain_scores`. +- `vestige_core::storage::migrations::tests::v12_default_values_empty_json` -- insert a row via raw SQL, read back, assert `domains == '[]'` and `domain_scores == '{}'`. +- `vestige_core::storage::migrations::tests::v12_is_replayable` -- rewind `schema_version` to 11, re-apply migrations, does not error (MUST use `CREATE TABLE IF NOT EXISTS`; `ALTER TABLE ADD COLUMN` will be skipped because the driver only re-runs migrations whose version > current -- already covered by `apply_migrations`). +- `vestige_core::storage::migrations::tests::v12_preserves_existing_rows` -- insert rows under V11 schema, upgrade to V12, assert `domains='[]'` on those rows. + +Supporting-type unit tests under `crates/vestige-core/src/storage/memory_store.rs`: + +- `vestige_core::storage::memory_store::tests::memory_store_error_from_storage_error` -- converts `StorageError::NotFound` to `MemoryStoreError::NotFound`. +- `vestige_core::storage::memory_store::tests::model_signature_serde_round_trip`. +- `vestige_core::storage::memory_store::tests::memory_record_serde_round_trip`. + +### Integration tests (`tests/phase_1/`) + +Each file is a standalone `[[test]]` target. The Cargo layout: + +- `tests/phase_1/Cargo.toml` with: + +```toml +[package] +name = "vestige-phase-1-tests" +version = "0.0.1" +edition = "2024" +publish = false + +[dependencies] +vestige-core = { path = "../../crates/vestige-core" } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tempfile = "3" +uuid = { version = "1", features = ["v4"] } +chrono = "0.4" +serde_json = "1" +rusqlite = { version = "0.38", features = ["bundled"] } +``` + +And added to the workspace `Cargo.toml` members. Each `.rs` file below is a `#[tokio::test]`-using integration test. + +#### `tests/phase_1/trait_round_trip.rs` + +- `round_trip::insert_get_update_delete` -- exercises CRUD via the trait. Inserts a record with `domains=[]`, gets it, asserts equality, updates content, deletes, asserts not found. +- `round_trip::scheduling_upsert_and_due_scan` -- upserts FSRS state for three memories with different `next_review`, calls `get_due_memories(Utc::now(), 10)`, asserts only past-due ones appear. +- `round_trip::edge_crud` -- add edge, list edges, remove edge, assert gone. +- `round_trip::search_hybrid_returns_results` -- insert three memories, embed one by content match only, one by semantic only, one by both, search with both `text` and `embedding`, assert all three appear with `fts_score`/`vector_score` correctly populated. +- `round_trip::count_and_stats_track_inserts` -- after 10 inserts, `count()` == 10 and `get_stats().total_memories` == 10. +- `round_trip::vacuum_after_deletes_reclaims` -- insert 50, delete 40, call `vacuum`, assert disk file size decreased (informational; test is lenient if VACUUM was a no-op). +- `round_trip::list_domains_empty_then_upsert_then_delete` -- Phase 1 has no discovery, but manual upsert/delete must work so Phase 2's Postgres impl can share the test. +- `round_trip::classify_with_no_domains_returns_empty` -- calls `classify(embedding)` on a fresh store, asserts `Vec<(String, f64)>` is empty. + +#### `tests/phase_1/embedding_model_registry.rs` + +- `model_registry::first_embedded_insert_auto_registers` -- fresh store; insert a record with a 256-dim vector using a `FastembedEmbedder`; subsequent `registered_model()` returns a `Some(ModelSignature)` with dim=256. +- `model_registry::second_insert_with_same_signature_succeeds`. +- `model_registry::second_insert_with_different_dimension_refused` -- register a 256-dim signature, try to insert a 512-dim vector, expect `MemoryStoreError::InvalidInput` (because dimension does not match registered). +- `model_registry::second_insert_with_different_model_name_refused` -- register signature A, call `register_model` with signature B (same dim, different name), expect `MemoryStoreError::ModelMismatch`. +- `model_registry::second_insert_with_different_hash_refused` -- register signature A, try to register signature A' with the same name and dim but a different hash, expect `MemoryStoreError::ModelMismatch`. +- `model_registry::no_embedding_insert_allowed_before_registration` -- a plain text memory without an embedding must insert successfully even when `registered_model()` is `None`. +- `model_registry::stats_reports_registered_model_after_first_write`. + +#### `tests/phase_1/domain_column_migration.rs` + +- `domain_columns::fresh_db_has_v12_schema` -- open a fresh store, query `PRAGMA table_info(knowledge_nodes)`, assert `domains` and `domain_scores` columns are present with the correct defaults. +- `domain_columns::v11_db_upgrades_cleanly` -- programmatically create a DB at V11 by running migrations up to V11 only, insert 5 rows, then invoke the V12 migration, assert all 5 rows now report `domains=='[]'` and `domain_scores=='{}'`. +- `domain_columns::empty_domains_serialize_as_brackets` -- insert a `MemoryRecord { domains: vec![], .. }`, then read the underlying SQLite row via a raw query, assert the stored value is `"[]"`, not `NULL`. +- `domain_columns::populated_domains_round_trip` -- insert a record with `domains=["dev","infra"]` and `domain_scores={"dev":0.82,"infra":0.71}`, read back via the trait, assert equality. +- `domain_columns::domains_table_exists` -- `SELECT name FROM sqlite_master WHERE name='domains'` returns one row. + +#### `tests/phase_1/cognitive_module_isolation.rs` + +- `cognitive_isolation::all_modules_compile_against_dyn_store` -- a test function that allocates a `let store: Arc = Arc::new(SqliteMemoryStore::new(...)?);`, then invokes a representative method from every cognitive module passing in records/vectors/edges it reads through the trait. The point is a compile-time gate: if any module still typed against `SqliteMemoryStore`, this would fail to compile. +- `cognitive_isolation::spreading_activation_traverses_via_trait` -- exercise `ActivationNetwork` seeded from `store.get_edges(...)` results. +- `cognitive_isolation::synaptic_tagging_consumes_records_via_trait` -- build `CapturedMemory` from `store.get(uuid)` and let the tagger compute retroactive importance. +- `cognitive_isolation::hippocampal_index_built_from_store` -- load memories via `store.fts_search`, build `HippocampalIndex`, assert queries against the index work. + +#### `tests/phase_1/send_bound_variant.rs` + +- `send_bound::arc_dyn_memory_store_moves_across_tokio_tasks` -- wrap `SqliteMemoryStore` in `Arc`, spawn 16 tokio tasks each inserting 10 memories, join all tasks, assert final `count() == 160`. This verifies the `#[trait_variant::make(MemoryStore: Send)]` emission actually produces a `Send`-bound future. +- `send_bound::concurrent_readers_one_writer` -- 32 concurrent readers calling `search` while one writer loops inserting; asserts no panics, no deadlocks, eventual consistency on `count`. + +#### `tests/phase_1/embedder_trait.rs` + +- `embedder::fastembed_implements_embedder_trait` -- `let e: Box = Box::new(FastembedEmbedder::new());` compiles and `e.dimension()` == 256. +- `embedder::signature_matches_memory_store_registry` -- take the signature from `Embedder::signature()`, register it via `MemoryStore::register_model`, assert `registered_model()` returns the same. + +### Regression verification + +- `cargo build -p vestige-core` -- zero warnings. +- `cargo build -p vestige-mcp` -- zero warnings. +- `cargo clippy --workspace --all-targets -- -D warnings` -- green. +- `cargo test -p vestige-core --lib` -- existing 352 core lib tests remain green. +- `cargo test -p vestige-mcp --lib` -- existing 406 mcp tests remain green. +- `cargo test -p vestige-core --lib storage::migrations::tests` -- explicitly invokes the migration tests added in Phase 1. +- `cargo test -p vestige-core --lib storage::sqlite::tests` -- invokes the trait-method unit tests added in Phase 1. +- `cargo test -p vestige-core --lib embedder::fastembed::tests` -- invokes embedder unit tests. +- `cargo test -p vestige-phase-1-tests --test trait_round_trip` -- Phase 1 integration test file 1. +- `cargo test -p vestige-phase-1-tests --test embedding_model_registry` -- Phase 1 integration test file 2. +- `cargo test -p vestige-phase-1-tests --test domain_column_migration` -- Phase 1 integration test file 3. +- `cargo test -p vestige-phase-1-tests --test cognitive_module_isolation` -- Phase 1 integration test file 4. +- `cargo test -p vestige-phase-1-tests --test send_bound_variant` -- Phase 1 integration test file 5. +- `cargo test -p vestige-phase-1-tests --test embedder_trait` -- Phase 1 integration test file 6. +- `cargo test -p vestige-phase-1-tests` -- convenience: runs all integration test binaries in the Phase 1 crate. +- `cargo test -p vestige-e2e` -- existing e2e harness runs unchanged; no new tests here but existing ones must pass. + +## Acceptance Criteria + +- [ ] `cargo build -p vestige-core` -- zero warnings. +- [ ] `cargo build -p vestige-mcp` -- zero warnings. +- [ ] `cargo build --workspace --all-targets` -- zero warnings. +- [ ] `cargo clippy --workspace --all-targets -- -D warnings` -- exits 0. +- [ ] `cargo test -p vestige-core` -- all 352 existing core tests plus new Phase 1 unit tests pass. +- [ ] `cargo test -p vestige-mcp` -- all 406 existing mcp tests pass, unchanged. +- [ ] `cargo test -p vestige-phase-1-tests` -- all Phase 1 integration tests pass. +- [ ] `cargo test -p vestige-e2e` -- existing e2e journey suite passes unchanged. +- [ ] Cumulative test count >= 758 (the pre-Phase-1 baseline) plus the new unit and integration additions. +- [ ] `git grep -n 'rusqlite::' crates/vestige-core/src/neuroscience/ crates/vestige-core/src/advanced/` -- zero hits (the single pre-existing doc-comment reference in `active_forgetting.rs` is acceptable and does not introduce SQL dependency; code references must be zero). +- [ ] `git grep -n 'SqliteMemoryStore' crates/vestige-core/src/neuroscience/ crates/vestige-core/src/advanced/` -- zero hits. +- [ ] `git grep -n 'fastembed::' crates/vestige-core/src/storage/sqlite.rs` -- zero hits (Storage must never call fastembed directly; embedding goes through the `Embedder` trait held on the caller side). +- [ ] `SqliteMemoryStore::insert` refuses a vector whose dimension disagrees with the registered model (returns `MemoryStoreError::InvalidInput`). +- [ ] `SqliteMemoryStore::register_model` returns `MemoryStoreError::ModelMismatch` when a second, different signature is provided after a first was already registered. +- [ ] After upgrading a V11 database to V12, every pre-existing row has `domains == "[]"` and `domain_scores == "{}"` with no NULLs. +- [ ] `#[trait_variant::make(MemoryStore: Send)]` compiles; `Arc` is movable across `tokio::spawn`. +- [ ] Migration V12 is idempotent on replay: `apply_migrations` rewound to V11, re-applied, succeeds without error. +- [ ] `vestige-core::storage::Storage` continues to resolve (via the `pub type` alias) at every current call site in `vestige-mcp`. +- [ ] The `embedding_model` table can only hold a single row (programmatic invariant -- verified by an integration test that attempts a second `INSERT INTO embedding_model (id = 1, ...)` and observes the CHECK-enforced uniqueness). +- [ ] `registered_model()` is cached on first read; no SELECT is issued against `embedding_model` after the first hit within the same process (verified by wrapping the reader in a counting proxy in a dedicated test). + +## Rollback Notes + +If Phase 1 fails mid-way, rollback granularity is per-deliverable and the DB can be downgraded by SQL. + +- **D1 (`memory_store.rs`)**: revert the new file. The trait has zero non-test consumers in Phase 1, so deletion is safe. +- **D2 (`storage/mod.rs`)**: revert to the prior export list. The only forward-facing identifier is the `pub type Storage = SqliteMemoryStore;` alias, which becomes `pub use sqlite::Storage;` again once `SqliteMemoryStore` is renamed back to `Storage`. +- **D3 (`sqlite.rs` rename + trait impl)**: revert the struct rename (`SqliteMemoryStore` -> `Storage`). The trait impl is a separate `impl` block and can be deleted wholesale. Inherent methods are unchanged and do not need to be touched. Net diff on revert: delete one `impl LocalMemoryStore for ...` block plus the two helper functions (`row_to_record`, `enforce_model`). +- **D4 (Migration V12)**: DOWN migration script: + +```sql +-- Phase 1 rollback: drop Phase 1 schema additions. +-- WARNING: this deletes any `domains` / `domain_scores` values stored under V12. +-- Execute ONLY when downgrading from V12 to V11 on a database where no Phase 4 +-- work has happened yet (otherwise you lose domain classifications). + +DROP TABLE IF EXISTS domains; +DROP INDEX IF EXISTS idx_nodes_domains; +DROP INDEX IF EXISTS idx_nodes_domain_scores; + +-- SQLite does not support DROP COLUMN before 3.35; the project's bundled +-- rusqlite uses 3.45+ (see `bundled-sqlite` feature). So the DROP COLUMN +-- form below is safe on every target platform. +ALTER TABLE knowledge_nodes DROP COLUMN domains; +ALTER TABLE knowledge_nodes DROP COLUMN domain_scores; + +DROP TABLE IF EXISTS embedding_model; + +UPDATE schema_version SET version = 11, applied_at = datetime('now'); +``` + + Operationally: the DOWN script is NOT included in the source migrations list (migrations are forward-only). If a rollback is required, it is applied manually via `sqlite3 vestige.db < rollback_v12.sql`. A backup via `storage.backup_to(...)` MUST be taken before the Phase 1 migration runs in production -- the `Storage::backup_to` method already exists (line 3903) and does not need changes. + +- **D5/D6 (`embedder/`)**: delete the module. `EmbeddingService` is untouched, so callers that still use it continue to work. The new `Embedder` trait has no pre-Phase-2 consumers. +- **D7 (`lib.rs`)**: revert the re-export additions. Zero downstream impact since the new symbols have no pre-Phase-2 consumers. +- **D8 (cognitive module audit)**: audit-only, no code changes. Nothing to roll back unless `consolidation/sleep.rs` was changed; if so, revert. +- **Crate-level considerations**: + - `trait-variant` must remain in `Cargo.toml` until every consumer of the trait alias has been reverted. Safe to leave in `[dependencies]` indefinitely; it has no runtime cost. + - `blake3` was going to be added in Phase 3 anyway; leaving it in on rollback is harmless. + - `rusqlite` version stays pinned; no bump required for Phase 1. + +## Open Implementation Questions + +Implementation-choice-only. Architectural questions are resolved in ADR 0001. + +1. **`MemoryRecord` vs `KnowledgeNode` as the trait currency.** + - Candidate A: `MemoryRecord` (new, lean type matching the PRD) -- chosen. + - Candidate B: use existing `KnowledgeNode` directly. + - **Recommendation: A.** `KnowledgeNode` carries 30+ FSRS / dual-strength / sentiment / temporal fields that bind callers to the SQLite columns. `MemoryRecord` is what `PgMemoryStore` and future backends will want. SQLite impl converts between the two at the boundary, which is a ~40-line `impl From for MemoryRecord` (and back) shim. Pays for itself in Phase 2. + +2. **`async fn` in traits vs `Box` via `async-trait`.** + - Candidate A: use `trait-variant` (RPITIT-based, MSRV 1.75+, our MSRV is 1.91). + - Candidate B: use `async-trait` (allocates one Box per call). + - **Recommendation: A.** `trait-variant` generates both the base `LocalMemoryStore` and the `Send`-bound `MemoryStore` from one definition, matches what the PRD explicitly calls out, and avoids the allocation overhead of boxed futures on every CRUD call. + +3. **Blocking SQLite under async signatures: spawn_blocking vs inline.** + - Candidate A: bodies call the existing sync `self.writer.lock()...` inline inside the `async fn`. + - Candidate B: bodies wrap in `tokio::task::spawn_blocking`. + - **Recommendation: A for Phase 1.** The current call sites are a mix of sync (CLI, bin/restore.rs) and async (MCP handlers). Introducing `spawn_blocking` would force a tokio runtime even for CLI use. Inline blocking under `async fn` is a documented pattern that compiles and works; under Phase 2 the Postgres impl uses `sqlx` which is natively async, and we can revisit Sqlite blocking policy at that point. Phase 1 priority is "no behavior change". + +4. **Where does `register_model` get called from: storage side auto-register, or caller-side explicit?** + - Candidate A: caller explicitly calls `store.register_model(embedder.signature())` once after `MemoryStore::init`. + - Candidate B: first `insert` with a vector auto-registers. + - **Recommendation: B.** The current code path (`Storage::ingest` -> `generate_embedding_for_node` -> INSERT into `node_embeddings`) has no explicit registration step and we want `--no behavior change`. Auto-register on first embedded write preserves the exact current UX. Callers who care (migration tooling, Phase 2 `--reembed`) can still call `register_model` explicitly; it is a no-op when idempotent. + +5. **`model_hash` content: fastembed ONNX bytes vs `(name, dim, crate_version)`.** + - Candidate A: hash the ONNX file bytes on disk (after model download). + - Candidate B: hash `(name, dim, vestige-core CARGO_PKG_VERSION)`. + - **Recommendation: B.** Fastembed caches ONNX files under `FASTEMBED_CACHE_PATH`; reading them from inside `FastembedEmbedder::new()` couples the embedder to fastembed's caching behavior and adds slow startup. Hashing `(name, dim, our crate version)` catches the "silent model drift between vestige versions" case the ADR calls out under Risks. Phase 2 can add a content-hashed `OnnxEmbedder` that loads any file and genuinely hashes it; the trait method signature stays the same. + +6. **`LocalMemoryStore` `Sync + 'static` or just `Sync`.** + - Candidate A: `Sync + 'static`. + - Candidate B: `Sync`. + - **Recommendation: A.** `'static` is required for `Arc` which is the target call pattern (Axum, MCP server, cognitive engine). Every impl we have in mind -- `SqliteMemoryStore`, `PgMemoryStore` -- holds owned state (connection pool, vector index), so `'static` is free. + +7. **Should trait methods appear on the SQLite impl instead of being separate?** + - Candidate A: keep the current ~85 inherent methods on `SqliteMemoryStore` AND add the `impl LocalMemoryStore` block. + - Candidate B: move every inherent method into the trait. + - **Recommendation: A.** Many inherent methods (e.g. `run_rac1_cascade_sweep`, `apply_rac1_cascade`, `save_insight`, `save_connection`, `preview_review`, `get_memory_subgraph`) have SQLite-specific semantics, transactional behavior, and call patterns that do not belong in a backend-agnostic trait. They will stay SQLite-only or be extracted into new traits in a post-Phase-4 cleanup. Phase 1's job is to expose the `~25 methods` contract the ADR specifies, not to retrofit the entire API. + +8. **Where do `Domain` bytes (centroid) live?** + - Candidate A: `BLOB` column on `domains` table. + - Candidate B: JSON-encoded array of f32 in a `TEXT` column. + - **Recommendation: A.** Consistent with how `node_embeddings.embedding` already stores vectors (little-endian f32 bytes via `Embedding::to_bytes`). JSON would triple the storage size and slow deserialization. The `Domain::centroid: Vec` field round-trips through the same codec. + +9. **Migration numbering when Phase 2 also wants to add a migration.** + - Candidate A: Phase 1 takes V12, Phase 2 takes V13. + - Candidate B: Phase 1 takes V12, Phase 2 re-shapes V12 to include its changes. + - **Recommendation: A.** Migrations are forward-only and append-only in this project. Phase 2 adds V13 (for `review_events` append-only table, if that lands in Phase 2 -- otherwise it is Phase 5 work). + +10. **Integration test crate location: sibling to `tests/e2e/` or inside `crates/vestige-core/tests/`.** + - Candidate A: new workspace member at `tests/phase_1/` (sibling to `tests/e2e/`). + - Candidate B: under `crates/vestige-core/tests/` (standard cargo integration-test layout). + - **Recommendation: A.** Matches the existing pattern of `tests/e2e/`, which is already a workspace member with its own `Cargo.toml`. Keeps the Phase 1 test binary outputs in a predictable location (`target/debug/deps/trait_round_trip-*`). Also avoids the build-graph cycle where `crates/vestige-core/tests/` would re-link everything under `vestige-core` each edit. + +### Critical Files for Implementation + +- /home/delandtj/prppl/vestige/crates/vestige-core/src/storage/memory_store.rs (new; contains the `LocalMemoryStore` / `MemoryStore` traits plus `MemoryRecord`, `SchedulingState`, `SearchQuery`, `SearchResult`, `MemoryEdge`, `Domain`, `ClassificationResult`, `StoreStats`, `HealthStatus`, `MemoryStoreError`, `ModelSignature`) +- /home/delandtj/prppl/vestige/crates/vestige-core/src/storage/sqlite.rs (rename `Storage` -> `SqliteMemoryStore`, add the `impl LocalMemoryStore` block and the `enforce_model` / `row_to_record` helpers; ~200 line diff on a 4592-line file) +- /home/delandtj/prppl/vestige/crates/vestige-core/src/storage/migrations.rs (append `Migration { version: 12, ... }` + `MIGRATION_V12_UP` constant; ~80 new lines) +- /home/delandtj/prppl/vestige/crates/vestige-core/src/embedder/mod.rs (new; `Embedder` + `LocalEmbedder` traits, `EmbedderError`, default `embed_batch`) +- /home/delandtj/prppl/vestige/crates/vestige-core/src/embedder/fastembed.rs (new; `FastembedEmbedder` implementation adapting the existing `EmbeddingService`) diff --git a/docs/plans/0002-phase-2-postgres-backend.md b/docs/plans/0002-phase-2-postgres-backend.md new file mode 100644 index 0000000..a372e27 --- /dev/null +++ b/docs/plans/0002-phase-2-postgres-backend.md @@ -0,0 +1,1269 @@ +# Phase 2 Plan: PostgreSQL Backend + +**Status**: Draft +**Depends on**: Phase 1 (MemoryStore + Embedder traits, embedding_model registry, domain columns) +**Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 2), docs/prd/001-getting-centralized-vestige.md + +--- + +## Scope + +### In scope + +- `PgMemoryStore` struct implementing the Phase 1 `MemoryStore` trait against `sqlx::PgPool`, including compile-time checked queries via `sqlx::query!` / `sqlx::query_as!`. +- First-class `pgvector` integration: typed `Vector` columns, HNSW index (`vector_cosine_ops`, `m = 16`, `ef_construction = 64`), and use of the cosine-distance operator `<=>`. +- First-class Postgres FTS: GENERATED `tsvector` column (`search_vec`) with `setweight` (A=content, B=node_type, C=tags), GIN index, and `websearch_to_tsquery` at query time. +- Hybrid search via Reciprocal Rank Fusion (RRF) expressed as a single SQL statement with CTEs for FTS and vector subqueries, with optional domain filter through array overlap (`&&`). +- sqlx migrations directory at `crates/vestige-core/migrations/postgres/`, numbered `{NNNN}_{name}.up.sql` / `{NNNN}_{name}.down.sql`, runnable by `sqlx::migrate!` at startup and by `sqlx-cli`. +- Offline query cache committed under `crates/vestige-core/.sqlx/` so a DATABASE_URL is not required at build time. +- Backend selection via `vestige.toml`: `[storage]` section with `backend = "sqlite" | "postgres"` plus the per-backend subsection (`[storage.sqlite]`, `[storage.postgres]`). Exclusive at compile time via `postgres-backend` feature, exclusive at runtime via the enum. +- CLI: `vestige migrate --from sqlite --to postgres --sqlite-path

--postgres-url ` -- streaming copy with progress output. +- CLI: `vestige migrate --reembed --model=` -- O(n) re-embed under a new `Embedder`, registry update, HNSW rebuild. +- Testcontainer-based integration tests using the `pgvector/pgvector:pg16` image, behind the `postgres-backend` feature so SQLite-only builds remain untouched. +- `PgMemoryStore` parity with `SqliteMemoryStore` across every public `MemoryStore` method defined in Phase 1. + +### Out of scope + +- Phase 3 (network access): HTTP MCP transport, API key auth, `vestige keys` CLI. The `api_keys` DDL is declared by Phase 3; Phase 2 does not create it. +- Phase 4 (emergent domain classification): `DomainClassifier`, HDBSCAN, discover / rename / merge CLI. Phase 2 provisions the `domains` and `domain_scores` columns and the `domains` table structure so Phase 4 slots in without further migration, but does not compute or classify. +- Phase 5 (federation): cross-node sync. The `review_events` table is declared in Phase 1; Phase 2 only references it where FSRS writes happen. +- Changes to the cognitive engine, Phase 1 traits, or the embedding pipeline itself. Phase 2 only adds a backend. +- SQLCipher parity for Postgres. Operator responsibility (TLS to Postgres, pgcrypto, disk-level encryption) is out of scope for this phase. + +--- + +## Prerequisites + +### Expected Phase 1 artifacts (consumed, not produced) + +Phase 2 treats all of the following as fixed interfaces. Each path is the expected Phase 1 location. + +- `crates/vestige-core/src/storage/mod.rs` -- re-exports the trait and the two concrete backends. +- `crates/vestige-core/src/storage/memory_store.rs` -- defines the `MemoryStore` trait (generated by `trait_variant::make` from `LocalMemoryStore`) with the full CRUD, search, FSRS, graph, and domain surface from the PRD. Phase 2 implements every method here. +- `crates/vestige-core/src/storage/types.rs` -- shared value types: `MemoryRecord`, `SchedulingState`, `SearchQuery`, `SearchResult`, `MemoryEdge`, `Domain`, `StoreStats`, `HealthStatus`. +- `crates/vestige-core/src/storage/error.rs` -- `StoreError` enum plus `pub type StoreResult = Result`. Phase 2 extends this with `StoreError::Postgres(sqlx::Error)` and `StoreError::Migrate(sqlx::migrate::MigrateError)` via `From` impls (the variants themselves MUST live behind `#[cfg(feature = "postgres-backend")]`). +- `crates/vestige-core/src/embedder/mod.rs` -- `Embedder` trait with `embed`, `model_name`, `dimension`, `model_hash`. Phase 2 calls `model_name()`, `dimension()`, and `model_hash()` for the registry. +- `crates/vestige-core/src/storage/sqlite.rs` -- `SqliteMemoryStore: MemoryStore`. Phase 2's `migrate --from sqlite --to postgres` uses this as the source. +- `crates/vestige-core/src/storage/registry.rs` -- `EmbeddingModelRegistry` abstraction that both backends implement. Phase 2 supplies a Postgres version writing to `embedding_model`. +- `crates/vestige-core/migrations/sqlite/` -- V12 (Phase 1) adds `domains TEXT` (JSON-encoded array), `domain_scores TEXT` (JSON), `embedding_model(name, dimension, hash, created_at)`, and `review_events(id, memory_id, timestamp, rating, prior_state, new_state)`. Phase 2 mirrors every column and table in Postgres. + +If any of the above is missing when Phase 2 starts, the first action is to surface the gap back to Phase 1 -- do NOT backfill a partial trait in Phase 2. + +### Required crates (declared in Phase 2, not installed by this doc) + +The agent running Phase 2 uses `cargo add` in `crates/vestige-core/` for each dependency below. Exact versions and feature flags: + +- `sqlx@0.8` with features `runtime-tokio`, `tls-rustls`, `postgres`, `uuid`, `chrono`, `json`, `migrate`, `macros`. Optional (gated by `postgres-backend`). +- `pgvector@0.4` with features `sqlx`. Optional (gated by `postgres-backend`). +- `deadpool` is NOT needed; `sqlx::PgPool` is the pool. +- `toml@0.8` (no features) for `vestige.toml` parsing. Moved to non-optional because both backends share the config surface. +- `figment@0.10` with features `toml`, `env` -- optional, only if Phase 1 has not already picked a config loader. If Phase 1 ships a loader, skip `figment` and reuse. +- `dirs@6` -- already a transitive `directories` dependency; reuse existing. +- `tokio-stream@0.1` (no features). Used by migrate commands for streamed iteration. +- `indicatif@0.17` (no features). Progress bars for the migrate CLI. +- `futures@0.3` with features `std`. Consumed by sqlx stream combinators. + +Dev-only (under `[dev-dependencies]` in `crates/vestige-core/Cargo.toml`, gated by `postgres-backend`): + +- `testcontainers@0.22` with features `blocking` off, `async` on (default). +- `testcontainers-modules@0.10` with features `postgres`. +- `tokio@1` features `macros`, `rt-multi-thread` (already present for core tests). +- `criterion@0.5` already present; add a new `[[bench]]` entry. + +Feature additions in `crates/vestige-core/Cargo.toml`: + +``` +[features] +postgres-backend = ["dep:sqlx", "dep:pgvector", "dep:tokio-stream", "dep:futures"] +``` + +`postgres-backend` is OFF by default. `default = ["embeddings", "vector-search", "bundled-sqlite"]` stays unchanged. `vestige-mcp` forwards a new feature `postgres-backend = ["vestige-core/postgres-backend"]`. + +### External tooling + +- PostgreSQL 16 or newer (uses `gen_random_uuid()` from `pgcrypto` bundled via `CREATE EXTENSION pgcrypto` in migration 0001; pgvector HNSW indexes require pgvector 0.5+). +- The `pgvector` extension installed in the target database (our migration issues `CREATE EXTENSION IF NOT EXISTS vector`). +- `sqlx-cli@0.8` installed on the developer machine for `cargo sqlx prepare --workspace` and `cargo sqlx migrate add` (not a build-time requirement once `.sqlx/` is committed). +- Docker or Podman reachable by the test harness for `testcontainers-modules::postgres` to spin up `pgvector/pgvector:pg16`. + +### Assumed Rust toolchain + +- Rust 2024 edition. +- MSRV 1.91 (per `CLAUDE.md`). `sqlx 0.8` is compatible. +- `rustflags` unchanged. No `nightly`-only features. + +--- + +## Deliverables + +1. Feature gate `postgres-backend` in `crates/vestige-core/Cargo.toml` and `crates/vestige-mcp/Cargo.toml` that cleanly disables all Postgres code paths when off. +2. `crates/vestige-core/src/storage/postgres/mod.rs` -- `PgMemoryStore` struct and `MemoryStore` trait impl (public entry point). +3. `crates/vestige-core/src/storage/postgres/pool.rs` -- `PgMemoryStore::connect(config)` and pool configuration. +4. `crates/vestige-core/src/storage/postgres/search.rs` -- RRF hybrid search query builder and row -> `SearchResult` mapping. +5. `crates/vestige-core/src/storage/postgres/migrations.rs` -- wraps `sqlx::migrate!("./migrations/postgres")` and surfaces typed errors. +6. `crates/vestige-core/src/storage/postgres/registry.rs` -- Postgres `EmbeddingModelRegistry` implementation writing `embedding_model`. +7. `crates/vestige-core/migrations/postgres/0001_init.up.sql` + `0001_init.down.sql` -- extensions, `memories`, `scheduling`, `edges`, `domains`, `embedding_model`, `review_events`, all indexes. +8. `crates/vestige-core/migrations/postgres/0002_hnsw.up.sql` + `0002_hnsw.down.sql` -- HNSW index creation separated so it can be `CREATE INDEX CONCURRENTLY` during reembed. +9. `crates/vestige-core/src/config.rs` -- `VestigeConfig`, `StorageConfig`, `SqliteConfig`, `PostgresConfig`, `EmbeddingsConfig`, plus a single `VestigeConfig::load(path: Option<&Path>)` returning `Result`. +10. `crates/vestige-core/src/storage/postgres/migrate_cli.rs` -- streaming SQLite-to-Postgres copy, domain-aware, with `indicatif` progress. +11. `crates/vestige-core/src/storage/postgres/reembed.rs` -- `ReembedPlan` and its driver; re-encodes all memories via a supplied `Embedder`, updates `embedding_model`, rebuilds HNSW. +12. `crates/vestige-mcp/src/bin/cli.rs` -- two new `clap` subcommands `Migrate` (union of `--from/--to` and `--reembed` variants, one subcommand or two, see Open Questions) wired to deliverables 10 and 11. +13. `crates/vestige-core/.sqlx/` -- offline query cache, committed. +14. `tests/phase_2/` -- six integration test files listed in the Test Plan. +15. `crates/vestige-core/benches/pg_hybrid_search.rs` -- Criterion benches for RRF search at 1k and 100k memories, gated by `postgres-backend`. +16. `docs/runbook/postgres.md` -- brief ops note covering extension install, `max_connections`, backup discipline, and rollback caveats. (Short; only required for the "rollback of migrate" deliverable.) + +--- + +## Detailed Task Breakdown + +### D1. `postgres-backend` feature gate + +- **File**: `crates/vestige-core/Cargo.toml`, `crates/vestige-mcp/Cargo.toml` +- **Depends on**: nothing; this is the first change. +- **Rust snippets**: + +```toml +# crates/vestige-core/Cargo.toml +[features] +default = ["embeddings", "vector-search", "bundled-sqlite"] +bundled-sqlite = ["rusqlite/bundled"] +encryption = ["rusqlite/bundled-sqlcipher"] +postgres-backend = [ + "dep:sqlx", + "dep:pgvector", + "dep:tokio-stream", + "dep:futures", +] + +[dependencies] +sqlx = { version = "0.8", default-features = false, features = [ + "runtime-tokio", "tls-rustls", "postgres", "uuid", "chrono", + "json", "migrate", "macros", +], optional = true } +pgvector = { version = "0.4", features = ["sqlx"], optional = true } +tokio-stream = { version = "0.1", optional = true } +futures = { version = "0.3", optional = true } +toml = "0.8" +indicatif = "0.17" +``` + +- **Behavior notes**: keep the two backends mutually compilable per `CLAUDE.md`. Every `use sqlx::...` sits under `#[cfg(feature = "postgres-backend")]`. Every module under `crates/vestige-core/src/storage/postgres/` carries `#![cfg(feature = "postgres-backend")]` as its file-level attribute. + +### D2. `PgMemoryStore` core struct + +- **File**: `crates/vestige-core/src/storage/postgres/mod.rs` +- **Depends on**: D1, Phase 1 `MemoryStore` trait and value types. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use pgvector::Vector; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::embedder::Embedder; +use crate::storage::error::{StoreError, StoreResult}; +use crate::storage::types::{ + Domain, HealthStatus, MemoryEdge, MemoryRecord, SchedulingState, + SearchQuery, SearchResult, StoreStats, +}; +use crate::storage::memory_store::LocalMemoryStore; + +pub mod migrations; +pub mod pool; +pub mod registry; +pub mod search; +pub mod migrate_cli; +pub mod reembed; + +/// Postgres-backed implementation of `MemoryStore`. +/// +/// Cheaply cloneable. Methods take `&self`; interior state lives inside +/// the `PgPool` (which already provides `Sync` via `Arc` internally). +#[derive(Clone)] +pub struct PgMemoryStore { + pool: PgPool, + embedding_dim: i32, + embedding_model: Arc, +} + +#[derive(Debug, Clone)] +pub struct EmbeddingModelDescriptor { + pub name: String, + pub dimension: i32, + pub hash: String, +} + +impl PgMemoryStore { + /// Construct a new store. Runs migrations, reads the registry, validates + /// that the embedder matches the registered model. + pub async fn connect( + url: &str, + max_connections: u32, + embedder: &dyn Embedder, + ) -> StoreResult; + + /// Low-level constructor for tests: supply an existing pool, skip migrate. + pub async fn from_pool( + pool: PgPool, + embedder: &dyn Embedder, + ) -> StoreResult; + + /// Accessor used by migrate/reembed CLI. + pub fn pool(&self) -> &PgPool { &self.pool } + + pub fn embedding_dim(&self) -> i32 { self.embedding_dim } +} + +#[trait_variant::make(crate::storage::memory_store::MemoryStore: Send)] +impl LocalMemoryStore for PgMemoryStore { + async fn init(&self) -> StoreResult<()>; + async fn health_check(&self) -> StoreResult; + + async fn insert(&self, record: &MemoryRecord) -> StoreResult; + async fn get(&self, id: Uuid) -> StoreResult>; + async fn update(&self, record: &MemoryRecord) -> StoreResult<()>; + async fn delete(&self, id: Uuid) -> StoreResult<()>; + + async fn search(&self, query: &SearchQuery) -> StoreResult>; + async fn fts_search(&self, text: &str, limit: usize) -> StoreResult>; + async fn vector_search(&self, embedding: &[f32], limit: usize) -> StoreResult>; + + async fn get_scheduling(&self, memory_id: Uuid) -> StoreResult>; + async fn update_scheduling(&self, state: &SchedulingState) -> StoreResult<()>; + async fn get_due_memories( + &self, + before: DateTime, + limit: usize, + ) -> StoreResult>; + + async fn add_edge(&self, edge: &MemoryEdge) -> StoreResult<()>; + async fn get_edges(&self, node_id: Uuid, edge_type: Option<&str>) -> StoreResult>; + async fn remove_edge(&self, source: Uuid, target: Uuid, edge_type: &str) -> StoreResult<()>; + async fn get_neighbors(&self, node_id: Uuid, depth: usize) -> StoreResult>; + + async fn list_domains(&self) -> StoreResult>; + async fn get_domain(&self, id: &str) -> StoreResult>; + async fn upsert_domain(&self, domain: &Domain) -> StoreResult<()>; + async fn delete_domain(&self, id: &str) -> StoreResult<()>; + async fn classify(&self, embedding: &[f32]) -> StoreResult>; + + async fn count(&self) -> StoreResult; + async fn get_stats(&self) -> StoreResult; + async fn vacuum(&self) -> StoreResult<()>; +} +``` + +- **SQL (inline within impl methods)**: every call uses `sqlx::query!` or `sqlx::query_as!` for compile-time validation. Examples: + +```rust +// insert +sqlx::query!( + r#" + INSERT INTO memories ( + id, domains, domain_scores, content, node_type, tags, + embedding, metadata, created_at, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7::vector, $8, $9, $10) + "#, + record.id, + &record.domains as &[String], + serde_json::to_value(&record.domain_scores)?, + record.content, + record.node_type, + &record.tags as &[String], + record.embedding.as_ref().map(|v| Vector::from(v.clone())) as Option, + record.metadata, + record.created_at, + record.updated_at, +) +.execute(&self.pool) +.await?; +``` + +- **Behavior notes**: + - `StoreError` gets two new variants behind the feature: + +```rust +#[cfg(feature = "postgres-backend")] +#[error("postgres error: {0}")] +Postgres(#[from] sqlx::Error), + +#[cfg(feature = "postgres-backend")] +#[error("postgres migration error: {0}")] +Migrate(#[from] sqlx::migrate::MigrateError), +``` + + - `classify()` on Postgres implements the PRD's cosine-similarity-to-centroid computation inside SQL using `1 - (centroid <=> $1::vector)` over the `domains` table and returns rows sorted descending. This mirrors the behavior a `DomainClassifier` in Phase 4 uses; Phase 2 ships the backend capability but does not call it. + - Connection pool defaults (see D3): `max_connections = 10`, `acquire_timeout = 30s`, `idle_timeout = 600s`, `test_before_acquire = false` (cheap queries; avoid per-acquire roundtrip). + - All methods are `async fn` and use sqlx's `tokio` runtime feature; no blocking `block_on`. + +### D3. Pool construction and config wiring + +- **File**: `crates/vestige-core/src/storage/postgres/pool.rs` +- **Depends on**: D1, D2, D9. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, PgPool}; +use std::str::FromStr; +use std::time::Duration; + +use crate::config::PostgresConfig; +use crate::storage::error::{StoreError, StoreResult}; + +pub async fn build_pool(cfg: &PostgresConfig) -> StoreResult { + let mut opts = PgConnectOptions::from_str(&cfg.url)?; + opts = opts + .application_name("vestige") + .statement_cache_capacity(256) + .log_statements(tracing::log::LevelFilter::Debug); + + let pool = PgPoolOptions::new() + .max_connections(cfg.max_connections.unwrap_or(10)) + .min_connections(0) + .acquire_timeout(Duration::from_secs(cfg.acquire_timeout_secs.unwrap_or(30))) + .idle_timeout(Some(Duration::from_secs(600))) + .max_lifetime(Some(Duration::from_secs(1800))) + .test_before_acquire(false) + .connect_with(opts) + .await?; + + Ok(pool) +} +``` + +- **Behavior notes**: acquire timeout chosen to exceed the 30-second testcontainer spin-up requirement. `application_name = "vestige"` makes `pg_stat_activity` readable from `psql` during debugging. + +### D4. sqlx migrations directory + +- **File**: `crates/vestige-core/migrations/postgres/0001_init.up.sql`, `0001_init.down.sql`, `0002_hnsw.up.sql`, `0002_hnsw.down.sql`. +- **Depends on**: none (pure SQL). + +`0001_init.up.sql`: + +```sql +-- Extensions +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS vector; + +-- Embedding model registry +-- Mirrors the SQLite table created in Phase 1. +CREATE TABLE embedding_model ( + id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1), + name TEXT NOT NULL, + dimension INTEGER NOT NULL CHECK (dimension > 0), + hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Domains table (populated by Phase 4 DomainClassifier; Phase 2 only creates +-- the empty table so list/get/upsert/delete work against both backends). +CREATE TABLE domains ( + id TEXT PRIMARY KEY, + label TEXT NOT NULL, + centroid vector, + top_terms TEXT[] NOT NULL DEFAULT '{}', + memory_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +-- Core memories table +CREATE TABLE memories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + domains TEXT[] NOT NULL DEFAULT '{}', + domain_scores JSONB NOT NULL DEFAULT '{}'::jsonb, + content TEXT NOT NULL, + node_type TEXT NOT NULL DEFAULT 'general', + tags TEXT[] NOT NULL DEFAULT '{}', + embedding vector, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + search_vec TSVECTOR GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(content, '')), 'A') || + setweight(to_tsvector('english', coalesce(node_type, '')), 'B') || + setweight(to_tsvector('english', coalesce(array_to_string(tags, ' '), '')), 'C') + ) STORED +); + +-- FSRS scheduling state (1:1 with memories) +CREATE TABLE scheduling ( + memory_id UUID PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE, + stability DOUBLE PRECISION NOT NULL DEFAULT 0.0, + difficulty DOUBLE PRECISION NOT NULL DEFAULT 0.0, + retrievability DOUBLE PRECISION NOT NULL DEFAULT 1.0, + last_review TIMESTAMPTZ, + next_review TIMESTAMPTZ, + reps INTEGER NOT NULL DEFAULT 0, + lapses INTEGER NOT NULL DEFAULT 0 +); + +-- Graph edges (spreading activation) +CREATE TABLE edges ( + source_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + target_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + edge_type TEXT NOT NULL DEFAULT 'related', + weight DOUBLE PRECISION NOT NULL DEFAULT 1.0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source_id, target_id, edge_type) +); + +-- FSRS review event log (Phase 1 creates this; Phase 2 mirrors it for Postgres). +-- Append-only. Used for future federation (Phase 5). +CREATE TABLE review_events ( + id BIGSERIAL PRIMARY KEY, + memory_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + rating SMALLINT NOT NULL, + prior_state JSONB NOT NULL, + new_state JSONB NOT NULL +); + +-- Indexes on memories (vector index is declared separately in 0002_hnsw.up.sql) +CREATE INDEX idx_memories_fts ON memories USING GIN (search_vec); +CREATE INDEX idx_memories_domains ON memories USING GIN (domains); +CREATE INDEX idx_memories_tags ON memories USING GIN (tags); +CREATE INDEX idx_memories_node_type ON memories (node_type); +CREATE INDEX idx_memories_created ON memories (created_at); +CREATE INDEX idx_memories_updated ON memories (updated_at); + +-- Indexes on scheduling +CREATE INDEX idx_scheduling_next_review ON scheduling (next_review); +CREATE INDEX idx_scheduling_last_review ON scheduling (last_review); + +-- Indexes on edges +CREATE INDEX idx_edges_target ON edges (target_id); +CREATE INDEX idx_edges_source ON edges (source_id); +CREATE INDEX idx_edges_type ON edges (edge_type); + +-- Indexes on review_events +CREATE INDEX idx_review_events_memory ON review_events (memory_id); +CREATE INDEX idx_review_events_ts ON review_events (timestamp); + +-- Update trigger on memories.updated_at +CREATE OR REPLACE FUNCTION memories_set_updated_at() RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_memories_updated_at +BEFORE UPDATE ON memories +FOR EACH ROW EXECUTE FUNCTION memories_set_updated_at(); +``` + +`0001_init.down.sql`: + +```sql +DROP TRIGGER IF EXISTS trg_memories_updated_at ON memories; +DROP FUNCTION IF EXISTS memories_set_updated_at(); + +DROP INDEX IF EXISTS idx_review_events_ts; +DROP INDEX IF EXISTS idx_review_events_memory; +DROP INDEX IF EXISTS idx_edges_type; +DROP INDEX IF EXISTS idx_edges_source; +DROP INDEX IF EXISTS idx_edges_target; +DROP INDEX IF EXISTS idx_scheduling_last_review; +DROP INDEX IF EXISTS idx_scheduling_next_review; +DROP INDEX IF EXISTS idx_memories_updated; +DROP INDEX IF EXISTS idx_memories_created; +DROP INDEX IF EXISTS idx_memories_node_type; +DROP INDEX IF EXISTS idx_memories_tags; +DROP INDEX IF EXISTS idx_memories_domains; +DROP INDEX IF EXISTS idx_memories_fts; + +DROP TABLE IF EXISTS review_events; +DROP TABLE IF EXISTS edges; +DROP TABLE IF EXISTS scheduling; +DROP TABLE IF EXISTS memories; +DROP TABLE IF EXISTS domains; +DROP TABLE IF EXISTS embedding_model; +``` + +`0002_hnsw.up.sql` (separated so reembed can drop-and-recreate without touching the rest of the schema): + +```sql +-- HNSW index on memories.embedding. +-- pgvector requires the column to have a typmod (fixed dimension) for HNSW. +-- The dimension is stamped by the application at startup via ALTER TABLE +-- using the embedder's dimension() method (see PgMemoryStore::connect). +-- We express the index with the generic vector_cosine_ops operator class. +CREATE INDEX idx_memories_embedding_hnsw + ON memories USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64); +``` + +`0002_hnsw.down.sql`: + +```sql +DROP INDEX IF EXISTS idx_memories_embedding_hnsw; +``` + +- **Behavior notes**: + - pgvector HNSW requires a typmod. `PgMemoryStore::connect` runs `ALTER TABLE memories ALTER COLUMN embedding TYPE vector($N)` with `$N = embedder.dimension()` exactly once, guarded by a check against `embedding_model` (first startup ever) or validated against it on subsequent starts. If `embedder.dimension()` differs from the stored one and `embedding_model` is non-empty, return `StoreError::EmbeddingDimensionMismatch` -- the user must run `vestige migrate --reembed`. + - `ALTER COLUMN ... TYPE vector($N)` on a populated column fails unless the data fits; that is the desired safety net. + - The `tsvector` GENERATED column uses `array_to_string(tags, ' ')` rather than `array_to_tsvector` from the PRD sketch, because `array_to_tsvector` is not a core function in Postgres 16 and would require an extension. The behavior is equivalent for weight C. + - `gen_random_uuid()` comes from `pgcrypto`. In Postgres 13+ it is also available from core; we keep the extension for older compatibility paths. + - MVCC: all table writes are transactional; no explicit locks. `INSERT ... ON CONFLICT DO UPDATE` is used in `upsert_domain`, `update_scheduling`, and edge idempotency. + +### D5. Hybrid search via RRF + +- **File**: `crates/vestige-core/src/storage/postgres/search.rs` +- **Depends on**: D2, D4. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use pgvector::Vector; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::storage::error::StoreResult; +use crate::storage::types::{SearchQuery, SearchResult}; + +const RRF_K: i32 = 60; // constant from Cormack et al. 2009 +const OVERFETCH_MULT: i64 = 3; // matches Phase 1 SQLite overfetch + +pub(crate) async fn rrf_search( + pool: &PgPool, + query: &SearchQuery, +) -> StoreResult>; +``` + +SQL for the full hybrid RRF query. Placeholders: +- `$1` = text query (string, may be empty) +- `$2` = embedding (vector) +- `$3` = overfetch limit per branch (int) +- `$4` = final limit (int) +- `$5` = domain filter (text[] or NULL) +- `$6` = node_type filter (text[] or NULL) +- `$7` = tag filter (text[] or NULL) + +```sql +WITH params AS ( + SELECT + $1::text AS q_text, + $2::vector AS q_vec, + $3::int AS overfetch, + $4::int AS final_limit, + $5::text[] AS dom_filter, + $6::text[] AS nt_filter, + $7::text[] AS tag_filter +), +fts AS ( + SELECT m.id, + ts_rank_cd(m.search_vec, websearch_to_tsquery('english', p.q_text)) AS score, + ROW_NUMBER() OVER ( + ORDER BY ts_rank_cd(m.search_vec, websearch_to_tsquery('english', p.q_text)) DESC + ) AS rank + FROM memories m, params p + WHERE p.q_text <> '' + AND m.search_vec @@ websearch_to_tsquery('english', p.q_text) + AND (p.dom_filter IS NULL OR m.domains && p.dom_filter) + AND (p.nt_filter IS NULL OR m.node_type = ANY(p.nt_filter)) + AND (p.tag_filter IS NULL OR m.tags && p.tag_filter) + LIMIT (SELECT overfetch FROM params) +), +vec AS ( + SELECT m.id, + 1 - (m.embedding <=> p.q_vec) AS score, + ROW_NUMBER() OVER ( + ORDER BY m.embedding <=> p.q_vec + ) AS rank + FROM memories m, params p + WHERE m.embedding IS NOT NULL + AND p.q_vec IS NOT NULL + AND (p.dom_filter IS NULL OR m.domains && p.dom_filter) + AND (p.nt_filter IS NULL OR m.node_type = ANY(p.nt_filter)) + AND (p.tag_filter IS NULL OR m.tags && p.tag_filter) + LIMIT (SELECT overfetch FROM params) +), +fused AS ( + SELECT COALESCE(f.id, v.id) AS id, + COALESCE(1.0 / (60 + f.rank), 0.0) + + COALESCE(1.0 / (60 + v.rank), 0.0) AS rrf_score, + f.score AS fts_score, + v.score AS vector_score + FROM fts f FULL OUTER JOIN vec v ON f.id = v.id +) +SELECT m.id AS "id!: Uuid", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.content AS "content!", + m.node_type AS "node_type!", + m.tags AS "tags!: Vec", + m.embedding AS "embedding?: Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: chrono::DateTime", + m.updated_at AS "updated_at!: chrono::DateTime", + fused.rrf_score AS "rrf_score!: f64", + fused.fts_score AS "fts_score?: f64", + fused.vector_score AS "vector_score?: f64" +FROM fused +JOIN memories m ON m.id = fused.id +ORDER BY fused.rrf_score DESC +LIMIT (SELECT final_limit FROM params); +``` + +- **Behavior notes**: + - `OVERFETCH_MULT * query.limit` is passed as `$3`. Final `$4` is `query.limit`. + - Empty text query is allowed; the `fts` CTE returns zero rows (`p.q_text <> ''`) and the result degrades to pure vector search, which matches `vector_search` behavior. + - Null embedding is allowed; the `vec` CTE returns zero rows and the result degrades to pure FTS, which matches `fts_search` behavior. + - `fts_search` and `vector_search` are separate public methods on the trait. Each uses a simpler single-CTE query derived from the above by removing the other branch. Implementing them as thin wrappers over `rrf_search` with nullified inputs is acceptable but adds one extra plan per call; the explicit implementations win on latency. + - `min_retrievability` in `SearchQuery` is applied as a final filter by joining on `scheduling` in the outer `SELECT`. Adding that join unconditionally regresses simple searches; add it only when `query.min_retrievability.is_some()`. + +### D6. `embedding_model` registry impl + +- **File**: `crates/vestige-core/src/storage/postgres/registry.rs` +- **Depends on**: D1, D4 (table exists), Phase 1 `EmbeddingModelRegistry` trait. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use sqlx::PgPool; + +use crate::embedder::Embedder; +use crate::storage::error::{StoreError, StoreResult}; + +pub(crate) async fn ensure_registry( + pool: &PgPool, + embedder: &dyn Embedder, +) -> StoreResult<()> { + let row = sqlx::query!( + r#"SELECT name, dimension, hash FROM embedding_model WHERE id = 1"# + ) + .fetch_optional(pool) + .await?; + + match row { + None => { + sqlx::query!( + r#" + INSERT INTO embedding_model (id, name, dimension, hash) + VALUES (1, $1, $2, $3) + "#, + embedder.model_name(), + embedder.dimension() as i32, + embedder.model_hash(), + ) + .execute(pool) + .await?; + + // First-ever run: stamp the vector column typmod. + let ddl = format!( + "ALTER TABLE memories ALTER COLUMN embedding TYPE vector({})", + embedder.dimension() + ); + sqlx::query(&ddl).execute(pool).await?; + Ok(()) + } + Some(r) if r.name == embedder.model_name() + && r.dimension == embedder.dimension() as i32 + && r.hash == embedder.model_hash() => Ok(()), + Some(r) => Err(StoreError::EmbeddingMismatch { + expected: format!("{} ({}d, {})", r.name, r.dimension, r.hash), + got: format!( + "{} ({}d, {})", + embedder.model_name(), + embedder.dimension(), + embedder.model_hash() + ), + }), + } +} + +pub(crate) async fn update_registry( + pool: &PgPool, + embedder: &dyn Embedder, +) -> StoreResult<()> { + // Used only by `vestige migrate --reembed` after a full re-encode. + sqlx::query!( + r#" + UPDATE embedding_model + SET name = $1, dimension = $2, hash = $3, created_at = now() + WHERE id = 1 + "#, + embedder.model_name(), + embedder.dimension() as i32, + embedder.model_hash(), + ) + .execute(pool) + .await?; + Ok(()) +} +``` + +- **Behavior notes**: + - `StoreError::EmbeddingMismatch { expected, got }` already exists in Phase 1; Phase 2 just constructs it. + - The `ALTER TABLE ... TYPE vector(N)` DDL is only issued on first init. On subsequent inits the existing typmod already matches. + - Re-embed flow also uses this module, but the DDL path is different -- see D11. + +### D7. `VestigeConfig`: `vestige.toml` backend selection + +- **File**: `crates/vestige-core/src/config.rs` (Phase 1 may already own this file; Phase 2 extends, not replaces) +- **Depends on**: D1. +- **Signatures**: + +```rust +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct VestigeConfig { + #[serde(default)] + pub embeddings: EmbeddingsConfig, + #[serde(default)] + pub storage: StorageConfig, + #[serde(default)] + pub server: ServerConfig, + #[serde(default)] + pub auth: AuthConfig, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct EmbeddingsConfig { + pub provider: String, // "fastembed" + pub model: String, // "BAAI/bge-base-en-v1.5" +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "backend", rename_all = "lowercase")] +pub enum StorageConfig { + Sqlite(SqliteConfig), + #[cfg(feature = "postgres-backend")] + Postgres(PostgresConfig), +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SqliteConfig { + pub path: PathBuf, +} + +#[cfg(feature = "postgres-backend")] +#[derive(Debug, Clone, Deserialize)] +pub struct PostgresConfig { + pub url: String, + #[serde(default)] + pub max_connections: Option, + #[serde(default)] + pub acquire_timeout_secs: Option, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ServerConfig { /* Phase 3 fills this in */ } + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct AuthConfig { /* Phase 3 fills this in */ } + +impl VestigeConfig { + pub fn load(path: Option<&Path>) -> Result; + pub fn default_path() -> PathBuf; // ~/.vestige/vestige.toml +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("io: {0}")] + Io(#[from] std::io::Error), + #[error("toml: {0}")] + Toml(#[from] toml::de::Error), + #[error("invalid config: {0}")] + Invalid(String), +} +``` + +- **Behavior notes**: + - The serde representation matches the PRD: `[storage]` with `backend = "sqlite"` and a matching `[storage.sqlite]` or `[storage.postgres]` subsection. + - Because `StorageConfig` is `#[serde(tag = "backend")]`, an unknown backend string returns a clear error. + - If `postgres-backend` is compiled off and the user writes `backend = "postgres"`, deserialization returns "unknown variant `postgres`" -- loud failure. Phase 2 wraps this into `ConfigError::Invalid("postgres-backend feature not compiled in")`. + - `env`-override hooks (e.g., `VESTIGE_POSTGRES_URL`) are a Phase 3 concern; not added here. + +### D8. `vestige migrate --from sqlite --to postgres` + +- **File**: `crates/vestige-core/src/storage/postgres/migrate_cli.rs` +- **Depends on**: D2, D6, D7, Phase 1 `SqliteMemoryStore`. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use std::path::Path; +use std::sync::Arc; + +use futures::{StreamExt, TryStreamExt}; +use indicatif::{ProgressBar, ProgressStyle}; +use uuid::Uuid; + +use crate::embedder::Embedder; +use crate::storage::error::{StoreError, StoreResult}; +use crate::storage::postgres::PgMemoryStore; +use crate::storage::sqlite::SqliteMemoryStore; + +#[derive(Debug, Clone)] +pub struct SqliteToPostgresPlan { + pub sqlite_path: std::path::PathBuf, + pub postgres_url: String, + pub max_connections: u32, + pub batch_size: usize, // default 500 +} + +pub struct MigrationReport { + pub memories_copied: u64, + pub scheduling_rows: u64, + pub edges_copied: u64, + pub review_events_copied: u64, + pub domains_copied: u64, + pub errors: Vec<(Uuid, StoreError)>, +} + +pub async fn run_sqlite_to_postgres( + plan: SqliteToPostgresPlan, + embedder: Arc, +) -> StoreResult; +``` + +Algorithm: + +1. Open source `SqliteMemoryStore` in read-only mode (`?mode=ro`). +2. Check source `embedding_model` registry; refuse if it disagrees with the supplied embedder unless the user also passed `--reembed`. +3. Open destination `PgMemoryStore` via `connect` (runs migrations, stamps dim). +4. Stream source rows in batches of `plan.batch_size` via a windowed query ordered by `created_at, id` (stable cursor; survives resume). +5. For each batch: begin a Postgres transaction, `INSERT INTO memories ... ON CONFLICT (id) DO NOTHING` for all rows, `INSERT INTO scheduling` likewise, commit. Copy domain assignments (`domains`, `domain_scores`) verbatim -- they are `[]` and `{}` for pre-Phase-4 SQLite data. +6. After memories finish, stream edges and review_events the same way. +7. Emit progress via `indicatif::ProgressBar` (one bar per table, multi-bar). Each 1000 rows log to tracing at INFO. +8. Return `MigrationReport` for the caller to print. + +- **Behavior notes**: + - Memory-bounded: batch size 500 and sqlx streams mean memory usage stays O(batch * row_size), not O(total_rows). + - Idempotent: re-running replays only the rows not already present; `ON CONFLICT DO NOTHING` means partial runs recover. + - UUID strings from SQLite are parsed via `Uuid::parse_str` -- any mangled ID pushes to `errors` instead of aborting. + - The FTS `search_vec` is regenerated by Postgres via the GENERATED column; no data to copy. + - `review_events` may not exist in Phase 1 SQLite for pre-V12 databases. The migrator detects missing tables via `SELECT name FROM sqlite_master` and skips gracefully. + - A separate `--dry-run` flag prints the counts per table without writing. + +### D9. `vestige migrate --reembed --model=` + +- **File**: `crates/vestige-core/src/storage/postgres/reembed.rs` +- **Depends on**: D2, D6, Phase 1 `Embedder`. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; +use std::time::Instant; + +use futures::TryStreamExt; +use indicatif::{ProgressBar, ProgressStyle}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::embedder::Embedder; +use crate::storage::error::{StoreError, StoreResult}; +use crate::storage::postgres::PgMemoryStore; + +#[derive(Debug, Clone)] +pub struct ReembedPlan { + pub batch_size: usize, // default 128 (embedder batch) + pub drop_hnsw_first: bool, // default true + pub concurrent_index: bool, // default false; use CREATE INDEX (not CONCURRENTLY) +} + +pub struct ReembedReport { + pub rows_updated: u64, + pub duration_secs: f64, + pub index_rebuild_secs: f64, +} + +pub async fn run_reembed( + store: &PgMemoryStore, + new_embedder: Arc, + plan: ReembedPlan, +) -> StoreResult; +``` + +Algorithm: + +1. Verify `new_embedder.dimension()` != stored dimension OR `new_embedder.model_hash()` != stored hash -- otherwise no-op and return `rows_updated = 0`. +2. `BEGIN; ALTER TABLE memories ALTER COLUMN embedding DROP NOT NULL`; not actually needed (column is already nullable) but shown here for documentation. +3. If `plan.drop_hnsw_first`, execute `DROP INDEX IF EXISTS idx_memories_embedding_hnsw;` so updates are not slowed by index maintenance. This is the recommended path; `REINDEX` is kept in the Open Questions as an alternative. +4. Stream all `id, content` from `memories` ordered by `id`. +5. For each batch of `plan.batch_size`: call `new_embedder.embed_batch(&texts)` (Phase 1 trait exposes batched embedding when available; otherwise loop single `embed`). Then: + +```sql +UPDATE memories +SET embedding = v.embedding::vector +FROM UNNEST($1::uuid[], $2::real[][]) AS v(id, embedding) +WHERE memories.id = v.id; +``` + +6. After all rows updated: run `ALTER TABLE memories ALTER COLUMN embedding TYPE vector($NEW_DIM)` if dimension changed. +7. Rebuild HNSW. If `plan.concurrent_index`, execute `CREATE INDEX CONCURRENTLY idx_memories_embedding_hnsw ...`; else `CREATE INDEX idx_memories_embedding_hnsw ...`. +8. `update_registry` with the new embedder. +9. Return `ReembedReport`. + +- **Behavior notes**: + - Memory-bounded: batch_size * 2 (old + new texts) vectors in RAM at any time. + - The dimension change must happen AFTER all rows are updated (pgvector validates typmod on write when a typmod is present; we relax-then-tighten). + - `CONCURRENTLY` builds do not hold `AccessExclusiveLock`, but fail inside a transaction. That's why the outer driver runs index DDL as an autocommit statement (sqlx `execute` outside a pool transaction). + - For `--dry-run`, emit what *would* happen (row count, estimated embedder calls, estimated time using `rows / 50`-per-second baseline for local fastembed) and exit. + +### D10. CLI wiring in `vestige-mcp` + +- **File**: `crates/vestige-mcp/src/bin/cli.rs` +- **Depends on**: D8, D9, D7. Requires `vestige-mcp` Cargo feature `postgres-backend`. +- **Signatures**: + +```rust +#[derive(Subcommand)] +enum Commands { + // existing variants: Stats, Health, Consolidate, Restore, Backup, + // Export, Gc, Dashboard, Ingest, Serve ... + + /// Migrate between backends or re-embed memories. + #[cfg(feature = "postgres-backend")] + Migrate(MigrateArgs), +} + +#[derive(clap::Args)] +#[cfg(feature = "postgres-backend")] +struct MigrateArgs { + #[command(subcommand)] + action: MigrateAction, +} + +#[derive(Subcommand)] +#[cfg(feature = "postgres-backend")] +enum MigrateAction { + /// Copy all memories from SQLite to Postgres. + #[command(name = "copy")] + Copy { + #[arg(long)] + from: String, // "sqlite" + #[arg(long)] + to: String, // "postgres" + #[arg(long)] + sqlite_path: PathBuf, + #[arg(long)] + postgres_url: String, + #[arg(long, default_value = "500")] + batch_size: usize, + #[arg(long)] + dry_run: bool, + }, + /// Re-embed all memories with a new embedder. + #[command(name = "reembed")] + Reembed { + #[arg(long)] + model: String, + #[arg(long, default_value = "128")] + batch_size: usize, + #[arg(long, default_value_t = true)] + drop_hnsw_first: bool, + #[arg(long)] + concurrent_index: bool, + #[arg(long)] + dry_run: bool, + }, +} +``` + +The user-facing invocation collapses to the exact string requested by the ADR: + +``` +vestige migrate copy --from sqlite --to postgres \ + --sqlite-path ~/.vestige/vestige.db \ + --postgres-url postgresql://localhost/vestige + +vestige migrate reembed --model=BAAI/bge-large-en-v1.5 +``` + +An alternate top-level layout (single `vestige migrate` with flags `--from`, `--to`, `--reembed`) is equivalent; the subcommand split is preferred because the two flag sets are disjoint (see Open Question 1). + +- **Behavior notes**: + - `--from`/`--to` values are validated; the current Phase 2 build accepts only `sqlite` and `postgres`. + - For `reembed`, the `--model` string resolves to an `Embedder` via a factory already provided by Phase 1 (`Embedder::from_name(&str)`); Phase 2 does not invent new embedder constructors. + - Progress output on `stderr`; machine-readable summary on `stdout` as one-line JSON when `--json` is set (skipped for Phase 2 unless trivial). + +### D11. Offline query cache (`.sqlx/`) + +- **File**: `crates/vestige-core/.sqlx/` (committed directory of `query-*.json`) +- **Depends on**: all `sqlx::query!` call sites being final. +- **Procedure**: the developer runs `cargo sqlx prepare --workspace` with a live Postgres having the schema applied. Output goes into `crates/vestige-core/.sqlx/`. This directory is committed. CI enforces freshness by running `cargo sqlx prepare --workspace --check` against the same live Postgres (or failing that, any dev can reproduce by setting `SQLX_OFFLINE=true`). +- **Behavior notes**: `SQLX_OFFLINE=true` in `build.rs` or env is the default on CI and for downstream consumers. The `vestige-core` docs add a one-liner in README for contributors: "if you change any SQL in Phase 2 modules, rerun `cargo sqlx prepare` with a live DB." + +### D12. Testcontainer harness (integration) + +- **File**: `tests/phase_2/common/mod.rs` (the `common` convention used in `tests/phase_2/` crates) +- **Depends on**: D2 through D11. +- **Signatures**: + +```rust +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; + +use testcontainers_modules::postgres::Postgres; +use testcontainers::{runners::AsyncRunner, ContainerAsync}; + +use vestige_core::embedder::Embedder; +use vestige_core::storage::postgres::PgMemoryStore; + +pub struct PgHarness { + pub container: ContainerAsync, + pub store: PgMemoryStore, +} + +impl PgHarness { + pub async fn start(embedder: Arc) -> anyhow::Result { + let container = Postgres::default() + .with_tag("pg16") + .with_name("pgvector/pgvector") + .start() + .await?; + let port = container.get_host_port_ipv4(5432).await?; + let url = format!( + "postgresql://postgres:postgres@127.0.0.1:{}/postgres", port + ); + let store = PgMemoryStore::connect(&url, 4, embedder.as_ref()).await?; + Ok(Self { container, store }) + } +} +``` + +- **Behavior notes**: + - Image `pgvector/pgvector:pg16` bundles pgvector into the official postgres:16 image. + - Pool size 4 is enough for tests without starving the container's default `max_connections = 100`. + - `ContainerAsync` is held for the whole test scope; drop tears down the container. + - A fake `TestEmbedder` in `common/test_embedder.rs` provides a deterministic hash-based embedding (no ONNX dependency in CI). + +--- + +## Test Plan + +### Unit tests (colocated in `src/`) + +Under `crates/vestige-core/src/storage/postgres/`: + +- `pool.rs` -- one test per `build_pool` branch: defaults, explicit `max_connections`, invalid URL returns `StoreError::Postgres`. +- `registry.rs` -- three tests: first-init writes row and alters typmod, reopen with same embedder returns Ok, reopen with different dimension returns `EmbeddingMismatch`. +- `search.rs` -- query-builder unit tests for parameter packing: empty text, null embedding, all three filters null, all three filters populated. +- `migrate_cli.rs` -- `SqliteToPostgresPlan::default` returns sane defaults; plan validation rejects empty URL. +- `reembed.rs` -- `ReembedPlan::no_change` returns `rows_updated == 0` when embedder matches registry (no network call). +- `config.rs` -- five tests covering: valid postgres config, valid sqlite config, unknown backend string, missing subsection, feature-gated postgres without feature compiled in. + +### Integration tests (in `tests/phase_2/`) + +Each file is a full integration test crate (`[[test]]` in workspace root Cargo). + +**`tests/phase_2/pg_trait_parity.rs`** + +- Declares the same test matrix as Phase 1's SQLite trait tests, parameterized over `impl MemoryStore`. +- Runs every method: `insert`, `get`, `update`, `delete`, `search`, `fts_search`, `vector_search`, `get_scheduling`, `update_scheduling`, `get_due_memories`, `add_edge`, `get_edges`, `remove_edge`, `get_neighbors`, `list_domains`, `get_domain`, `upsert_domain`, `delete_domain`, `classify`, `count`, `get_stats`, `vacuum`, `health_check`. +- Each test is written once as `async fn roundtrip_(store: &dyn MemoryStore)` and invoked from two wrappers, one for SQLite and one for Postgres. +- Acceptance: every method returns equal results (except for `Uuid` ordering in `list_domains` where the test sorts before comparing). + +**`tests/phase_2/pg_hybrid_search_rrf.rs`** + +- Inserts 20 memories with known content ("rust async trait", "postgres hnsw vector", "fastembed onnx model", ...). +- Case 1: pure FTS. `SearchQuery { text: Some("rust trait"), embedding: None, ... }` returns the three Rust-related rows in order; `fts_score` populated, `vector_score` null. +- Case 2: pure vector. `SearchQuery { text: None, embedding: Some(embed("rust trait")), ... }` returns the same three rows via cosine; `vector_score` populated, `fts_score` null. +- Case 3: hybrid. Both set -- top hit has both scores; `rrf_score >= 1/(60+1) + 1/(60+1) = 0.0328`. +- Case 4: domain filter. 10 memories tagged with `domains = ["dev"]`, 10 with `["home"]`. Query with `domains: Some(vec!["dev"])` returns only dev memories. +- Case 5: edge case -- empty FTS query plus an embedding behaves identically to `vector_search`; empty embedding plus FTS query behaves identically to `fts_search`. + +**`tests/phase_2/pg_migration_sqlite_to_postgres.rs`** + +- Populate a fresh SQLite with 10,000 memories (seeded RNG, deterministic content), 4,000 scheduling rows, 2,000 edges. +- Run `run_sqlite_to_postgres` with a test embedder. +- Assert: `count() == 10_000` on destination; spot-check 25 memories byte-for-byte (content, tags, metadata, domains, domain_scores). +- Assert: FSRS fields (`stability`, `difficulty`, `next_review`) preserved per memory. +- Assert: edges preserved by `(source_id, target_id, edge_type)`. +- Assert: re-running the migration is a no-op (`ON CONFLICT DO NOTHING` path); row count unchanged. + +**`tests/phase_2/pg_migration_reembed.rs`** + +- Start with a fresh store using `TestEmbedder768` (768-dim, hash `h1`). Insert 500 memories. +- Swap to `TestEmbedder1024` (1024-dim, hash `h2`). Run `run_reembed(store, Arc::new(TestEmbedder1024), ReembedPlan::default())`. +- Assert: `rows_updated == 500`; `embedding_model` now has `(name=TestEmbedder1024, dimension=1024, hash=h2)`. +- Assert: `SELECT DISTINCT vector_dims(embedding) FROM memories` returns only `1024`. +- Assert: HNSW index exists after reembed (`SELECT indexrelid FROM pg_indexes WHERE indexname = 'idx_memories_embedding_hnsw'`). +- Assert: memory IDs unchanged (compare pre/post id sets). +- Assert: a hybrid search using `TestEmbedder1024` returns results (post-reembed vectors are queryable). + +**`tests/phase_2/pg_config_parsing.rs`** + +- Parse six `vestige.toml` snippets: + - sqlite + fastembed -> `StorageConfig::Sqlite`. + - postgres + fastembed -> `StorageConfig::Postgres` with `max_connections = 10`. + - postgres with custom `max_connections = 25` and `acquire_timeout_secs = 60`. + - unknown backend `"mysql"` -> `ConfigError`. + - missing subsection `[storage.postgres]` while `backend = "postgres"` -> `ConfigError`. + - malformed URL (empty) -> `ConfigError::Invalid`. + +**`tests/phase_2/pg_concurrency.rs`** + +- Spawn 16 tasks, each inserting 100 memories in parallel for 1,600 total. +- Spawn 4 tasks concurrently running `search` queries; none should fail. +- Spawn 2 tasks concurrently running `update_scheduling` on overlapping IDs -- last write wins (MVCC), neither errors. +- Assert: all 1,600 rows present, no deadlocks, every task returns `Ok`. +- Run time < 10 seconds on a cold container. + +### Compile-time query verification + +- CI step: `cargo sqlx prepare --workspace --check` against a CI-provisioned Postgres (GitHub Actions / Forgejo Actions services block). Fails CI if any `query!` macro goes stale. +- Alternative offline run for contributors: `SQLX_OFFLINE=true cargo check -p vestige-core --features postgres-backend`. CI runs both forms to ensure `.sqlx/` is up to date. +- `.sqlx/` is committed to the repo. A `.gitattributes` entry marks it as `linguist-generated=true` so it doesn't inflate language stats. + +### Benchmarks + +Under `crates/vestige-core/benches/pg_hybrid_search.rs` (Criterion), gated by `postgres-backend`. + +- `pg_search_1k` -- populate 1,000 memories once per bench suite, measure `rrf_search` p50/p99 over 500 iterations. Target: p50 < 10ms, p99 < 30ms on a local container. +- `pg_search_100k` -- 100,000 memories. Target: p50 < 50ms, p99 < 150ms. Validates HNSW scaling. +- Testcontainer shared across both benches via `once_cell`. +- Bench entry in `vestige-core/Cargo.toml`: + +``` +[[bench]] +name = "pg_hybrid_search" +harness = false +required-features = ["postgres-backend"] +``` + +--- + +## Acceptance Criteria + +- [ ] `cargo build -p vestige-core --features postgres-backend` -- zero warnings. +- [ ] `cargo build -p vestige-core` (SQLite-only, default features) -- zero warnings; no Postgres symbols referenced. +- [ ] `cargo build -p vestige-mcp --features postgres-backend` -- zero warnings; `vestige` binary exposes the `migrate` subcommand. +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` -- clean. +- [ ] `cargo sqlx prepare --workspace --check` -- returns success; `.sqlx/` is current. +- [ ] `cargo test -p vestige-core --features postgres-backend --test pg_trait_parity --test pg_hybrid_search_rrf --test pg_migration_sqlite_to_postgres --test pg_migration_reembed --test pg_config_parsing --test pg_concurrency` -- all green. +- [ ] Testcontainer spin-up p50 under 30 seconds on a developer laptop with a warm Docker daemon. +- [ ] `pg_search_100k` Criterion bench reports p50 < 50ms on reference hardware (logged in the ADR comment trail). +- [ ] `vestige migrate copy --from sqlite --to postgres` on a 10,000-memory corpus completes without data loss: row count parity, content byte-parity on a 1 percent sample, FSRS state preserved (stability, difficulty, reps, lapses, next_review), edge count parity. +- [ ] `vestige migrate reembed` with a dimension-changing embedder returns to a fully queryable state: HNSW present, `embedding_model` updated, no stale vectors, memory IDs untouched. +- [ ] Trait parity: every method on `MemoryStore` has at least one passing test against `PgMemoryStore`. +- [ ] Phase 1's existing SQLite suite continues to pass with zero changes required (Phase 2 is additive). +- [ ] The `postgres-backend` feature does not compile in SQLCipher (`encryption`) simultaneously (mutually exclusive at compile time, per project rule). + +--- + +## Rollback Notes + +- Every `*.up.sql` has a matching `*.down.sql` in `crates/vestige-core/migrations/postgres/`. `sqlx migrate revert` walks them in reverse order. Manual operator procedure: `sqlx migrate revert --database-url $URL --source crates/vestige-core/migrations/postgres`. +- `vestige migrate copy` is a one-way operation. The source SQLite DB is read-only during the run and untouched afterward; users retain their original file indefinitely. Recommended discipline: copy the SQLite file aside before starting, retain for 30 days. +- `vestige migrate reembed` is destructive to the `embedding` column. Recommended discipline: take a logical backup (`pg_dump --table=memories --table=embedding_model --table=scheduling`) before a reembed run. The tool prints that recommendation before starting and exits non-zero unless `--yes` is passed or the user is on a TTY that confirms. +- Feature-gate strategy: the default build remains SQLite-only. Downstream users pull `postgres-backend` explicitly: `cargo install --features postgres-backend vestige-mcp`. If the Postgres implementation fails in the field, users fall back to SQLite simply by flipping `vestige.toml`'s `[storage] backend = "sqlite"` and restarting. No data re-migration is needed if they retained their SQLite file. +- The `docs/runbook/postgres.md` deliverable (D16) captures this discipline as a one-page ops note. + +--- + +## Open Implementation Questions + +Each item has a recommendation. Ship that unless a reviewer objects. + +### Q1. CLI shape: subcommand split vs flag union + +- **Options**: (a) `vestige migrate copy --from sqlite --to postgres ...` and `vestige migrate reembed --model=...` (subcommand split); (b) `vestige migrate --from sqlite --to postgres ...` and `vestige migrate --reembed --model=...` under one `clap` command with disjoint flag groups (flag union). +- **RECOMMENDATION**: (a) subcommand split. The flag sets do not overlap and clap expresses the constraint more cleanly. The ADR string `vestige migrate --from sqlite --to postgres` can still be documented as a canonical alias by having `copy` accept it verbatim when `--from` is present. + +### Q2. Feature flag name + +- **Options**: `postgres-backend`, `postgres`, `backend-postgres`, `pg`. +- **RECOMMENDATION**: `postgres-backend`. Matches the ADR text and is explicit in `Cargo.toml` feature listings. + +### Q3. sqlx offline mode strategy + +- **Options**: (a) commit `.sqlx/` so downstream builds never need DATABASE_URL; (b) require `DATABASE_URL` at build time. +- **RECOMMENDATION**: (a). The repo already ships as a library; many downstream users will build from crates.io with no Postgres available. Committing `.sqlx/` costs ~100 kB. + +### Q4. HNSW rebuild strategy during reembed + +- **Options**: (a) `DROP INDEX; CREATE INDEX`; (b) `REINDEX INDEX CONCURRENTLY`; (c) `CREATE INDEX CONCURRENTLY` on a new name then swap. +- **RECOMMENDATION**: (a) by default for speed on empty / near-empty tables; expose `--concurrent-index` for large production corpora where locking the table is unacceptable. `REINDEX CONCURRENTLY` on pgvector HNSW is supported in pgvector 0.6+ but the community still reports edge cases with `maintenance_work_mem` -- skip unless a user explicitly opts in. + +### Q5. Connection pool sizing default + +- **Options**: 4, 10, 20, `cpus() * 2`. +- **RECOMMENDATION**: 10. Matches the PRD example, covers a single-operator load, and does not exhaust the default Postgres `max_connections = 100`. Configurable via `vestige.toml`. + +### Q6. Testcontainer image pinning + +- **Options**: (a) `pgvector/pgvector:pg16`; (b) `pgvector/pgvector:pg16.2-0.7.4` (exact tag); (c) maintain local Dockerfile. +- **RECOMMENDATION**: (b) pin exact. The float tag `pg16` has shipped breaking changes in the past (e.g., pg 16.0 to 16.1 interop). Pin to a specific pgvector minor and Postgres patch. CI bumps the tag via a single-line change. + +### Q7. Empty-text and null-embedding behavior in `search` + +- **Options**: (a) return an error if both are missing; (b) return an empty result; (c) return all memories sorted by `created_at DESC`. +- **RECOMMENDATION**: (a). A `search` call with no query is a bug in the caller; returning empty silently would hide the bug. The existing Phase 1 SQLite behavior (TBD but likely errors) is the tiebreaker. + +### Q8. `classify()` SQL vs Rust + +- **Options**: (a) compute cosine to all centroids in SQL (`SELECT id, 1 - (centroid <=> $1::vector) FROM domains ORDER BY ...`); (b) load centroids, compute in Rust. +- **RECOMMENDATION**: (a). Leverages pgvector's SIMD paths and avoids round-tripping centroid vectors. At Phase 4 scale (tens of centroids) the difference is marginal, but the SQL path is simpler and matches the rest of the backend. + +### Q9. FSRS `review_events` writes: trait method vs implicit on `update_scheduling` + +- **Options**: (a) add an explicit `record_review(memory_id, rating, prior, new)` method to the Phase 1 trait; (b) have `update_scheduling` write the event atomically. +- **RECOMMENDATION**: this is a Phase 1 question, not Phase 2. Phase 2 implements whichever Phase 1 chose. If Phase 1 missed it, Phase 2 raises a blocker rather than deciding alone. + +### Q10. `tsvector` weight for tags -- PRD used `array_to_tsvector`, we used `array_to_string` + +- **Options**: (a) `array_to_tsvector(tags)` (requires the `tsvector_extra` extension or similar); (b) `to_tsvector('english', array_to_string(tags, ' '))` (plain core Postgres). +- **RECOMMENDATION**: (b). Equivalent ranking, zero extra extensions. If a future tag matches a stopword (`"the"`), it gets dropped, but that is correct behavior for ranking. + +### Q11. `PgMemoryStore::connect` runs migrations automatically? + +- **Options**: (a) always run `sqlx::migrate!` on connect; (b) require the user to run `vestige migrate-schema` explicitly before starting the server. +- **RECOMMENDATION**: (a) during Phase 2; revisit in Phase 3 when the server binary exists. Developer ergonomics win now, and the migrations are idempotent. + +### Q12. Offline query cache freshness vs `sqlx-cli` version skew + +- **Options**: (a) pin `sqlx-cli` version in CI `actions/cache` step; (b) let CI install whatever version `sqlx` depends on. +- **RECOMMENDATION**: (a) pin to the same 0.8.x as the crate. `sqlx prepare` output changes between 0.7 and 0.8 and must match the runtime. + +--- + +## Sequencing + +The Phase 2 agent executes deliverables in this order; deliverables not listed can run in any order relative to each other. + +1. D1 (feature gate + Cargo deps) -- unblocks everything. +2. D7 (config) -- required to construct `PgMemoryStore`. +3. D4 (migrations SQL) -- required before any `query!` compiles. +4. D3 (pool) + D6 (registry) -- small, used by D2. +5. D2 (`PgMemoryStore` core + trait impl) -- the bulk of Phase 2. +6. D5 (RRF search) -- after D2; requires the trait to exist. +7. D12 (test harness) + parity and search tests -- validates D2 and D5 in isolation. +8. D8 (sqlite->pg migrate) + its integration test. +9. D9 (reembed) + its integration test. +10. D10 (CLI wiring). +11. D11 (`.sqlx/` offline cache) -- last, after SQL is frozen. +12. D15 (benches) + D16 (runbook) -- after acceptance tests pass. + +Each deliverable PR includes its own tests; the final Phase 2 PR stacks them (or lands as a single branch if the Phase 1 trait is stable enough to avoid rebase churn). + +### Critical Files for Implementation + +- /home/delandtj/prppl/vestige/crates/vestige-core/src/storage/postgres/mod.rs +- /home/delandtj/prppl/vestige/crates/vestige-core/migrations/postgres/0001_init.up.sql +- /home/delandtj/prppl/vestige/crates/vestige-core/src/storage/postgres/search.rs +- /home/delandtj/prppl/vestige/crates/vestige-core/src/storage/postgres/migrate_cli.rs +- /home/delandtj/prppl/vestige/crates/vestige-mcp/src/bin/cli.rs diff --git a/docs/plans/0003-phase-3-network-access.md b/docs/plans/0003-phase-3-network-access.md new file mode 100644 index 0000000..500fd5a --- /dev/null +++ b/docs/plans/0003-phase-3-network-access.md @@ -0,0 +1,1435 @@ +# Phase 3 Plan: Network Access and Authentication + +**Status**: Draft +**Depends on**: Phase 1 (MemoryStore trait), Phase 2 (PgMemoryStore, backend config) +**Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 3) + +--- + +## Scope + +### In scope + +- HTTP MCP Streamable endpoint at `POST /mcp` (JSON-RPC body, keep existing + session semantics) and `GET /mcp` (Server-Sent Events for long-running + operations: dream, consolidate, discover, reassign). +- REST API under `/api/v1/` for direct HTTP clients that do not speak MCP + (memories CRUD, search, consolidate trigger, stats, domains + list/rename/merge/discover). +- `api_keys` table + enforcement (blake3-hashed, scopes `read`/`write`, optional + `domain_filter` TEXT[], `last_used` timestamp, `active` flag, revocation). +- Auth middleware with three resolution paths in priority order: + `Authorization: Bearer ` then `X-API-Key: ` then signed session + cookie. All three resolve to the same `ApiKeyIdentity`. +- Signed session cookie: `vestige_session`, SameSite=Strict, HttpOnly, + Secure-when-TLS, Path=/, Max-Age 8 hours. Signed with HMAC-SHA256 using a + key derived from `VESTIGE_SESSION_SECRET` (env) or generated + persisted to + `/session_secret` on first boot. +- `vestige keys create|list|revoke` CLI subcommand (plus `keys rotate` as a + convenience alias of `revoke` + `create`). +- Startup-time refusal to bind non-loopback with `auth.enabled = false` (hard + error, non-zero exit, stderr message, no fallback). +- Dashboard login flow: `POST /dashboard/login` with `{"api_key":"vst_..."}` + JSON body, `X-API-Key` header, or form body; sets signed cookie; returns 200 + JSON `{"ok":true}` for XHR or 303 to `/` if form. Logout at + `POST /dashboard/logout` clears cookie. +- Per-key `domain_filter` enforced inside the auth layer: if the key has + `domain_filter = ["dev","infra"]`, every handler that searches or lists sees + the filter pre-applied via a request extension. Optional + `X-Vestige-Domain: home` header may narrow further but may never escape the + key's filter. +- `[server]` and `[auth]` sections in `vestige.toml`, plus backward-compatible + env var bridges. +- `VESTIGE_AUTH_TOKEN` continues to work for one minor release as a synthetic + single-key fallback, but logs a deprecation warning. +- Per-request request IDs and structured tracing; `last_used` write-back on + successful auth. + +### Out of scope + +- Phase 4 HDBSCAN domain classifier itself. The REST surface exposes domain + endpoints but they may stub to empty results until Phase 4 lands. +- Real TLS termination. Assumed handled by a reverse proxy (nginx, Caddy, + Mycelium). An optional `tls_cert` / `tls_key` pair is documented but its + implementation may be deferred behind a `tls` Cargo feature. +- OAuth / OIDC / SSO. Future work. +- Rate limiting per key (documented in Open Questions, not implemented here). +- WebAuthn / passkey dashboard login. Future work. +- Fine-grained RBAC beyond `read` / `write` scopes. + +## Prerequisites + +Phase 1 artifacts: + +- `vestige_core::storage::MemoryStore` trait (with `Send` variant via + `trait_variant::make`). +- `Embedder` trait. +- `SqliteMemoryStore` implementing `MemoryStore`. + +Phase 2 artifacts: + +- `PgMemoryStore` implementing `MemoryStore`. +- `crates/vestige-core/migrations/postgres/` sqlx migrations; `api_keys` table + schema present but enforcement path is Phase 3's job. +- Runtime backend selection via `vestige.toml` `[storage]` section returning + an `Arc`. + +Assumed already available in workspace: + +- `axum = 0.8` (currently pinned in `crates/vestige-mcp/Cargo.toml`). +- `tower = 0.5`, `tower-http = 0.6` (`cors`, `set-header` features already on). +- `tokio`, `serde`, `serde_json`, `uuid`, `chrono`, `tracing`, + `tracing-subscriber`, `thiserror`, `anyhow`, `subtle`, `clap`, `directories`. + +New crates required (add via `cargo add -p vestige-mcp`): + +- `blake3 = "1"` -- key hashing. +- `rand = "0.9"` with `std_rng` (for key bytes; prefer `rand::rngs::OsRng`). +- `axum-extra = { version = "0.10", features = ["cookie-signed", "typed-header"] }` + -- `SignedCookieJar`, `Cookie`, `Key`. +- `hmac = "0.12"` + `sha2 = "0.10"` -- HMAC-SHA256 for the session secret + derivation (not required if `axum-extra`'s `SignedCookieJar` is used, but + retained for the pure-token-signing path). RECOMMENDATION: rely solely on + `axum-extra::extract::cookie::{Key, SignedCookieJar}`. +- `tower-http` features bump: add `trace` and `request-id`. +- `async-stream = "0.3"` -- emitting SSE events from async closures. +- `futures-util` already present -- for `Stream` adapters. +- `base64 = "0.22"` -- emitting / parsing the random bytes in the `vst_...` + prefix. Use the `URL_SAFE_NO_PAD` alphabet. +- `zeroize = "1"` (optional, recommended) -- scrub the plaintext key in RAM + after hashing. + +`cargo add` commands (do not execute here, leave to implementation): + + cargo add -p vestige-mcp blake3 rand base64 zeroize async-stream + cargo add -p vestige-mcp axum-extra --features cookie-signed,typed-header + cargo add -p vestige-mcp tower-http --features trace,request-id,cors,set-header + +JSON-RPC library: the project uses a hand-rolled `JsonRpcRequest` / +`JsonRpcResponse` pair in `crates/vestige-mcp/src/protocol/types.rs`. Keep it +in Phase 3 (no jsonrpsee migration). Streamable HTTP remains implemented as +`POST /mcp` + session header + `GET /mcp` SSE. See Open Questions for rationale. + +## Deliverables + +1. `crates/vestige-mcp/src/auth/` module (new). Houses key generation, key + verification, identity resolution, scopes, domain-filter extractor, session + key type, and error types. + +2. `crates/vestige-mcp/src/auth/keys.rs` -- key format, generation, + blake3 hashing, store-facing trait methods for list / create / revoke / + verify. + +3. `crates/vestige-mcp/src/auth/middleware.rs` -- axum `from_fn` middleware + that populates `Extension` on the request, rejects unauthenticated + requests with 401, insufficient scope with 403. + +4. `crates/vestige-mcp/src/auth/session.rs` -- `SignedCookieJar` integration, + `session_key()` loader (env or persisted file), `issue_session()` and + `revoke_session()` helpers. + +5. `crates/vestige-mcp/src/http/` module split out of `protocol/http.rs`: + - `http/mcp.rs` -- MCP JSON-RPC endpoint (adapted from the current + `post_mcp` / `delete_mcp`, with auth middleware now gating). + - `http/mcp_sse.rs` -- SSE handler for `GET /mcp` long-running ops. + - `http/rest.rs` -- `/api/v1/*` handlers. + - `http/mod.rs` -- `build_router()`, `start_server()`, bind-safety check, + layer stack assembly. + +6. `crates/vestige-mcp/src/http/errors.rs` -- uniform `ApiError` enum and + `IntoResponse` implementation. Maps to RFC 7807 problem+json for REST and + plain JSON for `/mcp`. + +7. Dashboard patch: `crates/vestige-mcp/src/dashboard/mod.rs` -- add the auth + middleware to the dashboard router, add `/dashboard/login` + `/dashboard/logout` + endpoints, keep `/api/health` unauthenticated. + +8. `crates/vestige-mcp/src/bin/cli.rs` -- new `Keys` subcommand group (`create`, + `list`, `revoke`, `rotate`). + +9. `crates/vestige-mcp/src/config.rs` (new file) -- typed `ServerConfig`, + `AuthConfig`, `StorageConfig` loader from `vestige.toml`, merging env var + overrides, validating the non-loopback + auth-disabled combination. + +10. SQL migration `crates/vestige-core/migrations/postgres/0300_api_keys_enforcement.sql` + and SQLite equivalent `crates/vestige-core/migrations/sqlite/0300_api_keys.sql`: + - `api_keys` table (if not already created in Phase 2), with `key_hash` + UNIQUE, `label` NOT NULL, `scopes` TEXT[] default `{read,write}`, + `domain_filter` TEXT[] default `{}`, `created_at`, `last_used`, + `active BOOLEAN DEFAULT true`. + - Index on `key_hash` (unique already), and on `active WHERE active`. + +11. `MemoryStore` trait extension (Phase 2 may already cover this; if not, + finalize in Phase 3): `list_api_keys`, `create_api_key`, + `revoke_api_key`, `find_api_key_by_hash`, `touch_api_key_last_used`. + +12. Docs updates: + - `docs/env-vars.md` (new) -- one sheet for all runtime env vars. + - `README.md` server-mode section. + - `docs/adr/0001-*.md` -- mark Phase 3 as Implemented when merged. + +## Detailed Task Breakdown + +### D1. Auth module skeleton + +Files: + +- `crates/vestige-mcp/src/auth/mod.rs` +- `crates/vestige-mcp/src/auth/keys.rs` +- `crates/vestige-mcp/src/auth/session.rs` +- `crates/vestige-mcp/src/auth/middleware.rs` +- `crates/vestige-mcp/src/auth/errors.rs` + +`auth/mod.rs`: + + pub mod errors; + pub mod keys; + pub mod middleware; + pub mod session; + + pub use errors::AuthError; + pub use keys::{ApiKey, ApiKeyPlaintext, ApiKeyRecord, Scope}; + pub use middleware::{Identity, auth_layer}; + pub use session::{SessionConfig, session_key}; + +`auth/errors.rs`: + + use axum::http::StatusCode; + use axum::response::{IntoResponse, Response}; + use serde::Serialize; + use thiserror::Error; + + #[derive(Debug, Error)] + pub enum AuthError { + #[error("missing credentials")] + MissingCredentials, + #[error("invalid credentials")] + InvalidCredentials, + #[error("key revoked")] + Revoked, + #[error("insufficient scope: required {required}")] + InsufficientScope { required: &'static str }, + #[error("domain not permitted for this key: {domain}")] + DomainNotAllowed { domain: String }, + #[error("internal auth error")] + Internal, + } + + #[derive(Serialize)] + struct Problem<'a> { + #[serde(rename = "type")] + kind: &'a str, + title: &'a str, + status: u16, + detail: &'a str, + } + + impl IntoResponse for AuthError { + fn into_response(self) -> Response { + let (status, title) = match self { + AuthError::MissingCredentials => (StatusCode::UNAUTHORIZED, "unauthorized"), + AuthError::InvalidCredentials => (StatusCode::UNAUTHORIZED, "unauthorized"), + AuthError::Revoked => (StatusCode::UNAUTHORIZED, "unauthorized"), + AuthError::InsufficientScope { .. } => (StatusCode::FORBIDDEN, "forbidden"), + AuthError::DomainNotAllowed { .. } => (StatusCode::FORBIDDEN, "forbidden"), + AuthError::Internal => (StatusCode::INTERNAL_SERVER_ERROR, "internal"), + }; + let detail = self.to_string(); + let body = axum::Json(Problem { + kind: "about:blank", + title, + status: status.as_u16(), + detail: &detail, + }); + let mut r = (status, body).into_response(); + r.headers_mut().insert( + axum::http::header::CONTENT_TYPE, + axum::http::HeaderValue::from_static("application/problem+json"), + ); + r + } + } + +### D2. Key format and generation + +File: `crates/vestige-mcp/src/auth/keys.rs` + +- Key on wire: `vst_<22-byte base64url-no-pad>`. 22 bytes = 176 bits entropy. + Encoded length ~30 chars. Full string ~34 chars including the `vst_` prefix. +- Hash stored in DB: `blake3(key_plaintext)` hex lowercase (32 bytes -> 64 + hex chars). +- Hash prefix on list: first 12 hex characters, e.g. `key_hash[..12]` for + human display. + +Signatures: + + use blake3::Hasher; + use rand::rngs::OsRng; + use rand::TryRngCore; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use base64::Engine; + use zeroize::Zeroize; + + const KEY_PREFIX: &str = "vst_"; + const KEY_RANDOM_BYTES: usize = 22; + + #[derive(Clone, Debug, PartialEq, Eq)] + pub enum Scope { + Read, + Write, + } + + impl Scope { + pub fn as_str(&self) -> &'static str { + match self { + Scope::Read => "read", + Scope::Write => "write", + } + } + pub fn from_str(s: &str) -> Option { + match s { + "read" => Some(Scope::Read), + "write" => Some(Scope::Write), + _ => None, + } + } + } + + /// The plaintext key. Shown to the user exactly once. + /// Zeroed on drop. + pub struct ApiKeyPlaintext(String); + + impl ApiKeyPlaintext { + pub fn as_str(&self) -> &str { &self.0 } + pub fn into_inner(mut self) -> String { + std::mem::take(&mut self.0) + } + } + + impl Drop for ApiKeyPlaintext { + fn drop(&mut self) { self.0.zeroize(); } + } + + #[derive(Clone, Debug)] + pub struct ApiKeyRecord { + pub id: uuid::Uuid, + pub key_hash: String, // hex-encoded blake3(plaintext) + pub label: String, + pub scopes: Vec, + pub domain_filter: Vec, + pub created_at: chrono::DateTime, + pub last_used: Option>, + pub active: bool, + } + + pub fn generate_key() -> ApiKeyPlaintext { + let mut bytes = [0u8; KEY_RANDOM_BYTES]; + OsRng.try_fill_bytes(&mut bytes).expect("OsRng"); + let encoded = URL_SAFE_NO_PAD.encode(&bytes); + bytes.zeroize(); + ApiKeyPlaintext(format!("{}{}", KEY_PREFIX, encoded)) + } + + pub fn hash_key(plaintext: &str) -> String { + let mut hasher = Hasher::new(); + hasher.update(plaintext.as_bytes()); + hasher.finalize().to_hex().to_string() + } + + pub fn verify_key(plaintext: &str, stored_hash_hex: &str) -> bool { + use subtle::ConstantTimeEq; + let computed = hash_key(plaintext); + computed.as_bytes().ct_eq(stored_hash_hex.as_bytes()).unwrap_u8() == 1 + } + +Helpers on a thin repository trait that both backends implement through +`MemoryStore` (Phase 2 already adds the required columns; Phase 3 wires the +methods): + + #[async_trait::async_trait] + pub trait ApiKeyStore: Send + Sync + 'static { + async fn create_api_key(&self, rec: &ApiKeyRecord) -> anyhow::Result<()>; + async fn find_api_key_by_hash(&self, hash: &str) -> anyhow::Result>; + async fn list_api_keys(&self) -> anyhow::Result>; + async fn revoke_api_key(&self, id: uuid::Uuid) -> anyhow::Result; + async fn touch_api_key_last_used(&self, id: uuid::Uuid) -> anyhow::Result<()>; + } + +(If Phase 2 already bolted these onto `MemoryStore`, `ApiKeyStore` is simply a +re-export of the relevant subset.) + +### D3. Session cookie + +File: `crates/vestige-mcp/src/auth/session.rs` + +- Cookie name: `vestige_session`. +- Cookie attributes: `HttpOnly`, `SameSite=Strict`, `Path=/`, `Max-Age=28800` + (8h), `Secure` when the server is running behind TLS (detected from + `config.server.tls_cert.is_some()` or the `X-Forwarded-Proto` trusted header; + default: set `Secure` whenever `config.server.bind` is non-loopback). +- Payload: serialized `SessionClaims { key_id: Uuid, issued_at: i64, + expires_at: i64 }` encoded as `serde_json` then base64url. The signing is + handled by `axum-extra::extract::cookie::SignedCookieJar` (HMAC via a 64-byte + `Key`). Any tampering or truncation is rejected by the jar automatically. +- Key material: 64 random bytes, stored at `/session_secret` (mode + 0600) or overridden by `VESTIGE_SESSION_SECRET` (base64url-encoded 64 bytes, + reject if shorter). + +Signatures: + + use axum_extra::extract::cookie::{Cookie, Key, SameSite, SignedCookieJar}; + use chrono::{Duration, Utc}; + use serde::{Deserialize, Serialize}; + + const COOKIE_NAME: &str = "vestige_session"; + const DEFAULT_TTL: Duration = Duration::hours(8); + + #[derive(Clone, Serialize, Deserialize)] + pub struct SessionClaims { + pub key_id: uuid::Uuid, + pub iat: i64, + pub exp: i64, + } + + pub fn session_key(data_dir: &std::path::Path) -> anyhow::Result { + // 1) env override + if let Ok(env_val) = std::env::var("VESTIGE_SESSION_SECRET") { + let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(env_val.trim())?; + anyhow::ensure!(raw.len() >= 64, "VESTIGE_SESSION_SECRET must be >= 64 bytes"); + return Ok(Key::from(&raw)); + } + // 2) persisted file + let path = data_dir.join("session_secret"); + if path.exists() { + let bytes = std::fs::read(&path)?; + return Ok(Key::from(&bytes)); + } + // 3) generate + use rand::TryRngCore; + let mut bytes = [0u8; 64]; + rand::rngs::OsRng.try_fill_bytes(&mut bytes)?; + #[cfg(unix)] + { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + std::fs::create_dir_all(data_dir).ok(); + let mut f = std::fs::OpenOptions::new() + .create_new(true).write(true).mode(0o600).open(&path)?; + f.write_all(&bytes)?; + f.sync_all()?; + } + #[cfg(not(unix))] + std::fs::write(&path, &bytes)?; + Ok(Key::from(&bytes)) + } + + pub fn issue_session( + jar: SignedCookieJar, + key_id: uuid::Uuid, + secure: bool, + ) -> SignedCookieJar { + let now = Utc::now(); + let claims = SessionClaims { + key_id, + iat: now.timestamp(), + exp: (now + DEFAULT_TTL).timestamp(), + }; + let value = serde_json::to_string(&claims).expect("serialize claims"); + let mut cookie = Cookie::new(COOKIE_NAME, value); + cookie.set_http_only(true); + cookie.set_same_site(SameSite::Strict); + cookie.set_path("/"); + cookie.set_max_age(cookie::time::Duration::seconds(DEFAULT_TTL.num_seconds())); + cookie.set_secure(secure); + jar.add(cookie) + } + + pub fn revoke_session(jar: SignedCookieJar) -> SignedCookieJar { + jar.remove(Cookie::from(COOKIE_NAME)) + } + + pub fn claims_from(jar: &SignedCookieJar) -> Option { + let c = jar.get(COOKIE_NAME)?; + let claims: SessionClaims = serde_json::from_str(c.value()).ok()?; + if claims.exp < Utc::now().timestamp() { return None; } + Some(claims) + } + +### D4. Auth middleware + +File: `crates/vestige-mcp/src/auth/middleware.rs` + +Identity carried through the request: + + #[derive(Clone, Debug)] + pub struct Identity { + pub key_id: uuid::Uuid, + pub label: String, + pub scopes: Vec, + pub domain_filter: Vec, + pub via: AuthVia, + } + + #[derive(Clone, Copy, Debug)] + pub enum AuthVia { + Bearer, + ApiKeyHeader, + SessionCookie, + } + +Middleware (axum 0.8): + + use axum::extract::{Request, State}; + use axum::http::{header, StatusCode}; + use axum::middleware::Next; + use axum::response::{IntoResponse, Response}; + use axum_extra::extract::cookie::SignedCookieJar; + use std::sync::Arc; + + pub async fn auth_layer( + State(state): State>, + jar: SignedCookieJar, + mut request: Request, + next: Next, + ) -> Response { + // Allowlist endpoints that never require auth: + let path = request.uri().path(); + if path == "/api/health" || path == "/api/v1/health" || + path == "/dashboard/login" { + return next.run(request).await; + } + + let via_and_key = extract_credentials(request.headers(), &jar); + let outcome = match via_and_key { + Some((AuthVia::Bearer, key)) | Some((AuthVia::ApiKeyHeader, key)) => { + resolve_by_plaintext(&state, &key).await.map(|id| (id, via_and_key.unwrap().0)) + } + Some((AuthVia::SessionCookie, key_id_str)) => { + let id = uuid::Uuid::parse_str(&key_id_str).map_err(|_| AuthError::InvalidCredentials)?; + resolve_by_key_id(&state, id).await.map(|id| (id, AuthVia::SessionCookie)) + } + None => Err(AuthError::MissingCredentials), + }; + + let identity = match outcome { + Ok((id, via)) => Identity { via, ..id }, + Err(e) => return e.into_response(), + }; + + // touch last_used asynchronously; do not block request path + let st2 = state.clone(); + let kid = identity.key_id; + tokio::spawn(async move { let _ = st2.store.touch_api_key_last_used(kid).await; }); + + request.extensions_mut().insert(identity); + next.run(request).await + } + +Credential extraction (priority: Bearer > X-API-Key > cookie): + + fn extract_credentials( + headers: &axum::http::HeaderMap, + jar: &SignedCookieJar, + ) -> Option<(AuthVia, String)> { + if let Some(v) = headers.get(header::AUTHORIZATION).and_then(|h| h.to_str().ok()) { + if let Some(rest) = v.strip_prefix("Bearer ") { + return Some((AuthVia::Bearer, rest.trim().to_string())); + } + } + if let Some(v) = headers.get("x-api-key").and_then(|h| h.to_str().ok()) { + return Some((AuthVia::ApiKeyHeader, v.trim().to_string())); + } + if let Some(claims) = crate::auth::session::claims_from(jar) { + return Some((AuthVia::SessionCookie, claims.key_id.to_string())); + } + None + } + +Resolution helpers: + + async fn resolve_by_plaintext(st: &AppCtx, key: &str) -> Result { + let hash = crate::auth::keys::hash_key(key); + let rec = st.store.find_api_key_by_hash(&hash).await + .map_err(|_| AuthError::Internal)? + .ok_or(AuthError::InvalidCredentials)?; + if !rec.active { return Err(AuthError::Revoked); } + Ok(Identity { + key_id: rec.id, label: rec.label, scopes: rec.scopes, + domain_filter: rec.domain_filter, via: AuthVia::Bearer, + }) + } + + async fn resolve_by_key_id(st: &AppCtx, id: uuid::Uuid) -> Result { + let rec = st.store.find_api_key_by_id(id).await + .map_err(|_| AuthError::Internal)? + .ok_or(AuthError::InvalidCredentials)?; + if !rec.active { return Err(AuthError::Revoked); } + Ok(Identity { + key_id: rec.id, label: rec.label, scopes: rec.scopes, + domain_filter: rec.domain_filter, via: AuthVia::SessionCookie, + }) + } + +Scope guard extractor (per-handler opt-in): + + pub struct RequireScope; + impl axum::extract::FromRequestParts for RequireScope + where S: Send + Sync, + { + type Rejection = AuthError; + async fn from_request_parts( + parts: &mut axum::http::request::Parts, _state: &S, + ) -> Result { + let id = parts.extensions.get::().ok_or(AuthError::MissingCredentials)?; + let need = if WRITE { Scope::Write } else { Scope::Read }; + if !id.scopes.contains(&need) { + return Err(AuthError::InsufficientScope { + required: if WRITE { "write" } else { "read" }, + }); + } + Ok(RequireScope) + } + } + +Domain scoping: + + /// Returns the effective domain filter for the request: + /// - Intersect the key's domain_filter with any X-Vestige-Domain header. + /// - Empty key filter means "all domains", so the header is authoritative. + /// - A header that names a domain outside the key filter returns + /// `Err(DomainNotAllowed)`. + pub fn effective_domain_filter( + id: &Identity, header: Option<&str>, + ) -> Result>, AuthError> { + let header_dom = header.map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + match (id.domain_filter.as_slice(), header_dom) { + ([], None) => Ok(None), + ([], Some(h)) => Ok(Some(vec![h])), + (filter, None) => Ok(Some(filter.to_vec())), + (filter, Some(h)) => { + if filter.iter().any(|d| d == &h) { + Ok(Some(vec![h])) + } else { + Err(AuthError::DomainNotAllowed { domain: h }) + } + } + } + } + +### D5. Layer ordering + +Router assembly in `http/mod.rs::build_router`: + + let trace = tower_http::trace::TraceLayer::new_for_http(); + let request_id = tower_http::request_id::SetRequestIdLayer::x_request_id( + tower_http::request_id::MakeRequestUuid); + let propagate_id = tower_http::request_id::PropagateRequestIdLayer::x_request_id(); + + let cors = CorsLayer::new() + .allow_origin(cfg.server.allowed_origins()) + .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE, Method::OPTIONS]) + .allow_headers([header::CONTENT_TYPE, header::AUTHORIZATION, + HeaderName::from_static("x-api-key"), + HeaderName::from_static("x-vestige-domain"), + HeaderName::from_static("mcp-session-id")]) + .allow_credentials(true); + + let app = Router::new() + // Unauth routes first (not subjected to auth_layer by path allowlist) + .route("/api/health", get(health)) + .route("/dashboard/login", post(login)) + .route("/dashboard/logout", post(logout)) + // MCP + REST + dashboard + .route("/mcp", post(http::mcp::post_mcp).get(http::mcp_sse::get_mcp_sse) + .delete(http::mcp::delete_mcp)) + .nest("/api/v1", http::rest::router()) + .merge(dashboard::router()) + // Auth middleware applied via from_fn_with_state (allowlist inside) + .layer(axum::middleware::from_fn_with_state(ctx.clone(), auth_layer)) + // Outermost: tracing, request-id, cors, body limit, concurrency + .layer( + ServiceBuilder::new() + .layer(trace) + .layer(request_id) + .layer(propagate_id) + .layer(cors) + .layer(DefaultBodyLimit::max(MAX_BODY_SIZE)) + .layer(ConcurrencyLimitLayer::new(CONCURRENCY_LIMIT)) + ) + .with_state(ctx); + +Axum applies `layer()` calls outermost-first in the order they are declared. +The result here: request -> trace -> request-id -> CORS -> body-limit -> +concurrency -> auth -> handler. Auth must wrap the handlers but be inside +tracing so its spans can log auth outcomes. + +### D6. MCP endpoints + +File: `crates/vestige-mcp/src/http/mcp.rs` + +`POST /mcp` -- keep the session-based structure already in `protocol/http.rs` +but use the `Identity` injected by the auth layer instead of a shared +`auth_token`: + + pub async fn post_mcp( + State(ctx): State>, + Extension(id): Extension, + headers: HeaderMap, + Json(request): Json, + ) -> Response { ... } + +Auth happens in the layer, so this handler cannot be reached without a valid +`Identity`. Scope check: all MCP writes (tools that mutate) require +`RequireScope`. Use an enum of MCP methods or a method -> required-scope +map. `tools/list`, `resources/list`, `initialize`, `ping` are read-only. +`tools/call` is conservatively classified as write; the per-tool dispatch +inside `McpServer::handle_tools_call` may further reject writes when the tool +name is read-only and the key lacks write. + +`DELETE /mcp` -- unchanged semantics, drops the session. + +`GET /mcp` -- SSE. Implementation in `http/mcp_sse.rs`: + + use axum::response::sse::{Event, KeepAlive, Sse}; + use axum::extract::Query; + use futures_util::stream::Stream; + use async_stream::stream; + use std::time::Duration; + + #[derive(serde::Deserialize)] + pub struct SseParams { + pub op: String, // "dream" | "consolidate" | "discover" | "reassign" + pub session: Option, // optional operation correlation id + } + + pub async fn get_mcp_sse( + State(ctx): State>, + Extension(_id): Extension, + Query(params): Query, + ) -> Result>>, AuthError> { + let op = params.op.clone(); + let ctx2 = ctx.clone(); + let s = stream! { + yield Ok(Event::default().event("start").data(format!("{{\"op\":\"{}\"}}", op))); + match op.as_str() { + "dream" => { + let mut rx = ctx2.cognitive.lock().await.begin_dream_stream().await; + while let Some(ev) = rx.recv().await { + yield Ok(Event::default().event("progress").json_data(ev)?); + } + yield Ok(Event::default().event("done").data("{}")); + } + "consolidate" => { /* same pattern over Storage::run_consolidation_stream */ } + "discover" => { /* Phase 4 */ } + "reassign" => { /* Phase 4 */ } + other => { + yield Ok(Event::default().event("error") + .data(format!("{{\"message\":\"unknown op {}\"}}", other))); + } + } + }; + Ok(Sse::new(s).keep_alive(KeepAlive::new().interval(Duration::from_secs(15)))) + } + +SSE event shape (stable contract, document in `docs/http-api.md`): + + event: start + data: {"op":"dream"} + + event: progress + data: {"stage":"replay","processed":12,"total":50} + + event: progress + data: {"stage":"cross_reference","processed":25,"total":50} + + event: done + data: {"nodes_processed":50,"duration_ms":14320} + +The `keep-alive` hint is 15s to survive most proxy timeouts. + +### D7. REST API + +File: `crates/vestige-mcp/src/http/rest.rs` + +Routes: + + pub fn router() -> Router> { + Router::new() + .route("/health", get(health)) + .route("/memories", post(create_memory).get(list_memories)) + .route("/memories/{id}", get(get_memory).put(update_memory).delete(delete_memory)) + .route("/memories/{id}/promote", post(promote_memory)) + .route("/memories/{id}/demote", post(demote_memory)) + .route("/search", post(search_memories)) + .route("/consolidate", post(trigger_consolidation)) + .route("/stats", get(get_stats)) + .route("/domains", get(list_domains)) + .route("/domains/discover", post(trigger_discovery)) + .route("/domains/{id}", put(rename_domain).delete(delete_domain)) + .route("/domains/{id}/merge", post(merge_domain)) + .route("/keys", post(create_key).get(list_keys)) + .route("/keys/{id}", delete(revoke_key)) + } + +Representative signatures: + + #[derive(serde::Deserialize)] + pub struct CreateMemoryReq { + pub content: String, + pub node_type: Option, + pub tags: Option>, + pub source: Option, + pub metadata: Option, + } + + #[derive(serde::Serialize)] + pub struct MemoryView { /* flat projection of MemoryRecord */ } + + pub async fn create_memory( + State(ctx): State>, + Extension(id): Extension, + _: RequireScope, + Json(req): Json, + ) -> Result<(StatusCode, Json), ApiError> { + let effective = effective_domain_filter(&id, None)?; + let rec = ctx.store.insert_from_rest(req, effective).await?; + Ok((StatusCode::CREATED, Json(MemoryView::from(rec)))) + } + + pub async fn search_memories( + State(ctx): State>, + Extension(id): Extension, + _: RequireScope, + headers: HeaderMap, + Json(req): Json, + ) -> Result, ApiError> { + let dom_header = headers.get("x-vestige-domain").and_then(|h| h.to_str().ok()); + let effective = effective_domain_filter(&id, dom_header)?; + let q = SearchQuery { domains: effective, ..req.into() }; + let res = ctx.store.search(&q).await?; + Ok(Json(SearchResp::from(res))) + } + +`trigger_consolidation` returns 202 Accepted + a JSON body with a `session_id` +the client may pass to `GET /mcp?op=consolidate&session=...` to stream +progress. + +### D8. Error mapping + +File: `crates/vestige-mcp/src/http/errors.rs` + + #[derive(Debug, thiserror::Error)] + pub enum ApiError { + #[error(transparent)] Auth(#[from] AuthError), + #[error("bad request: {0}")] BadRequest(String), + #[error("not found")] NotFound, + #[error("conflict: {0}")] Conflict(String), + #[error(transparent)] Store(#[from] anyhow::Error), + } + + impl IntoResponse for ApiError { + fn into_response(self) -> Response { + match self { + ApiError::Auth(a) => a.into_response(), + ApiError::BadRequest(m) => (StatusCode::BAD_REQUEST, problem(400, "bad_request", &m)).into_response(), + ApiError::NotFound => (StatusCode::NOT_FOUND, problem(404, "not_found", "")).into_response(), + ApiError::Conflict(m) => (StatusCode::CONFLICT, problem(409, "conflict", &m)).into_response(), + ApiError::Store(e) => { + tracing::error!(err = %e, "store error"); + (StatusCode::INTERNAL_SERVER_ERROR, problem(500, "internal", "internal error")).into_response() + } + } + } + } + +All MCP JSON-RPC error mapping is unchanged (done in `McpServer`); only +transport-level errors (401/403) leave that path. + +### D9. Config loader and bind-safety check + +File: `crates/vestige-mcp/src/config.rs` + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct ServerConfig { + #[serde(default = "default_bind")] + pub bind: String, // "127.0.0.1:3928" + #[serde(default = "default_dashboard_port")] + pub dashboard_port: u16, + #[serde(default)] pub tls_cert: Option, + #[serde(default)] pub tls_key: Option, + #[serde(default)] pub allowed_origins: Vec, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct AuthConfig { + #[serde(default = "default_true")] + pub enabled: bool, + #[serde(default)] pub session_secret_file: Option, + } + + impl ServerConfig { + pub fn parsed_bind(&self) -> anyhow::Result { + self.bind.parse().map_err(|e: std::net::AddrParseError| + anyhow::anyhow!("invalid bind {}: {}", self.bind, e)) + } + } + +Bind-safety check (called during `start_server`): + + pub fn enforce_bind_safety(server: &ServerConfig, auth: &AuthConfig) -> anyhow::Result<()> { + let addr = server.parsed_bind()?; + let is_loopback = match addr.ip() { + std::net::IpAddr::V4(v) => v.is_loopback(), + std::net::IpAddr::V6(v) => v.is_loopback(), + }; + if !is_loopback && !auth.enabled { + anyhow::bail!( + "refusing to bind {} with auth disabled; \ + set [auth] enabled = true in vestige.toml or \ + change [server] bind to a loopback address", + addr + ); + } + Ok(()) + } + +`main.rs` and the `serve` CLI both call `enforce_bind_safety` before +`TcpListener::bind`. On failure: `eprintln!` the error, `std::process::exit(2)`. + +Env bridge: + +- `VESTIGE_HTTP_BIND` (existing) -> `server.bind` host part. +- `VESTIGE_HTTP_PORT` (existing) -> `server.bind` port part. +- `VESTIGE_DASHBOARD_PORT` (existing) -> `server.dashboard_port`. +- `VESTIGE_AUTH_TOKEN` (deprecated) -- when set, synthesize a virtual + `ApiKeyRecord` with `id = all-zero UUID`, `scopes = [read, write]`, + `domain_filter = []`, `active = true`, hash stored in memory only. Log a + warning on every startup: `VESTIGE_AUTH_TOKEN is deprecated; use 'vestige + keys create' and set auth.enabled=true instead. Will be removed in v2.2.0.` +- `VESTIGE_SESSION_SECRET` -- see D3. + +### D10. Dashboard login + logout + +File: `crates/vestige-mcp/src/dashboard/handlers.rs` (additions). + + #[derive(serde::Deserialize)] + pub struct LoginBody { + pub api_key: String, + } + + pub async fn login( + State(state): State, + jar: SignedCookieJar, + headers: HeaderMap, + body: Option>, + ) -> Result<(SignedCookieJar, Json), AuthError> { + // Accept key in either JSON body or X-API-Key header + let plaintext = body.map(|b| b.0.api_key) + .or_else(|| headers.get("x-api-key").and_then(|h| h.to_str().ok()).map(String::from)) + .ok_or(AuthError::MissingCredentials)?; + + let hash = crate::auth::keys::hash_key(&plaintext); + let rec = state.store.find_api_key_by_hash(&hash).await + .map_err(|_| AuthError::Internal)? + .ok_or(AuthError::InvalidCredentials)?; + if !rec.active { return Err(AuthError::Revoked); } + + let secure = state.config.server.tls_cert.is_some(); + let jar = crate::auth::session::issue_session(jar, rec.id, secure); + + Ok((jar, Json(serde_json::json!({ + "ok": true, "key_id": rec.id, "label": rec.label, + "scopes": rec.scopes.iter().map(|s| s.as_str()).collect::>(), + "domains": rec.domain_filter, + })))) + } + + pub async fn logout(jar: SignedCookieJar) + -> (SignedCookieJar, Json) + { + (crate::auth::session::revoke_session(jar), + Json(serde_json::json!({"ok": true}))) + } + +Dashboard router integration: login/logout are appended before `auth_layer` +is applied, so they are reachable unauthenticated. The dashboard SPA asset +routes (`/dashboard`, `/dashboard/{*path}`) remain publicly readable so the +login page can load; the `/api/*` dashboard endpoints are gated by +`auth_layer`. (The existing health endpoint keeps its current behaviour.) + +### D11. `vestige keys` CLI + +File: `crates/vestige-mcp/src/bin/cli.rs` additions. + + #[derive(Subcommand)] + enum Commands { + // ... existing + /// Manage API keys + Keys { + #[command(subcommand)] + sub: KeyCmd, + }, + } + + #[derive(Subcommand)] + enum KeyCmd { + /// Create a new API key + Create { + #[arg(long)] label: String, + #[arg(long, value_delimiter = ',', default_values_t = ["read".to_string(), "write".to_string()])] + scopes: Vec, + /// Restrict the key to listed domains (comma-separated). Empty = all domains. + #[arg(long, value_delimiter = ',')] + domains: Vec, + }, + /// List existing keys (never shows plaintext) + List { + /// Include revoked keys in the output + #[arg(long)] all: bool, + }, + /// Revoke a key by id or by hash prefix + Revoke { + /// Id (UUID) or hash prefix (first 12 hex chars) + id_or_prefix: String, + }, + /// Revoke and re-create with the same scopes/label + Rotate { + id_or_prefix: String, + }, + } + +`Create` outputs the plaintext exactly once on stdout (for piping into env +files) and a confirmation on stderr. Use colored output only on stderr to keep +stdout machine-readable. + + fn run_keys_create(...) -> anyhow::Result<()> { + let store = open_store()?; // Arc + let plaintext = crate::auth::keys::generate_key(); + let hash = crate::auth::keys::hash_key(plaintext.as_str()); + let rec = ApiKeyRecord { + id: uuid::Uuid::new_v4(), + key_hash: hash, label, scopes, domain_filter: domains, + created_at: chrono::Utc::now(), + last_used: None, active: true, + }; + block_on(store.create_api_key(&rec))?; + + // stderr: human-readable + eprintln!("{} {}", "Created key:".green().bold(), rec.label); + eprintln!(" id: {}", rec.id); + eprintln!(" scopes: {}", rec.scopes.iter().map(|s| s.as_str()).collect::>().join(",")); + eprintln!(" domains: {}", if rec.domain_filter.is_empty() { "all".to_string() } else { rec.domain_filter.join(",") }); + eprintln!(); + eprintln!("{}", "Store the plaintext key now. It will not be shown again.".yellow()); + // stdout: ONLY the plaintext, for scripting + println!("{}", plaintext.as_str()); + Ok(()) + } + +`List`: + + kid label scopes domains last_used hash + d3a8... macbook read,write all 2026-04-20 11:02 a1b2c3d4e5f6 + ... + +Never print the plaintext. Show only `hash[..12]`. + +### D12. Migrations + +Postgres `0300_api_keys.sql` (idempotent; Phase 2 may have already created the +table, in which case this migration is a no-op `CREATE TABLE IF NOT EXISTS`): + + CREATE TABLE IF NOT EXISTS api_keys ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + key_hash TEXT NOT NULL UNIQUE, + label TEXT NOT NULL, + scopes TEXT[] NOT NULL DEFAULT ARRAY['read','write'], + domain_filter TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used TIMESTAMPTZ, + active BOOLEAN NOT NULL DEFAULT true + ); + + CREATE INDEX IF NOT EXISTS idx_api_keys_active + ON api_keys (active) WHERE active; + +SQLite `0300_api_keys.sql`: + + CREATE TABLE IF NOT EXISTS api_keys ( + id TEXT PRIMARY KEY, + key_hash TEXT NOT NULL UNIQUE, + label TEXT NOT NULL, + scopes TEXT NOT NULL DEFAULT 'read,write', -- comma-joined + domain_filter TEXT NOT NULL DEFAULT '', -- comma-joined, '' = all + created_at TEXT NOT NULL DEFAULT (datetime('now')), + last_used TEXT, + active INTEGER NOT NULL DEFAULT 1 + ); + + CREATE INDEX IF NOT EXISTS idx_api_keys_active + ON api_keys (active) WHERE active = 1; + +Both backends' trait impls convert to/from `ApiKeyRecord`. + +### D13. Wiring main.rs and the `serve` CLI path + +`main.rs` refactor: + +1. `Config::load()` reads `vestige.toml` (if present) and overlays env vars. +2. Run `enforce_bind_safety(&cfg.server, &cfg.auth)` before spawning any + listener. On failure, print to stderr and exit 2. +3. Build `AppCtx` with `Arc`, `CognitiveEngine`, + event bus, `session_key`, `config`. +4. `build_router(ctx)` returns a single Axum `Router` that covers MCP, REST, + and dashboard. +5. `axum::serve(listener, app).await`. +6. The stdio MCP transport continues to run in parallel (unchanged) for + desktop / Claude Code single-user scenarios. + +`serve` CLI subcommand: identical flow minus stdio. + +### D14. Docs + +- `docs/env-vars.md` new: table of every supported env var, default, purpose, + deprecation status. +- Section in `README.md`: "Running Vestige as a network server". +- Cheat-sheet section in `CLAUDE.md` for: create a key, start the server, + curl smoke test. + +## Test Plan + +### Unit tests (colocated under `#[cfg(test)]`) + +- `auth/keys.rs`: + - `generate_key_has_prefix_and_length()` -- asserts `vst_` prefix and 34-ish + char total, regex `^vst_[A-Za-z0-9_-]{29}$`. + - `hash_key_blake3_is_stable_and_hex()` -- fixed vector test. + - `verify_key_accepts_same_input()` / `verify_key_rejects_tampered()` / + `verify_key_rejects_length_mismatch()`. + - `keys_are_unique_in_a_loop()` -- 10_000 iterations, no collisions. + - `plaintext_zeroed_on_drop()` -- unsafe peek into the backing buffer + through a wrapper that exposes bytes for the test only. + +- `auth/session.rs`: + - `round_trip_claims_through_signed_jar()`. + - `expired_cookie_is_rejected()` -- mint a cookie with `exp = iat - 60` and + confirm `claims_from` returns `None`. + - `tampered_cookie_is_rejected()` -- flip one byte in the signed segment, + confirm the jar drops it. + - `session_key_env_overrides_file()`. + - `session_key_generated_file_has_mode_0600_on_unix()`. + +- `auth/middleware.rs`: + - `extract_credentials_prefers_bearer_over_api_key_header()`. + - `extract_credentials_falls_back_to_cookie()`. + - `effective_domain_filter_empty_means_all()`. + - `effective_domain_filter_header_narrows_within_key_filter()`. + - `effective_domain_filter_rejects_header_outside_key_filter()`. + - `missing_credentials_returns_401()`. + - `revoked_key_returns_401()`. + - `insufficient_scope_returns_403()`. + +- `config.rs`: + - `parse_vestige_toml_with_server_and_auth_sections()`. + - `env_vars_override_toml_bind()`. + - `enforce_bind_safety_rejects_0_0_0_0_with_auth_disabled()`. + - `enforce_bind_safety_allows_0_0_0_0_with_auth_enabled()`. + - `enforce_bind_safety_allows_loopback_with_auth_disabled()`. + +- `http/errors.rs`: + - `not_found_emits_problem_json_with_correct_content_type()`. + - `bad_request_includes_detail_field()`. + +- `http/mcp.rs`: + - `post_mcp_unauth_returns_401()` (this would normally be caught by the + middleware; kept as a unit test by constructing the Router minus the + middleware to exercise the handler's own error paths). + +### Integration tests (`tests/phase_3/`) + +All tests spin up the full Axum stack in-process on a random port via +`tokio::net::TcpListener::bind("127.0.0.1:0")`, wire a `SqliteMemoryStore` in +a `TempDir`, and issue HTTP calls with `reqwest`. + +Files (each one a standalone binary test file): + +- `phase_3/common/mod.rs` -- shared harness (`spawn_server()`, + `create_test_key()`, `client()`). + +- `phase_3/http_mcp_round_trip.rs` -- boot server, mint a key, send + `initialize` over `POST /mcp` with `Authorization: Bearer vst_...`, follow + with `tools/list`, assert we see the expected tool count (greater than 20). + +- `phase_3/http_sse_stream.rs` -- `POST /api/v1/consolidate` returns 202 + + `session_id`. `GET /mcp?op=consolidate&session=...` streams at least one + `progress` and one `done` event. Use `eventsource-client` dev dep, or parse + the stream manually. + +- `phase_3/rest_api_crud.rs` -- exercises each REST endpoint in turn: + - `POST /api/v1/memories` -> 201 + body. + - `GET /api/v1/memories/{id}` -> 200. + - `PUT /api/v1/memories/{id}` -> 200. + - `POST /api/v1/search` -> 200 with the new memory in results. + - `POST /api/v1/memories/{id}/promote` -> 200. + - `GET /api/v1/stats` -> 200. + - `GET /api/v1/domains` -> 200 (likely empty). + - `DELETE /api/v1/memories/{id}` -> 204. + +- `phase_3/auth_bearer_token.rs`: + - unauth: `GET /api/v1/stats` returns 401 and `Content-Type: + application/problem+json`. + - valid Bearer: same call returns 200. + - revoked key: `POST /api/v1/keys/{id}` DELETE then reuse -> 401. + - tampered Bearer (last char flipped) -> 401. + +- `phase_3/auth_api_key_header.rs`: + - `X-API-Key: vst_...` alone -> 200. + - Both Bearer and X-API-Key with different values -> Bearer wins (asserted + via a key that is read-only in Bearer + full-scope X-API-Key, then + confirming a write 403s). + +- `phase_3/auth_session_cookie.rs`: + - `POST /dashboard/login` with valid key -> 200 + `Set-Cookie: + vestige_session=...; HttpOnly; SameSite=Strict; Path=/`. + - reuse cookie: `GET /api/v1/stats` returns 200. + - tampered cookie (change one char): -> 401. + - `POST /dashboard/logout` -> `Set-Cookie: vestige_session=; Max-Age=0`. + +- `phase_3/auth_domain_filter.rs`: + - Key with `domain_filter = ["dev"]`: + - `POST /api/v1/search` without header -> search is scoped to `["dev"]` + (insert fixtures with two domains, assert only `dev` rows returned). + - `X-Vestige-Domain: dev` -> same. + - `X-Vestige-Domain: home` -> 403 with detail `domain not permitted`. + - Key with empty filter + `X-Vestige-Domain: dev` -> scoped to `["dev"]`. + - Key with empty filter + no header -> no scoping. + +- `phase_3/auth_scope_enforcement.rs`: + - read-only key cannot call `POST /api/v1/memories` -> 403. + - read-only key CAN call `POST /api/v1/search` -> 200. + +- `phase_3/bind_safety_nonlocalhost_without_auth.rs`: + - Spawn `vestige serve --bind 0.0.0.0:0` as a subprocess with `auth.enabled + = false` via a temp `vestige.toml`. + - Assert: non-zero exit, stderr contains `refusing to bind`, no listener + ever opens (confirm by trying to connect to the configured port and + expecting connection refused after a short timeout). + +- `phase_3/cli_keys_create_list_revoke.rs`: + - Spawn the `vestige` CLI binary with `--data-dir `. + - Run `vestige keys create --label test --scopes read,write`; capture + stdout (the plaintext) and stderr (the human summary). Assert `vst_` + prefix in stdout. + - Run `vestige keys list`; assert no plaintext, label `test` present. + - Run `vestige keys revoke `; confirm exit 0. + - Run `vestige keys list`; assert label no longer visible without `--all`. + +- `phase_3/dashboard_login_flow.rs`: + - Full loop: login -> fetch `/dashboard` (gets SPA index, unauthed ok) -> + fetch `/api/memories` (authed via cookie) -> logout -> fetch `/api/memories` + (401). + +- `phase_3/deprecation_auth_token.rs`: + - Start the server with `VESTIGE_AUTH_TOKEN=test12345...` and no created + keys. Send a Bearer request with that token -> 200. Assert stderr log + contains `deprecated`. + +### Smoke test (`tests/phase_3/smoke/`) + +- `remote_mcp_client.sh`: + + #!/usr/bin/env bash + set -euo pipefail + KEY="${VESTIGE_TEST_KEY:?set me}" + HOST="${VESTIGE_HOST:-http://127.0.0.1:3928}" + # Initialize a session + RESP=$(curl -sS -D /tmp/h -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize", + "params":{"protocolVersion":"2025-11-25", + "clientInfo":{"name":"smoke","version":"0"}, + "capabilities":{}}}' \ + "$HOST/mcp") + SID=$(grep -i 'mcp-session-id:' /tmp/h | awk '{print $2}' | tr -d '\r') + # tools/list + curl -sS -H "Authorization: Bearer $KEY" \ + -H "Mcp-Session-Id: $SID" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \ + "$HOST/mcp" | jq '.result.tools | length' + echo "smoke ok" + +## Acceptance Criteria + +- [ ] `cargo build -p vestige-mcp` -- zero warnings, all feature combinations + (`--no-default-features`, default, `--features ort-dynamic`). +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings`. +- [ ] `cargo fmt --all --check`. +- [ ] All `tests/phase_3/*.rs` pass, plus phase_1 and phase_2 remain green. +- [ ] Unauth request to `POST /mcp` returns 401 with + `Content-Type: application/problem+json` and a body containing `status`, + `title`, `detail`. +- [ ] Binding `0.0.0.0:` with `[auth].enabled = false` makes the + process exit with code 2 and print `refusing to bind` to stderr. +- [ ] `vestige keys create --label X` prints exactly one line on stdout + matching `^vst_[A-Za-z0-9_-]+$`; `vestige keys list` never prints that + line back. +- [ ] Dashboard login from a browser-like client (tested via the reqwest + `Client::cookie_store(true)` harness) yields a `Set-Cookie` with + `HttpOnly`, `SameSite=Strict`, `Path=/`, and Max-Age present. +- [ ] A second machine can run a curl-based MCP client against the server + (smoke test) and receive successful `tools/list` responses. +- [ ] `VESTIGE_AUTH_TOKEN` still works and emits the deprecation warning. +- [ ] `tests/phase_3/auth_domain_filter.rs` demonstrates that a key scoped to + `dev` cannot read `home`-domain memories via any of the three auth modes + and cannot escape with `X-Vestige-Domain`. + +## Rollback Notes + +- Ship behind an on-by-default Cargo feature `http-server` on + `vestige-mcp`. Disabling it reverts to stdio + existing localhost HTTP + (`protocol/http.rs` in its current form) with zero behaviour change. +- SQL: migration `0300_api_keys.sql` is additive only; rollback is a single + `DROP TABLE api_keys;` in `0300_api_keys.down.sql` for both backends. Keep a + row count safety check in the down migration and log the deletion. +- Session secret file: deleting `/session_secret` invalidates every + outstanding cookie; users simply log in again. Safe to rotate. +- Env var sunset schedule: + - v2.1.x: `VESTIGE_AUTH_TOKEN` emits a warning, still works. + - v2.2.0: `VESTIGE_AUTH_TOKEN` refused with an error pointing at + `vestige keys create`. +- Downgrade procedure: `git revert` the Phase 3 merge, then run the down + migration. No data loss; plaintext keys were only ever in user-side + secret managers. + +## Open Implementation Questions + +1. JSON-RPC library: hand-rolled vs jsonrpsee? + + - Candidate A: keep the hand-rolled types in `protocol/types.rs` plus the + session-aware `post_mcp` handler already in `protocol/http.rs`. + - Candidate B: switch to `jsonrpsee = "0.24"` with the `server` feature + and adapt it to Axum via `jsonrpsee::server::Server`. + + RECOMMENDATION: A. Phase 3 is about auth and transport surfaces, not + library rewrites. The existing types are already correct, tested, and + compatible with Streamable HTTP; the 29 cognitive modules depend on + `McpServer::handle_request`, which does not map 1:1 to jsonrpsee's + `RpcModule` trait. Re-evaluate in a future phase only if we need subscription + notifications beyond SSE. + +2. Streamable HTTP vs plain POST-with-JSON? + + - The MCP spec titled "Streamable HTTP" defines: `POST /mcp` for + request/response, `GET /mcp` for SSE where the client subscribes to + server-initiated messages, and an `Mcp-Session-Id` header for session + correlation. The current implementation already covers POST + session + header + DELETE; Phase 3 adds the GET/SSE half. + + RECOMMENDATION: implement the full Streamable HTTP transport. Long-running + tools (dream, consolidate, discover) benefit from SSE progress events, and + Claude Desktop / Claude Code both speak Streamable HTTP natively. Keeping + POST-only would work for short calls but block the UX we want for + background jobs. + +3. Session cookie crate? + + - Candidate A: `axum-extra::extract::cookie::SignedCookieJar` with a 64-byte + `Key`. + - Candidate B: `tower-sessions = "0.13"` with the `MemoryStore` or + `PostgresStore` session backend. + - Candidate C: stateless JWT via `jsonwebtoken`. + + RECOMMENDATION: A. We do not need server-side session state (the `api_keys` + row is the state; the cookie is merely a signed pointer to it). B adds a + whole storage backend we do not need. C adds signing-algorithm surface area + and revocation becomes awkward ("revoked key" with a long-lived JWT). + `SignedCookieJar` gives us HMAC-signed cookies for free, integrates with + axum extractors, and the payload is tiny. + +4. Key format and length? + + - 22 random bytes base64url-no-pad = 176 bits entropy, encoded ~30 chars, + full key ~34 chars with the `vst_` prefix. Long enough to make + brute-force infeasible, short enough to paste into config files. + - Alternatives: 32 bytes (40 chars, overkill), 16 bytes (128 bits, marginal + for secret material shared over networks). + + RECOMMENDATION: 22 bytes. Prefix `vst_` is already documented in the PRD + and gives grep-ability. + +5. Rate limiting: in scope for Phase 3? + + - Useful: mitigates slow brute force, runaway agents. + - Expensive to design well (per-key, per-IP, per-endpoint). + + RECOMMENDATION: OUT of scope. Track as `docs/adr/0002-rate-limiting.md` + follow-up. Axum + `tower` has `ConcurrencyLimitLayer` (already used); a + follow-up can add `governor` or `tower_governor` behind the auth layer so + identity is available. + +6. CORS policy defaults for dashboard in server mode? + + - Candidate A: allow only origins derived from `server.bind` host + the + dashboard port. + - Candidate B: allow user-listed origins via `server.allowed_origins` + config, with A as fallback. + - Candidate C: open CORS to `*` when TLS is configured. + + RECOMMENDATION: B. Auto-populate `allowed_origins` from the bind address + and dashboard port at start time; if the operator sets the config list, + use that list verbatim. Never `*` (`allow_credentials = true` is + incompatible with `*` anyway). + +7. Dashboard session lifetime? + + - 8 hours for default; configurable via `auth.session_ttl_hours`. + - Rotate on each write? (Rolling sessions.) + + RECOMMENDATION: 8 hours fixed, non-rolling. Revisit if users complain. + +8. Handling `tools/call` scope granularity? + + - Today, `tools/call` is a single MCP method. Read-only tools like + `search`, `deep_reference`, `predict` should be callable with a + read-only key. + + RECOMMENDATION: map tool names to scopes in `McpServer::handle_tools_call`. + Read-only names: `search`, `session_context`, `memory` with action in + `{get, state, get_batch}`, `deep_reference`, `cross_reference`, `predict`, + `explore_connections`, `find_duplicates`, `memory_timeline`, + `memory_changelog`, `memory_health`, `memory_graph`, `importance_score`, + `system_status`. Everything else requires `write`. If a read-only key + calls a write tool, return a JSON-RPC error with code `-32003` + ("server not initialized" is close but wrong; reuse `-32603 internal` with + a descriptive message or add a new `-32004 UnauthorizedTool`). RECOMMEND + adding `-32004`. + +9. How to bridge `MemoryStore` trait with dashboard state (`AppState`)? + + - Today `AppState.storage: Arc` is a concrete type. + - Phase 2 introduces `Arc`. + + RECOMMENDATION: in Phase 3, introduce `AppCtx { store: Arc, + cognitive, config, event_tx }` as the single state type for the unified + router. Keep `AppState` as a thin wrapper (or alias) if the dashboard + handlers need to stay untouched in this phase. Migrate the dashboard + handlers to the trait in a follow-up refactor to contain the blast radius. + +10. Windows support for `session_secret` and `auth_token` file modes? + + - Unix gets `0600` via `OpenOptionsExt`. + - Windows has no direct equivalent; ACLs differ. + + RECOMMENDATION: document the limitation; use default permissions on + Windows. Add a `#[cfg(windows)]` placeholder to set owner-only ACLs via + `windows-acl` in a follow-up, not Phase 3. + +### Critical Files for Implementation + +- /home/delandtj/prppl/vestige/crates/vestige-mcp/src/protocol/http.rs +- /home/delandtj/prppl/vestige/crates/vestige-mcp/src/dashboard/mod.rs +- /home/delandtj/prppl/vestige/crates/vestige-mcp/src/main.rs +- /home/delandtj/prppl/vestige/crates/vestige-mcp/src/bin/cli.rs +- /home/delandtj/prppl/vestige/crates/vestige-mcp/Cargo.toml diff --git a/docs/plans/0004-phase-4-emergent-domain-classification.md b/docs/plans/0004-phase-4-emergent-domain-classification.md new file mode 100644 index 0000000..d9f2355 --- /dev/null +++ b/docs/plans/0004-phase-4-emergent-domain-classification.md @@ -0,0 +1,883 @@ +# Phase 4 Plan: Emergent Domain Classification + +**Status**: Draft +**Depends on**: Phase 1 (domain columns on memories, `Domain` struct + `DomainStore` methods on `MemoryStore`, `Embedder` trait), Phase 2 (Postgres JSONB + TEXT[] support for domain fields, `embedding_model` registry parity), Phase 3 (Axum HTTP server, REST `/api/v1/` scaffolding, API key auth middleware, signed dashboard session cookies) +**Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 4), docs/prd/001-getting-centralized-vestige.md (Emergent Domain Model) + +--- + +## Scope + +### In scope + +- `DomainClassifier` cognitive module under `crates/vestige-core/src/neuroscience/domain_classifier.rs`, alongside existing neuroscience modules (spreading_activation, synaptic_tagging, ...). +- HDBSCAN discovery pipeline using the `hdbscan` crate (v0.10): load all embeddings, cluster, extract centroids, extract top-terms via TF-IDF over cluster members, persist via the trait's `DomainStore` methods. +- Soft-assignment pipeline: for each memory, compute `cosine_similarity(memory.embedding, domain.centroid)` for every domain, store raw scores in `domain_scores` JSONB, threshold into `domains[]` using `assign_threshold` (default 0.65). +- Automatic classification on ingest: run through `CognitiveEngine` / `smart_ingest` so new memories get classified against existing centroids immediately; skip when `domain_count == 0` (Phase 0 accumulation). +- Re-cluster hook in dream consolidation: every Nth four-phase dream cycle (N=5 default) triggers a discovery pass and generates proposals (split / merge / none). Proposals land in a new `domain_proposals` table, surface in the dashboard, and are never auto-applied (conservative drift, ADR Q7). +- Context signals: `SignalSource` trait with `GitRepoSignal` (detects `.git` in CWD or `metadata.cwd`) and `IdeHintSignal` (reads `metadata.editor` / `metadata.ide`). Each returns a `boost_map` of `domain_id -> additive delta` (typical +0.05). Injected as a `signal_boost: Option>` parameter into `DomainClassifier::classify`. +- Cross-domain spreading activation decay: `ActivationNetwork` traversal multiplies the edge's effective weight by `cross_domain_decay` (default 0.5) when `target.domains` and `source.domains` are disjoint. Strict "no overlap" policy, not graded. +- CLI subcommands (in `crates/vestige-mcp/src/bin/cli.rs`, under a new `Domains` command group): `list`, `discover [--min-cluster-size N] [--force]`, `rename `, `merge [--into ]`. Human-readable tables on stdout; JSON via `--json`. +- Dashboard UI additions (`apps/dashboard/src/routes/(app)/domains/`): list page, per-domain detail (memories, centroid top_terms, score histogram, proposal review controls). +- REST endpoints under `/api/v1/domains` (introduced by Phase 3 skeleton, implemented in Phase 4): list, discover, rename, merge, proposal list / accept / reject. +- Config additions: `[domains]` section in `vestige.toml` covering `assign_threshold`, `recluster_interval`, `min_cluster_size`, `cross_domain_decay`, `discovery_threshold`, `merge_threshold`, `signal_boost` (per-signal toggle). + +### Out of scope + +- Phase 5 federation (explicit separate ADR). Domain centroids are installation-local; no sync. +- Learned re-weighting of domain scores (future, only if retrieval-quality metrics show a need). +- Interactive cluster-membership editing in the UI (drag-and-drop reassign) -- future enhancement. +- Multi-user domain namespaces. One domain set per installation; API keys that carry `domain_filter` just restrict access, they do not create namespaces. +- Auto-sweep of `min_cluster_size` / auto-tuned `assign_threshold` (ADR resolution Q6 + Q9: static defaults, user tunes). +- Graded cross-domain decay (`|A intersect B| / max(|A|,|B|)`) -- strict "no overlap" is the Phase 4 rule. + +--- + +## Prerequisites + +Artifacts that Phases 1-3 are expected to have landed: + +- In `vestige-core`: + - `Embedder` trait (`crates/vestige-core/src/embedder/`). + - `MemoryStore` trait (`crates/vestige-core/src/storage/trait.rs` or similar) including `DomainStore` methods: `list_domains`, `get_domain`, `upsert_domain`, `delete_domain`, `classify(&[f32]) -> Vec<(String, f64)>`, plus a bulk accessor such as `all_embeddings()` (already present in sqlite.rs as `get_all_embeddings`) and a `get_all_memories_with_embeddings()` iterator for discovery. The trait must expose a method to batch-update `(domains, domain_scores)` for a memory id. + - `Domain` struct: `{ id: String, label: String, centroid: Vec, top_terms: Vec, memory_count: usize, created_at: DateTime }`. + - Columns on memories in both SQLite and Postgres: `domains TEXT[]` (or JSON array on SQLite) and `domain_scores JSONB` (or TEXT JSON on SQLite). + - The `domains` table in both backends (see PRD schema sketch). +- In `vestige-mcp`: + - Axum `/api/v1/` router prefix with auth middleware. + - CLI skeleton (`bin/cli.rs`) using `clap`; Phase 4 adds a `Domains` subcommand tree. + - REST handlers file structure ready under `crates/vestige-mcp/src/dashboard/handlers.rs` (legacy) and a dedicated REST handler under `/api/v1/`; Phase 4 adds `domains.rs` handler module. + - SvelteKit dashboard (`apps/dashboard/`) with existing `(app)/memories`, `(app)/timeline`, `(app)/stats`, etc. Phase 4 adds `(app)/domains/`. + +New workspace crate additions required (added manually to `Cargo.toml`, since `cargo add` is not run from the plan): + +- `hdbscan = "0.10"` in `crates/vestige-core/Cargo.toml` (feature-gated behind `domain-classification`). +- Optional: a lightweight stop-word constant inline; no external stop-word crate -- the neuroscience modules already do tokenization on whitespace + length>3 (see `dreams.rs::content_similarity`). Reuse that style; no `ndarray` needed because `hdbscan` v0.10 accepts `&[Vec]` directly (verified from PRD snippet). +- No new deps in `vestige-mcp` for Phase 4 -- CLI reuses `clap` / `colored` / `comfy-table` if already present, otherwise a hand-rolled padded print. We pick hand-rolled to avoid adding a table crate; this matches the existing style of `run_stats` in `cli.rs`. + +Test fixtures: + +- A JSON seed corpus checked into `tests/phase_4/fixtures/seed_500.json` containing >= 500 memories drawn from three plausible clusters. A builder function `tests/phase_4/support/fixtures.rs::build_seed_corpus()` deterministically generates or loads this corpus. Each record has `content`, `tags`, `embedding` (768D bge-base-en-v1.5; use a committed vector or a deterministic mock embedder in tests). For deterministic tests we fake embeddings by hashing content -- acceptable as long as the fake preserves cluster separability (prefix-based: "DEV-...", "INFRA-...", "HOME-..." seeds three Gaussian blobs). +- Reuse `Embedder` mock from Phase 1 tests (`MockEmbedder`) for discovery tests that need real cosine similarity. +- A minimal git-repo fixture created in a tempdir (`tempfile::tempdir` + `std::process::Command::new("git").arg("init")`) for context-signal tests. + +--- + +## Deliverables + +1. `DomainClassifier` cognitive module: struct, defaults, `classify`, `classify_with_boost`, `reassign_all`, `discover`. +2. `domain_terms` helper (TF-IDF over cluster members, returning `top_k` terms). +3. `cli domains discover` subcommand. +4. `cli domains list` / `rename` / `merge` subcommands. +5. Auto-classify hook on ingest (wired into the cognitive engine's ingest pipeline before persistence). +6. Re-cluster hook in dream consolidation (`DreamEngine::run` orchestrator gets an optional `DomainReClusterHook`; triggers every Nth dream). +7. Context signal extractor module (`crates/vestige-core/src/neuroscience/context_signals.rs`) with `SignalSource` trait + `GitRepoSignal` + `IdeHintSignal`. +8. Cross-domain spreading activation decay in `ActivationNetwork::activate` (config-driven). +9. `vestige.toml` `[domains]` section + defaults loader. +10. Dashboard UI: SvelteKit routes `(app)/domains/+page.svelte` (list), `(app)/domains/[id]/+page.svelte` (detail), `(app)/domains/proposals/+page.svelte` (review). +11. REST endpoints under `/api/v1/domains` + `/api/v1/domains/proposals`. +12. `domain_proposals` table + migration + `DomainProposal` trait methods on `MemoryStore`. +13. WebSocket event `VestigeEvent::DomainProposalCreated` so the dashboard gets a live notification after a re-cluster fires. + +--- + +## Detailed Task Breakdown + +### 1. `DomainClassifier` cognitive module + +**File**: `crates/vestige-core/src/neuroscience/domain_classifier.rs` +**Export**: in `crates/vestige-core/src/neuroscience/mod.rs`, add `pub mod domain_classifier;` and re-export `pub use domain_classifier::{DomainClassifier, ClassificationResult, DomainProposal, ProposalKind};` +**Deps**: `hdbscan = "0.10"`, `serde`, `serde_json`, `chrono`, `tracing`, existing `crate::storage::Domain`, `crate::storage::MemoryStore` trait. + +Struct and defaults (match PRD exactly): + +```rust +pub struct DomainClassifier { + pub assign_threshold: f64, // default 0.65 + pub discovery_threshold: usize, // default 150 + pub recluster_interval: usize, // default 5 (every 5th dream) + pub min_cluster_size: usize, // default 10 + pub min_samples: usize, // default 5 (HDBSCAN) + pub cross_domain_decay: f64, // default 0.5 + pub merge_threshold: f64, // default 0.90 (centroid cosine) + pub top_terms_k: usize, // default 10 +} + +impl Default for DomainClassifier { ... } +``` + +Result types: + +```rust +#[derive(Debug, Clone)] +pub struct ClassificationResult { + pub scores: HashMap, // raw per-domain similarities + pub domains: Vec, // above assign_threshold +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProposalKind { + Split { parent: String, children: Vec }, + Merge { targets: Vec, suggested_label: String }, + NewCluster { top_terms: Vec }, +} + +#[derive(Debug, Clone)] +pub struct DomainProposal { + pub id: String, // uuid v4 + pub kind: ProposalKind, + pub rationale: String, + pub confidence: f64, + pub created_at: DateTime, + pub status: ProposalStatus, // Pending | Accepted | Rejected +} +``` + +Key methods (all pure where possible; all pub): + +```rust +impl DomainClassifier { + pub fn classify(&self, embedding: &[f32], domains: &[Domain]) -> ClassificationResult; + + pub fn classify_with_boost( + &self, + embedding: &[f32], + domains: &[Domain], + boost: Option<&HashMap>, + ) -> ClassificationResult; + + pub async fn reassign_all( + &self, + store: &dyn MemoryStore, + domains: &[Domain], + ) -> Result; + + pub async fn discover( + &self, + store: &dyn MemoryStore, + ) -> Result, StorageError>; + + pub async fn propose_changes( + &self, + store: &dyn MemoryStore, + existing: &[Domain], + newly_discovered: &[Domain], + ) -> Result, StorageError>; + + pub async fn apply_proposal( + &self, + store: &dyn MemoryStore, + proposal: &DomainProposal, + ) -> Result<(), StorageError>; +} +``` + +Behavior notes: + +- `classify` returns empty `{ scores: {}, domains: [] }` iff `domains.is_empty()` (accumulation phase). This matches the PRD snippet verbatim. +- `classify_with_boost` adds the boost delta to each score AFTER cosine, before thresholding. It clamps to `[0.0, 1.0]`. Boost keys not present in `domains` are ignored. +- `reassign_all` streams memories in batches of 500 (iterator on the store) to keep memory bounded; for each memory issues a single `UPDATE memories SET domains = ?, domain_scores = ? WHERE id = ?` call. Returns count of memories whose `domains` vector actually changed. +- `discover` loads all `(id, embedding)` pairs via an `all_embeddings()` method on the store (exists under `#[cfg(all(feature = "embeddings", feature = "vector-search"))]` in `sqlite.rs::get_all_embeddings`; Phase 1 should promote this onto the trait -- if not yet promoted, add the method). Then: + 1. Build `Vec>` and index -> id map. + 2. `Hdbscan::default_hyper_params(&embeddings).min_cluster_size(self.min_cluster_size).min_samples(self.min_samples).build()` (exact builder depends on hdbscan 0.10 surface; see Open Question). + 3. `let labels = clusterer.cluster()?;` + 4. `let centers = clusterer.calc_centers(Center::Centroid, &labels)?;` + 5. Group indices by label ignoring -1 (noise). For each cluster compute `top_terms` via `compute_top_terms`. + 6. Preserve stable IDs where possible: match each new cluster centroid to the closest existing domain by cosine; if similarity > 0.85, reuse the existing domain id + label. Otherwise generate a fresh id `cluster_{n}` with a label derived from the first 2 terms. + 7. Upsert all resulting `Domain`s via the store. +- `propose_changes` compares old vs new clusters: + - **Split**: an old domain that best-matches two or more new domains each with >= `min_cluster_size` members. Rationale: "domain `dev` is now 2 clusters of >=10 memories: `systems` and `networking`". + - **Merge**: two old domains whose centroids now satisfy `cosine > merge_threshold` get a merge proposal. + - **NewCluster**: a new cluster that doesn't match any old domain above 0.85 similarity. +- `apply_proposal` runs the split or merge against the store (reassign memberships via `reassign_all`), then marks the proposal `Accepted`. It never runs automatically -- only via the CLI or dashboard. + +Helper: + +```rust +fn compute_top_terms(documents: &[&str], k: usize) -> Vec; +``` + +Uses TF-IDF with IDF computed over the entire passed-in corpus (the `documents` slice), tokenization = whitespace split, lowercase, strip non-alphanumeric, drop tokens shorter than 4 chars and a small built-in stop-word list (`the`, `and`, `for`, `that`, `with`, ...). Matches the tokenizer used in `dreams.rs::content_similarity` and `dreams.rs::extract_patterns` so behavior is predictable. + +Cosine similarity helper: + +```rust +fn cosine_similarity(a: &[f32], b: &[f32]) -> f64; +``` + +Keep the existing crate-level `cosine_similarity` if already present (check `embeddings::` or `search::`); otherwise add a private one. Returns 0.0 on dimension mismatch, panics would be a bug. + +### 2. Top-terms computation helper + +**File**: same module, private section. + +- `fn tokenize(text: &str) -> Vec`: lowercase, split on non-alphanumeric, filter len >= 4, drop stop-words. +- `fn tfidf_top_k(docs: &[&str], k: usize) -> Vec`: + 1. `tf[doc_idx][term] = count / total_terms`. + 2. `df[term] = docs containing term`. + 3. `idf[term] = log((N + 1) / (df[term] + 1)) + 1` (smoothed). + 4. For each term, average `tf` across docs in the cluster; multiply by `idf`; sort desc; return top `k`. + +Cluster top-terms are computed over cluster members only, with IDF over the **whole corpus** (all memory contents), not the cluster, so common words get penalized globally. Recompute global IDF once per `discover` call. + +### 3. CLI subcommand: `vestige domains discover` + +**File**: `crates/vestige-mcp/src/bin/cli.rs` + +Add to `enum Commands`: + +```rust +/// Emergent domain management +Domains { + #[command(subcommand)] + action: DomainAction, +}, +``` + +```rust +#[derive(clap::Subcommand)] +enum DomainAction { + /// List all discovered domains + List { + #[arg(long)] json: bool, + }, + /// Run HDBSCAN discovery on all embeddings and propose domains + Discover { + #[arg(long, default_value_t = 10)] min_cluster_size: usize, + /// Skip the proposal flow and write new domains directly (first-time use) + #[arg(long)] force: bool, + #[arg(long)] json: bool, + }, + /// Rename a domain (by id) + Rename { + id: String, + new_label: String, + }, + /// Merge two domains + Merge { + a: String, + b: String, + #[arg(long)] into: Option, // default: `a` + }, +} +``` + +Handler plumbing lives in `run_domains(action)` dispatching to `run_domains_list`, `run_domains_discover`, `run_domains_rename`, `run_domains_merge`. Each opens the default `Storage`, constructs a `DomainClassifier::default()`, and invokes the appropriate method. + +Output format for `list`: + +``` +ID LABEL MEMORIES TOP TERMS +dev Development 87 rust, trait, async, tokio, zinit +infra Infrastructure 47 bgp, sonic, vlan, frr, peering +home Home 31 solar, kwh, battery, pool, esphome +(unclassified) 12 +``` + +Produced via plain `print!` with `%-15s %-18s %-10d %s` style padding. `--json` emits `serde_json::to_string_pretty(&domains)`. + +Output format for `discover` with `--force`: + +``` +HDBSCAN: 500 embeddings, min_cluster_size=10, min_samples=5 +Found 3 clusters (ignoring 14 noise points) + cluster_0 (N=47) top: bgp, sonic, vlan, frr, peering + cluster_1 (N=31) top: solar, kwh, battery, pool, esphome + cluster_2 (N=22) top: rust, trait, async, tokio, zinit + +Writing 3 domains to the store... +Soft-assigning 500 memories against centroids... + multi-domain: 43 + single-domain: 412 + unclassified (below threshold 0.65): 45 +Done in 7.4s. +``` + +Output format for `discover` without `--force` (post-Phase-0): + +``` +HDBSCAN: 623 embeddings, min_cluster_size=10 +Comparing to existing 3 domains... + +Proposals (pending, accept via dashboard or `vestige domains proposals`): + [split] dev -> (systems:34, networking:28) confidence 0.82 + [new] cluster_5 (books, novels, reading) confidence 0.71 + +Run `vestige domains proposals` to review, or open the dashboard. +``` + +### 4. CLI: `list`, `rename`, `merge` + +- `list`: calls `store.list_domains()`, fetches unclassified count via `store.count_memories_without_domains()` (Phase 1 should have provided this; if not, Phase 4 adds it to the trait and both backends). +- `rename`: `store.get_domain(id)` -> mutate `label` -> `store.upsert_domain`. No memory touch. +- `merge`: load both, compute blended centroid (weighted by `memory_count`), merge `top_terms` (union, recompute TF-IDF rank if both sides share the corpus), delete the non-`into` domain, call `reassign_all`. Wrapped in a transaction on Postgres; on SQLite rely on the existing writer-lock pattern. + +### 5. Auto-classify on ingest + +**File**: `crates/vestige-core/src/cognitive.rs` (or equivalent ingest entry in `vestige-mcp/src/tools/smart_ingest.rs`). + +Integration point: just before the record is persisted in the smart-ingest path, after the embedder has produced `embedding` and before `storage.insert(...)`. Trace the current call site -- today `Storage::ingest(IngestInput)` computes embedding inside storage; in Phase 1 the embedder becomes external (ADR decision Q2), so classification can hook right there in the cognitive engine. + +Pseudocode: + +```rust +let embedding = embedder.embed(&input.content).await?; +let domains = store.list_domains().await?; + +let (domains_assigned, domain_scores) = if domains.is_empty() { + (Vec::new(), HashMap::new()) +} else { + let boost = context_signals.gather_boost(&input.metadata, &domains); + let result = classifier.classify_with_boost(&embedding, &domains, boost.as_ref()); + (result.domains, result.scores) +}; + +record.embedding = Some(embedding); +record.domains = domains_assigned; +record.domain_scores = domain_scores; +store.insert(&record).await?; +``` + +Edge cases: + +- Accumulation phase (`domains.is_empty()`): skip classification entirely. Zero overhead. +- Embedding failed / skipped: leave `domains = []`, `domain_scores = {}`. Never fail ingest because of classification. +- Metric: emit `VestigeEvent::MemoryClassified { id, domains, top_score }` on the WebSocket bus so the dashboard sees it live. + +### 6. Re-cluster hook in dream consolidation + +**File**: `crates/vestige-core/src/advanced/dreams.rs` (long file, 1131-line `dream()` entry on the `MemoryDreamer` impl) plus `crates/vestige-core/src/consolidation/phases.rs` (the `DreamEngine::run` orchestrator). + +Design: the `DreamEngine::run(...)` returns `FourPhaseDreamResult`. It does not currently know how many times it has run. Phase 4 introduces a persistent counter on disk (column `dream_cycle_count` on a new singleton `system_state` table, or a simple row in the existing `metadata` / `embedding_model` registry). After the Integration phase finishes, the cognitive engine increments the counter and, if `counter % recluster_interval == 0`, launches discovery asynchronously: + +Extension struct in `phases.rs`: + +```rust +pub struct DreamReClusterHook<'a> { + pub classifier: &'a DomainClassifier, + pub store: &'a dyn MemoryStore, + pub event_tx: Option<&'a tokio::sync::mpsc::UnboundedSender>, +} + +impl<'a> DreamReClusterHook<'a> { + pub async fn tick(&self, cycle_count: usize) -> Result, StorageError> { + if cycle_count == 0 || cycle_count % self.classifier.recluster_interval != 0 { + return Ok(vec![]); + } + let existing = self.store.list_domains().await?; + let rediscovered = self.classifier.discover(self.store).await?; + let proposals = self + .classifier + .propose_changes(self.store, &existing, &rediscovered) + .await?; + for p in &proposals { + self.store.insert_domain_proposal(p).await?; + if let Some(tx) = self.event_tx { + let _ = tx.send(VestigeEvent::DomainProposalCreated { + id: p.id.clone(), + kind: format!("{:?}", p.kind), + confidence: p.confidence, + timestamp: Utc::now(), + }); + } + } + Ok(proposals) + } +} +``` + +Caller wires `tick()` after `DreamEngine::run()` returns, at the ingest/consolidation orchestrator level. The hook never mutates existing domains -- it only writes proposals. The acceptance path is manual (CLI or dashboard). + +Counter storage: add method `store.bump_dream_cycle_count() -> Result` returning the new count. Single-row table: + +```sql +CREATE TABLE IF NOT EXISTS system_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +-- seed: ('dream_cycle_count', '0') +``` + +### 7. Context signal extractor + +**File**: `crates/vestige-core/src/neuroscience/context_signals.rs` + +```rust +pub trait SignalSource: Send + Sync { + /// Returns domain_id -> additive boost (positive or negative, typically in [-0.1, +0.1]). + fn boost_map( + &self, + input_metadata: &serde_json::Value, + domains: &[Domain], + ) -> HashMap; + + fn name(&self) -> &'static str; +} + +pub struct GitRepoSignal { + pub boost: f64, // default +0.05 +} + +pub struct IdeHintSignal { + pub boost: f64, +} + +pub struct ContextSignals { + sources: Vec>, +} + +impl ContextSignals { + pub fn gather_boost( + &self, + input_metadata: &serde_json::Value, + domains: &[Domain], + ) -> Option>; +} +``` + +Signal encoding convention (document in the module header): + +- A signal is a **soft prior**. It nudges the post-cosine score by a small additive delta, clamped to `[-0.10, +0.10]` per signal. +- Multiple signals sum, then the final boost per domain is clamped to `[-0.15, +0.15]` so signals cannot by themselves push a memory into or out of a domain; the embedding similarity dominates. +- Signals target domains by heuristic: `GitRepoSignal` boosts any domain whose `top_terms` overlaps `{"rust","async","trait","function","class","def","git","commit","fn","code"}`. `IdeHintSignal` does the same for `{"file","line","editor","vscode","neovim","rust-analyzer","lsp"}`. +- All signal boosts are logged via `tracing::debug!` so users can audit why a memory picked up a domain. + +`GitRepoSignal::boost_map` implementation: + +```rust +fn boost_map(&self, meta: &Value, domains: &[Domain]) -> HashMap { + let is_git = meta.get("cwd") + .and_then(|v| v.as_str()) + .map(|cwd| std::path::Path::new(cwd).join(".git").exists()) + .unwrap_or(false) + || meta.get("git_repo").is_some(); + if !is_git { return HashMap::new(); } + let mut out = HashMap::new(); + for d in domains { + let code_hits = d.top_terms.iter() + .filter(|t| CODE_TERMS.contains(t.as_str())) + .count(); + if code_hits > 0 { out.insert(d.id.clone(), self.boost); } + } + out +} +``` + +Config knob in `[domains.signals]`: `git = true`, `ide = true`, `git_boost = 0.05`, `ide_boost = 0.05`. + +### 8. Cross-domain spreading activation decay + +**File**: `crates/vestige-core/src/neuroscience/spreading_activation.rs` + +Modify `ActivationConfig`: + +```rust +pub struct ActivationConfig { + pub decay_factor: f64, + pub max_hops: u32, + pub min_threshold: f64, + pub allow_cycles: bool, + pub cross_domain_decay: f64, // NEW, default 0.5 +} +``` + +Domain metadata on nodes: the current `ActivationNode` has `id`, `activation`, `last_activated`, `edges: Vec`. Phase 4 adds `pub domains: Vec`. Populated when nodes get added (propagated from the memory's `domains` field). The network is rebuilt on each search from the store; if the in-memory network is persisted (check `ActivationNetwork` lifetime in `CognitiveEngine`), the population happens in the engine at boot and on insert. + +Traversal change, in `ActivationNetwork::activate` loop, replacing the single line `let propagated = current_activation * edge.strength * self.config.decay_factor;`: + +```rust +let cross_penalty = { + let src_doms = self.nodes.get(¤t_id).map(|n| &n.domains); + let tgt_doms = self.nodes.get(&target_id).map(|n| &n.domains); + match (src_doms, tgt_doms) { + (Some(s), Some(t)) if !s.is_empty() && !t.is_empty() => { + let overlap = s.iter().any(|d| t.contains(d)); + if overlap { 1.0 } else { self.config.cross_domain_decay } + } + _ => 1.0, // unclassified on either side: no penalty + } +}; +let propagated = current_activation * edge.strength * self.config.decay_factor * cross_penalty; +``` + +Rationale for "unclassified -> no penalty": unclassified memories are Phase-0 or low-confidence corpus members; penalizing them would block useful cross-pollination during the accumulation ramp. + +API to update a node's domains after reclassification: + +```rust +pub fn set_node_domains(&mut self, id: &str, domains: Vec); +``` + +Called by the reassignment pipeline after `reassign_all`. + +### 9. `vestige.toml` `[domains]` section + +**File**: wherever `vestige.toml` is loaded (search for `[storage]` / `[server]` loaders). Add: + +```toml +[domains] +assign_threshold = 0.65 +discovery_threshold = 150 +recluster_interval = 5 +min_cluster_size = 10 +min_samples = 5 +cross_domain_decay = 0.5 +merge_threshold = 0.90 +top_terms_k = 10 + +[domains.signals] +git = true +ide = true +git_boost = 0.05 +ide_boost = 0.05 +``` + +Rust-side: `DomainsConfig { ... }` struct with `serde(default)` so `vestige.toml` without a `[domains]` section falls back to hard-coded defaults. `DomainClassifier::from_config(cfg: &DomainsConfig) -> Self`. + +### 10. Dashboard UI additions + +**SvelteKit routes** (`apps/dashboard/src/routes/(app)/domains/`): + +- `+page.svelte` (list): fetches `GET /api/v1/domains` and `GET /api/v1/domains/unclassified-count`. Renders a table: `label`, `memories`, `top_terms` chips, `created_at`. Each row links to `/domains/[id]`. A "Discover" button posts `POST /api/v1/domains/discover`. +- `[id]/+page.svelte` (detail): fetches `GET /api/v1/domains/:id`, `GET /api/v1/domains/:id/memories?limit=100`, `GET /api/v1/domains/:id/score-histogram`. Renders: + - Header: label (editable, triggers `PUT /api/v1/domains/:id`), top-terms chips, memory count, created_at. + - Histogram: a vertical bar chart of `domain_scores[:id]` buckets 0-0.1, 0.1-0.2, ..., 0.9-1.0 across all memories. Data source: server precomputes buckets so the client does not need to fetch all scores. + - Memory list: paginated, each row shows the raw score for this domain. +- `proposals/+page.svelte`: fetches `GET /api/v1/domains/proposals?status=pending`. Each pending proposal card shows `kind`, `rationale`, `confidence`, `created_at`, buttons "Accept" (posts `POST /api/v1/domains/proposals/:id/accept`) and "Reject" (`POST .../reject`). Live updates via the existing WebSocket channel (`/ws`) reacting to `DomainProposalCreated` events. + +Styling reuses the existing Tailwind + shadcn-svelte conventions in `apps/dashboard/src/lib/components/`. + +Existing `(app)/stats` and `(app)/feed` pages get a small "Domains" summary panel that links to `/domains`. + +### 11. REST endpoints + +**File**: `crates/vestige-mcp/src/protocol/http.rs` or a new `crates/vestige-mcp/src/api/domains.rs` module, wired into the `/api/v1/` router. + +| Method | Path | Handler | +|--------|------|---------| +| GET | `/api/v1/domains` | `list_domains` -- returns `[Domain...]` + unclassified count | +| POST | `/api/v1/domains/discover` | `trigger_discover` -- body `{ min_cluster_size?: usize, force?: bool }`, returns proposals or applied domains | +| GET | `/api/v1/domains/:id` | `get_domain` | +| PUT | `/api/v1/domains/:id` | `update_domain` -- rename | +| DELETE | `/api/v1/domains/:id` | `delete_domain` -- with `?merge_into=other_id` | +| GET | `/api/v1/domains/:id/memories` | paginated memories in this domain | +| GET | `/api/v1/domains/:id/score-histogram` | precomputed buckets | +| GET | `/api/v1/domains/proposals` | `list_proposals?status=pending` | +| POST | `/api/v1/domains/proposals/:id/accept` | `accept_proposal` | +| POST | `/api/v1/domains/proposals/:id/reject` | `reject_proposal` | + +All handlers go through the Phase 3 auth middleware (Bearer / X-API-Key / session cookie). Responses are JSON; error paths use `StatusCode::*` with a small `{"error": "..."}` body. + +### 12. `domain_proposals` table + trait methods + +Postgres migration (`crates/vestige-core/migrations/postgres/00XX_domain_proposals.sql`): + +```sql +CREATE TABLE domain_proposals ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + kind TEXT NOT NULL, -- 'split' | 'merge' | 'new_cluster' + payload JSONB NOT NULL, -- serialized ProposalKind body + rationale TEXT NOT NULL, + confidence DOUBLE PRECISION NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', -- pending|accepted|rejected + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + resolved_at TIMESTAMPTZ +); +CREATE INDEX idx_domain_proposals_status ON domain_proposals (status, created_at DESC); +``` + +SQLite migration: same table, `UUID` -> `TEXT`, `JSONB` -> `TEXT` with JSON-encoded bodies, `TIMESTAMPTZ` -> `TEXT` ISO-8601. + +`MemoryStore` trait additions: + +```rust +async fn insert_domain_proposal(&self, p: &DomainProposal) -> Result<()>; +async fn list_domain_proposals(&self, status: Option<&str>) -> Result>; +async fn get_domain_proposal(&self, id: &str) -> Result>; +async fn set_proposal_status(&self, id: &str, status: &str) -> Result<()>; +``` + +### 13. WebSocket event for proposals + +**File**: `crates/vestige-mcp/src/dashboard/events.rs` + +Add variant: + +```rust +pub enum VestigeEvent { + // ... existing ... + DomainProposalCreated { + id: String, + kind: String, + confidence: f64, + timestamp: DateTime, + }, + MemoryClassified { + id: String, + domains: Vec, + top_score: f64, + timestamp: DateTime, + }, +} +``` + +The SvelteKit dashboard's WS client reacts to both events: classified events refresh any open domain-detail page; proposal events push a toast and a badge on the navbar. + +--- + +## Test Plan + +Test root: `tests/phase_4/` (a new member of the workspace; mirror the `tests/e2e` layout). + +`tests/phase_4/Cargo.toml`: + +```toml +[package] +name = "vestige-phase4-tests" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +vestige-core = { path = "../../crates/vestige-core", features = ["embeddings", "vector-search", "domain-classification"] } +vestige-mcp = { path = "../../crates/vestige-mcp" } +tokio = { workspace = true } +anyhow = "1" +tempfile = "3" +serde_json = { workspace = true } +uuid = { workspace = true } +``` + +### Unit tests (colocated in `domain_classifier.rs::tests`, `context_signals.rs::tests`, `spreading_activation.rs::tests`) + +Each public function must have at least one test: + +- `classify_empty_domains_returns_empty`: `classify(&[0.0; 768], &[])` returns `ClassificationResult { scores: {}, domains: [] }`. +- `classify_single_domain_scores`: one `Domain` with a known centroid; input embedding equal to centroid; expect score 1.0 and `domains == [id]`. +- `classify_multi_domain_overlap`: two domains A, B; input halfway between centroids; expect both scores >= `assign_threshold`; expect `domains == [A, B]` (order not guaranteed). +- `classify_below_threshold_returns_empty_domains_but_scores_filled`: input orthogonal to all centroids; expect `scores` populated, `domains` empty. +- `classify_with_boost_adds_delta`: same input as above, with `boost = {A: 0.4}`; expect A now above threshold, B unchanged. +- `classify_boost_clamps_to_unit`: `boost = {A: 5.0}`; resulting `scores[A]` must be <= 1.0. +- `tfidf_top_k_returns_distinct_terms`: given three fake docs, `top_k=3` returns three non-duplicate strings, in descending TF-IDF order. +- `tfidf_top_k_drops_stopwords`: `["the and for"]` + real content -> stop-words absent. +- `compute_top_terms_handles_empty_cluster`: returns `vec![]` (no panic). +- `signal_git_present_vs_absent`: `GitRepoSignal` given metadata with `.git` in cwd returns non-empty map; without it returns empty. +- `signal_ide_present_vs_absent`: `IdeHintSignal` ditto for `metadata.editor == "vscode"`. +- `signal_combined_clamped`: two signals both firing each at +0.10 -> combined map values <= +0.15. +- `cross_domain_decay_full_weight_on_overlap`: graph with node A in domain `dev`, node B in domain `dev`, edge A->B strength 1.0; after `activate`, B's activation equals the standard `initial * strength * decay_factor` (no extra penalty). +- `cross_domain_decay_half_weight_no_overlap`: A in `dev`, B in `infra`, same edge -> B's activation is 0.5x that of the overlap case. +- `cross_domain_decay_unclassified_no_penalty`: A classified, B unclassified -> full weight. +- `propose_changes_detects_split`: existing domain `dev`; new discovery returns two clusters whose centroids both sit close to old `dev` centroid, each >= min_cluster_size members -> proposal of kind `Split { parent: "dev", children: [a, b] }`. +- `propose_changes_detects_merge`: two existing domains whose new centroids now have cosine > `merge_threshold` -> proposal of kind `Merge`. +- `propose_changes_detects_new_cluster`: a new cluster with no match >= 0.85 to any existing -> `NewCluster`. +- `apply_proposal_split_updates_memberships`: after accept, memories previously in `dev` get reassigned (some to child a, some to child b) via `reassign_all`. + +### Integration tests (`tests/phase_4/tests/`) + +One file per behavior listed in the Phase 4 acceptance sheet. + +- `discover_seed_corpus.rs` -- loads the 500-memory fixture, runs `classifier.discover(&store).await`, asserts at least 3 clusters, asserts per-cluster intra-similarity mean > 0.6, asserts discovery wall time < 10s in release. Also asserts `top_terms` for each cluster contains at least one expected keyword per cluster (dev: contains any of `rust/trait/async`; infra: `bgp/vlan/network`; home: `solar/battery/pool`). +- `soft_assign_multi_domain.rs` -- inserts a memory "deploy zinit containers over BGP network"; after classify, `domains` contains both `dev` and `infra` (from a known centroid setup). +- `auto_classify_on_ingest.rs` -- with three existing domains, a fresh `smart_ingest` of a dev-ish sentence ends up with `domains == ["dev"]` and non-empty `domain_scores`. +- `reembed_triggers_recluster.rs` -- after `vestige migrate --reembed`, centroids must be recomputed; verify `list_domains()` returns fresh `centroid` values (different from pre-reembed). +- `dream_consolidation_recluster_hook.rs` -- run 5 dream cycles with heavy synthetic memory insertion; after the 5th, assert `list_domain_proposals("pending")` has at least one proposal. +- `proposal_accept_applies_changes.rs` -- accept a split proposal via `apply_proposal`; verify that memories in `dev` are now distributed across the new children and that the old `dev` domain is removed. +- `proposal_reject_leaves_state.rs` -- reject a proposal; verify all domains and memberships unchanged. +- `drift_is_proposal_only.rs` -- over 5 dream cycles with new inserts, never call accept; verify every memory's `domains` field equals its initial post-discovery value. No auto-apply. +- `cross_domain_activation_decay.rs` -- build a `ActivationNetwork` with two memories linked by a strength-1.0 edge, one in `dev`, one in `infra`; activate `dev` memory with 1.0; assert `infra` memory's activation == `0.5 * decay_factor` (0.35 with default decay_factor 0.7). Then set both to `dev` and reassert activation == `0.7`. +- `cli_domains_discover.rs` -- spawn `cargo run -- domains discover --force --json`, parse stdout, assert at least 3 clusters and valid JSON shape. +- `cli_domains_rename_merge.rs` -- happy-path rename then merge, with stdout assertions. +- `context_signal_git_repo.rs` -- ingest the same sentence from inside a tempdir with `.git` vs outside; assert the git-run produces slightly higher `domain_scores` for the code-related domain (diff >= 0.04, matches `git_boost = 0.05`). +- `threshold_tunable.rs` -- same memory, two runs with `assign_threshold = 0.40` vs `0.85`; the low-threshold run assigns more domains than the high-threshold run for the same content. +- `signal_boost_clamped.rs` -- artificially configure `git_boost = 5.0` and assert the resulting per-domain score is still <= 1.0. +- `discover_preserves_stable_ids.rs` -- run discover twice with no new memories; the second run's domain ids match the first's (via centroid-similarity stable-ID matching above 0.85). + +### Dashboard UI tests (`tests/phase_4/ui/`) + +Use curl-driven smoke tests (avoids adding Playwright as a new hard dep; Playwright already exists at `apps/dashboard/playwright.config.ts` and can be extended later). + +- `domains_list_renders.sh` -- `curl -H "X-API-Key: $KEY" http://localhost:3927/api/v1/domains` returns 200 + JSON array with expected keys. +- `domain_detail_histogram.sh` -- `curl .../api/v1/domains/dev/score-histogram` returns 10 buckets. +- `proposal_review_flow.sh` -- create a pending proposal via SQL insert; `curl POST .../api/v1/domains/proposals//accept`; `curl GET .../proposals?status=accepted` shows it. +- `unauth_domain_list_rejected.sh` -- no auth header -> 401. + +### Benchmarks (`tests/phase_4/benches/`) + +Criterion benches: + +- `bench_discover_10k.rs` -- synthetic 10k x 768D embeddings drawn from 5 blobs; assert `discover` wall p95 < 30s on a warm release build. +- `bench_auto_classify_single.rs` -- 20 domains in memory, classify one 768D vector; assert p99 < 5ms. +- `bench_reassign_all.rs` -- 10k memories, 5 domains; assert full `reassign_all` wall time < 90s (100 rows/ms baseline). + +--- + +## Acceptance Criteria + +- [ ] `cargo build -p vestige-core --features domain-classification` zero warnings. +- [ ] `cargo build -p vestige-mcp` zero warnings. +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` clean. +- [ ] `cargo test -p vestige-phase4-tests` -- all tests in `tests/phase_4/` pass. +- [ ] On a 500+ memory seed corpus covering three natural clusters (dev / infra / home), `vestige domains discover --force` produces sensible top-terms matching the expected keyword sets and labels are stable on a second run. +- [ ] `vestige search` with domain filter `["dev"]` excludes any memory whose `domains` array does not include `dev`. +- [ ] After 5 dream cycles with ongoing inserts, no existing memory's `domains` has silently changed; proposals exist in `domain_proposals` table; accepting a proposal reassigns as described. +- [ ] Cross-domain spreading activation: a query in `dev` that crosses a single edge into an `infra`-only memory still returns the memory but with activation `cross_domain_decay * in-domain_activation`. +- [ ] `vestige domains discover --min-cluster-size 20` produces strictly fewer or equal clusters than the default, and with larger per-cluster membership. +- [ ] Dashboard `/dashboard/domains` route renders all domains within 2 seconds on the seed corpus. +- [ ] Proposal UI flow (open pending, accept, confirmed in store) works end-to-end. +- [ ] Benchmarks meet targets (discover 10k p95 < 30s, auto-classify p99 < 5ms). + +--- + +## Rollback Notes + +- **Feature gate**: add `domain-classification` to `crates/vestige-core/Cargo.toml`'s `[features]`. When disabled, the `DomainClassifier` module is not compiled, the classification call in the ingest path is a no-op (`#[cfg]`-guarded), and cross-domain decay collapses to `1.0`. The CLI `domains` subcommand emits "domain classification is disabled in this build". +- **Revert strategy**: drop the two new tables `domains` (if created in Phase 1 is retained) or `domain_proposals` (Phase 4). A DOWN migration clears `memories.domains` and `memories.domain_scores`. Existing memories simply lose their domain assignments; all search and retrieval paths work unchanged because `domains = []` is the documented "unclassified" state. +- **Idempotency**: rerunning `discover` is always safe. Cluster numeric IDs may differ between runs, but the stable-ID match by centroid similarity preserves user-assigned labels. Do not persist cluster ids in client-side bookmarks; link via the user-assigned label. +- **Data-loss risk**: `apply_proposal` is a destructive operation (it deletes the old parent domain in a split or merges two). The dashboard's accept button double-confirms with a modal that shows the number of affected memories. + +--- + +## Open Implementation Questions + +Each question + candidates + RECOMMENDATION. + +### OQ1. Top-terms extraction: TF-IDF vs BM25 vs frequency? +- TF-IDF with smoothed IDF -- standard, cheap, good-enough. +- BM25 -- better for long-document discrimination, overkill for short memory contents. +- Raw frequency -- noisy; stop-words dominate. +**RECOMMENDATION**: TF-IDF with global IDF over the entire memory corpus (not just cluster members), recomputed once per `discover` call. Same tokenizer as the `dreams.rs::content_similarity` Jaccard for consistency. + +### OQ2. Proposal persistence: DB table vs in-memory with dashboard notification? +- DB table (`domain_proposals`) -- durable, surfaces across restarts, enables audit. +- In-memory only -- simpler, but loses proposals on server restart. +**RECOMMENDATION**: DB table. Proposals are rare (every 5th dream) and valuable user-facing artifacts; durability is mandatory. + +### OQ3. `hdbscan` crate: f32 vs f64 input, exact API surface? +- v0.10 historically takes `&[Vec]`; embeddings are `Vec`. +- Cost of converting f32 -> f64 at discovery time: `10k * 768 = 7.68M` f64 doubles ~ 60MB transient, acceptable. +**RECOMMENDATION**: verify v0.10's type signature at implementation time; if it requires f64, perform the conversion in `discover()` behind a single allocation. Document in module header. If the crate API diverged from the PRD snippet, fall back to the manual builder style (`HdbscanHyperParams::builder().min_cluster_size(n).min_samples(s).build()`). + +### OQ4. Stable domain IDs across discover re-runs? +- Option A: numeric IDs from HDBSCAN labels -- unstable, re-runs shuffle them. +- Option B: hash(top_terms) -- stable if top-terms stable, but top-terms drift. +- Option C (recommended): after computing new centroids, match each to the closest existing domain by centroid cosine; if similarity > 0.85, reuse the existing domain's `id` and `label`. Otherwise mint a fresh `id = "cluster_"`. +**RECOMMENDATION**: Option C. Preserves user-assigned labels across drift. Threshold 0.85 is config-tunable via `stable_id_threshold` if needed later. + +### OQ5. Context signal injection site: ingest handler vs embedder vs classifier? +- Embedder -- would alter embedding; signals are not about embedding quality. +- Ingest handler -- signals known there, but then `DomainClassifier` cannot be tested in isolation. +- Classifier as a `classify_with_boost(boost: Option<&HashMap>)` parameter -- pure, testable, composable. +**RECOMMENDATION**: classifier parameter. The cognitive engine constructs the boost map via `ContextSignals::gather_boost(&metadata, &domains)` and hands it to the classifier. Keeps the classifier stateless w.r.t. signals. + +### OQ6. Re-cluster proposal cadence: event-based (every Nth dream) vs time-based (weekly)? +- ADR resolution Q7: every Nth dream (N=5 default). +- Alternative: once per week regardless of dream cadence. +**RECOMMENDATION**: stick with every Nth dream. Users who dream rarely re-cluster rarely -- that matches the philosophy ("memory work triggers memory bookkeeping"). Note the alternative as future consideration; if users complain about never seeing proposals, add a time-based fallback. + +### OQ7. Minimum corpus size for first discover? +- PRD default: 150. +- Too low -> noisy initial clusters, proposals every dream. +- Too high -> user waits forever for domains to appear. +**RECOMMENDATION**: 150 as the default discovery gate; HDBSCAN's `min_cluster_size=10` will produce 0 clusters for < 100 memories, so the system gracefully produces no domains until the corpus is large enough. Test with `N=80, 150, 500` in `threshold_tunable.rs` to confirm sensible behavior. + +### OQ8. Cross-domain decay: strict no-overlap vs graded? +- Strict: `1.0` if any overlap, `cross_domain_decay` otherwise. +- Graded: `max(cross_domain_decay, |A intersect B| / max(|A|, |B|))`. +**RECOMMENDATION**: strict for Phase 4. Easier to reason about, easier to tune, easier to test. Graded is a marked future enhancement; file an issue if retrieval-quality metrics justify it. + +### OQ9. Classifier invocation from remote HTTP clients? +- In server mode, an agent posts `smart_ingest` -> server embeds -> server classifies. +- All the work stays server-side; MCP clients never do classification. +**RECOMMENDATION**: confirmed server-side-only. Document in the MCP tool schema that `smart_ingest` now returns `domains` and `domain_scores` in its response so clients can display the classification to the user. + +### OQ10. Where to store the dream-cycle counter? +- In-memory on `CognitiveEngine` -- lost on restart, miscounts cadence. +- New `system_state` singleton table. +**RECOMMENDATION**: `system_state` table. Survives restarts. Also useful for future metrics (total memories ever, total dreams ever). + +### OQ11. Scope of `reassign_all` after a proposal accept vs a normal discover? +- On discover --force (first-time), run `reassign_all` against all memories. +- On proposal accept (split / merge), run `reassign_all` only on affected memories (parent's members for split; both parents' members for merge) to avoid touching unrelated records. +**RECOMMENDATION**: scoped reassignment where possible; fall back to full `reassign_all` only on `discover --force` or when the set of domains has fundamentally changed. Reduces write amplification on large corpora. + +### OQ12. Proposal freshness? +- Multiple re-clusters could stack up pending proposals. +**RECOMMENDATION**: before inserting a new proposal, check for existing pending proposals with the same `kind + targets`; if present, bump `created_at` and `confidence` instead of creating a duplicate. Add a `confidence_history` array in the `payload` JSONB for audit. + +--- + +## Implementation Sequencing (suggested order) + +1. Land the `DomainClassifier` struct, `classify` / `classify_with_boost`, unit tests. (Day 1) +2. Add `compute_top_terms` + TF-IDF helper, tests. (Day 1) +3. Wire `discover` end-to-end against SQLite; `discover_seed_corpus` integration test. (Day 2) +4. Add `domain_proposals` table migrations + trait methods; both backends. (Day 2) +5. Implement `propose_changes` + `apply_proposal`; proposal unit tests. (Day 3) +6. Context signals module + tests. (Day 3) +7. Hook classifier into ingest path; `auto_classify_on_ingest` integration test. (Day 4) +8. Cross-domain decay in spreading activation; unit + integration tests. (Day 4) +9. Dream re-cluster hook + `system_state` counter; integration tests for drift-only behavior. (Day 5) +10. CLI subcommands. (Day 6) +11. REST endpoints. (Day 6) +12. SvelteKit dashboard routes + WebSocket event wiring. (Day 7-8) +13. Benchmarks + acceptance sweep on the 500-memory seed. (Day 9) + +--- + +## File Map (everything Phase 4 touches or creates) + +Creates: + +- `crates/vestige-core/src/neuroscience/domain_classifier.rs` +- `crates/vestige-core/src/neuroscience/context_signals.rs` +- `crates/vestige-core/migrations/postgres/00XX_domain_proposals.sql` +- `crates/vestige-core/migrations/sqlite/00XX_domain_proposals.sql` (or inline in `storage/migrations.rs`) +- `crates/vestige-mcp/src/api/domains.rs` (REST handlers) +- `apps/dashboard/src/routes/(app)/domains/+page.svelte` +- `apps/dashboard/src/routes/(app)/domains/[id]/+page.svelte` +- `apps/dashboard/src/routes/(app)/domains/proposals/+page.svelte` +- `apps/dashboard/src/lib/api/domains.ts` +- `tests/phase_4/Cargo.toml` +- `tests/phase_4/tests/*.rs` (per the Integration test list) +- `tests/phase_4/fixtures/seed_500.json` +- `tests/phase_4/support/fixtures.rs` + +Modifies: + +- `crates/vestige-core/Cargo.toml` -- add `hdbscan = "0.10"` under a new `domain-classification` feature. +- `crates/vestige-core/src/neuroscience/mod.rs` -- register new modules, re-exports. +- `crates/vestige-core/src/neuroscience/spreading_activation.rs` -- `cross_domain_decay` field in `ActivationConfig`, `domains` field on `ActivationNode`, decay math in `activate`. +- `crates/vestige-core/src/consolidation/phases.rs` -- `DreamReClusterHook`. +- `crates/vestige-core/src/advanced/dreams.rs` -- accept a hook callback from the orchestrator (if the orchestration is done at this level). +- `crates/vestige-core/src/storage/trait.rs` -- add proposal + system_state methods. +- `crates/vestige-core/src/storage/sqlite.rs` -- implement proposal + system_state methods + `all_embeddings_with_meta` if not already on the trait. +- `crates/vestige-core/src/storage/postgres.rs` (Phase 2) -- same. +- `crates/vestige-core/src/lib.rs` -- re-exports. +- `crates/vestige-core/src/cognitive.rs` (or equivalent ingest orchestrator) -- auto-classify injection. +- `crates/vestige-mcp/src/bin/cli.rs` -- `Domains` subcommand + dispatch. +- `crates/vestige-mcp/src/dashboard/mod.rs` -- wire new REST routes. +- `crates/vestige-mcp/src/dashboard/events.rs` -- new event variants. +- `crates/vestige-mcp/src/dashboard/handlers.rs` -- if legacy dashboard gets a domains panel (optional). +- `vestige.toml` config loader -- `[domains]` section + struct + defaults. +- Root `Cargo.toml` workspace members -- add `tests/phase_4`. + +--- + +## Risks + +- **HDBSCAN determinism**: HDBSCAN is deterministic given input order; sorting embeddings by memory id before feeding the clusterer guarantees reproducibility across runs -- do this in `discover()` and document it. +- **Embedding dimension drift**: Phase 1's `embedding_model` registry blocks writes from mismatched embedders. If `discover()` ever sees two dimensions, it bails with a clear error and points at `vestige migrate --reembed`. +- **Classification latency on ingest**: for users with thousands of domains (unlikely but possible), `classify` is O(n_domains * dim). 20 domains * 768 f32 = 15k flops per classification, trivial. Still, expose a `classify_budget_ms` config knob for paranoia. +- **Re-cluster proposal storms**: if the corpus is borderline-stable, small changes can produce conflicting proposals on consecutive dreams. Mitigation: OQ12 (dedup by target set, bump confidence instead of stacking). +- **Dashboard feature gap**: if the SvelteKit app lands with the domains route but the REST endpoints are not yet deployed, the route 404s. Mitigation: ship the REST endpoints in the same release; a feature flag on the client toggles the nav entry. + +--- + +## Non-Goals Reminder + +- No Phase 5 federation concerns in this plan. +- No cross-installation domain sync. +- No automatic accept of proposals, ever. +- No graded cross-domain decay; strict only. +- No ML-based domain label suggestion (top-terms are enough for v1). +- No editing individual memory memberships from the UI in this phase. diff --git a/docs/prd/001-getting-centralized-vestige.md b/docs/prd/001-getting-centralized-vestige.md new file mode 100644 index 0000000..9d86087 --- /dev/null +++ b/docs/prd/001-getting-centralized-vestige.md @@ -0,0 +1,751 @@ +# RFC: Pluggable Storage Backend + Network Access for Vestige + +**Status**: Draft / Discussion +**Author**: Jan +**Date**: 2026-02-26 +**Vestige version**: v2.x (current main) + +## Summary + +Add a pluggable storage backend trait to Vestige, enabling PostgreSQL (+pgvector) as an alternative to the current SQLite+FTS5+USearch stack. Simultaneously add HTTP MCP transport with API key authentication to enable centralized/remote deployment. + +This keeps the existing local-first SQLite mode fully intact while opening up a server deployment model. + +## Motivation + +Vestige currently runs as a local process per machine (MCP via stdio, SQLite in `~/.vestige/`). This works great for single-machine use but doesn't support: + +- **Multi-machine access**: Same memory brain from laptop, desktop, and server +- **Multi-agent access**: Multiple AI clients hitting one memory store concurrently +- **Future federation**: Syncing memory between decentralized nodes (e.g., MOS/Threefold grid) + +SQLite's single-writer model and lack of native network protocol make it unsuitable as a centralized server. PostgreSQL is a natural fit: built-in concurrency (MVCC), authentication, replication, and with `pgvector` + built-in FTS it collapses three separate storage layers into one. + +## Design + +### Storage Trait + +The core abstraction. All 29 cognitive modules interact with storage exclusively through this trait (or a small family of traits). + +```rust +use std::collections::HashMap; +use uuid::Uuid; + +/// Core memory record, backend-agnostic +#[derive(Debug, Clone)] +pub struct MemoryRecord { + pub id: Uuid, + pub domains: Vec, // [] = unclassified, ["dev"], ["dev", "infra"], etc. + pub domain_scores: HashMap, // raw similarities: {"dev": 0.82, "infra": 0.71} + pub content: String, + pub node_type: String, + pub tags: Vec, + pub embedding: Option>, // dimensionality is runtime config + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub metadata: serde_json::Value, +} + +/// FSRS scheduling state, stored alongside each memory +#[derive(Debug, Clone)] +pub struct SchedulingState { + pub memory_id: Uuid, + pub stability: f64, + pub difficulty: f64, + pub retrievability: f64, + pub last_review: Option>, + pub next_review: Option>, + pub reps: u32, + pub lapses: u32, +} + +/// Hybrid search request +#[derive(Debug, Clone)] +pub struct SearchQuery { + pub domains: Option>, // None = search all domains + pub text: Option, // FTS query + pub embedding: Option>, // vector similarity + pub tags: Option>, // tag filter + pub node_types: Option>, + pub limit: usize, + pub min_retrievability: Option, // filter by FSRS state +} + +#[derive(Debug, Clone)] +pub struct SearchResult { + pub record: MemoryRecord, + pub score: f64, // combined/fused score + pub fts_score: Option, + pub vector_score: Option, +} + +/// Connection/edge between memories (for spreading activation) +#[derive(Debug, Clone)] +pub struct MemoryEdge { + pub source_id: Uuid, + pub target_id: Uuid, + pub edge_type: String, + pub weight: f64, + pub created_at: chrono::DateTime, +} + +/// Main storage trait — one impl per backend +/// trait_variant generates a Send-bound `MemoryStore` alias, +/// enabling Arc without manual boxing. +#[trait_variant::make(MemoryStore: Send)] +pub trait LocalMemoryStore: Sync + 'static { + // --- Lifecycle --- + async fn init(&self) -> Result<()>; + async fn health_check(&self) -> Result; + + // --- CRUD --- + async fn insert(&self, record: &MemoryRecord) -> Result; + async fn get(&self, id: Uuid) -> Result>; + async fn update(&self, record: &MemoryRecord) -> Result<()>; + async fn delete(&self, id: Uuid) -> Result<()>; + + // --- Search --- + async fn search(&self, query: &SearchQuery) -> Result>; + async fn fts_search(&self, text: &str, limit: usize) -> Result>; + async fn vector_search(&self, embedding: &[f32], limit: usize) -> Result>; + + // --- FSRS Scheduling --- + async fn get_scheduling(&self, memory_id: Uuid) -> Result>; + async fn update_scheduling(&self, state: &SchedulingState) -> Result<()>; + async fn get_due_memories(&self, before: chrono::DateTime, limit: usize) -> Result>; + + // --- Graph (spreading activation) --- + async fn add_edge(&self, edge: &MemoryEdge) -> Result<()>; + async fn get_edges(&self, node_id: Uuid, edge_type: Option<&str>) -> Result>; + async fn remove_edge(&self, source: Uuid, target: Uuid) -> Result<()>; + async fn get_neighbors(&self, node_id: Uuid, depth: usize) -> Result>; + + // --- Bulk / Maintenance --- + async fn count(&self) -> Result; + async fn get_stats(&self) -> Result; + async fn vacuum(&self) -> Result<()>; +} +``` + +**Design notes:** + +- `trait_variant::make` generates a `MemoryStore` trait alias with `Send`-bound futures, allowing `Arc` for runtime backend selection. `LocalMemoryStore` is the base (usable in single-threaded contexts), `MemoryStore` is the Send variant for Axum/tokio. +- `embedding: Option>` — dimensions determined at runtime by the configured fastembed model. The backend stores whatever it gets. +- The trait is intentionally flat. The cognitive modules (FSRS-6, spreading activation, synaptic tagging, prediction error gating, etc.) sit *above* this trait and don't need to know about the backend. +- `search()` does hybrid RRF fusion at the backend level — both SQLite and Postgres implementations handle this internally. + +### Backend: SQLite (existing, refactored) + +Wraps the current implementation behind the trait: + +``` +SqliteMemoryStore +├── rusqlite connection pool (r2d2 or deadpool) +├── FTS5 virtual table (keyword search) +├── USearch HNSW index (vector search, behind RwLock) +└── WAL mode + busy timeout for concurrent readers +``` + +No behavioral changes — just the trait boundary. + +### Backend: PostgreSQL (new) + +``` +PgMemoryStore +├── sqlx::PgPool (connection pool, compile-time checked queries) +├── tsvector + GIN index (keyword search) +├── pgvector + HNSW index (vector search) +└── Standard PostgreSQL MVCC concurrency +``` + +**Schema sketch:** + +```sql +CREATE EXTENSION IF NOT EXISTS vector; + +-- Domain registry — populated by clustering, not by user +CREATE TABLE domains ( + id TEXT PRIMARY KEY, -- auto-generated or user-named + label TEXT NOT NULL, -- human label (suggested or user-provided) + centroid vector, -- mean embedding of domain members + top_terms TEXT[] NOT NULL DEFAULT '{}', -- top keywords for display + memory_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}' +); + +CREATE TABLE memories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + domains TEXT[] NOT NULL DEFAULT '{}', -- [] = unclassified + domain_scores JSONB NOT NULL DEFAULT '{}', -- {"dev": 0.82, "infra": 0.71} raw similarities + content TEXT NOT NULL, + node_type TEXT NOT NULL DEFAULT 'general', + tags TEXT[] NOT NULL DEFAULT '{}', + embedding vector, -- dimension set at table creation or unconstrained + metadata JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + -- FTS: auto-maintained tsvector column + search_vec TSVECTOR GENERATED ALWAYS AS ( + setweight(to_tsvector('english', content), 'A') || + setweight(to_tsvector('english', coalesce(node_type, '')), 'B') || + setweight(array_to_tsvector(tags), 'C') + ) STORED +); + +-- FTS index +CREATE INDEX idx_memories_fts ON memories USING GIN (search_vec); + +-- Vector similarity (HNSW) +CREATE INDEX idx_memories_embedding ON memories + USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64); + +-- Common filters +CREATE INDEX idx_memories_domains ON memories USING GIN (domains); +CREATE INDEX idx_memories_node_type ON memories (node_type); +CREATE INDEX idx_memories_tags ON memories USING GIN (tags); +CREATE INDEX idx_memories_created ON memories (created_at); + +-- FSRS scheduling state +CREATE TABLE scheduling ( + memory_id UUID PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE, + stability DOUBLE PRECISION NOT NULL DEFAULT 0.0, + difficulty DOUBLE PRECISION NOT NULL DEFAULT 0.0, + retrievability DOUBLE PRECISION NOT NULL DEFAULT 1.0, + last_review TIMESTAMPTZ, + next_review TIMESTAMPTZ, + reps INTEGER NOT NULL DEFAULT 0, + lapses INTEGER NOT NULL DEFAULT 0 +); + +CREATE INDEX idx_scheduling_next ON scheduling (next_review); + +-- Graph edges (spreading activation) +-- Edges can cross domain boundaries — spreading activation respects +-- domain filters when provided, traverses freely when searching all domains. +CREATE TABLE edges ( + source_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + target_id UUID NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + edge_type TEXT NOT NULL DEFAULT 'related', + weight DOUBLE PRECISION NOT NULL DEFAULT 1.0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source_id, target_id, edge_type) +); + +CREATE INDEX idx_edges_target ON edges (target_id); + +-- API keys +CREATE TABLE api_keys ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + key_hash TEXT NOT NULL UNIQUE, -- blake3 + label TEXT NOT NULL, + scopes TEXT[] NOT NULL DEFAULT '{read,write}', + domain_filter TEXT[] NOT NULL DEFAULT '{}', -- {} = access all domains + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_used TIMESTAMPTZ, + active BOOLEAN NOT NULL DEFAULT true +); +``` + +**Hybrid search in SQL:** + +```sql +-- RRF (Reciprocal Rank Fusion) combining FTS + vector +-- $1 = query text, $2 = embedding, $3 = limit, $4 = domain filter (NULL for all) +WITH fts AS ( + SELECT id, ts_rank_cd(search_vec, websearch_to_tsquery('english', $1)) AS score, + ROW_NUMBER() OVER (ORDER BY ts_rank_cd(search_vec, websearch_to_tsquery('english', $1)) DESC) AS rank + FROM memories + WHERE search_vec @@ websearch_to_tsquery('english', $1) + AND ($4::text[] IS NULL OR domains && $4) -- array overlap: any match + LIMIT 50 +), +vec AS ( + SELECT id, 1 - (embedding <=> $2::vector) AS score, + ROW_NUMBER() OVER (ORDER BY embedding <=> $2::vector) AS rank + FROM memories + WHERE embedding IS NOT NULL + AND ($4::text[] IS NULL OR domains && $4) + LIMIT 50 +) +SELECT COALESCE(f.id, v.id) AS id, + COALESCE(1.0 / (60 + f.rank), 0) + COALESCE(1.0 / (60 + v.rank), 0) AS rrf_score, + f.score AS fts_score, + v.score AS vector_score +FROM fts f FULL OUTER JOIN vec v ON f.id = v.id +ORDER BY rrf_score DESC +LIMIT $3; +``` + +### Embedding Configuration + +The embedding layer stays external to the storage backend. fastembed runs locally and produces vectors that get passed into `MemoryRecord.embedding`. + +```toml +# vestige.toml +[embeddings] +provider = "fastembed" # only local for now +model = "BAAI/bge-base-en-v1.5" # 768 dimensions +# model = "BAAI/bge-large-en-v1.5" # 1024 dimensions +# model = "BAAI/bge-small-en-v1.5" # 384 dimensions + +[storage] +backend = "postgres" # or "sqlite" + +[storage.sqlite] +path = "~/.vestige/vestige.db" + +[storage.postgres] +url = "postgresql://vestige:secret@localhost:5432/vestige" +max_connections = 10 +``` + +On init, the backend reads the embedding dimension from the first stored vector (or from config) and validates consistency. + +For pgvector: you can either create the column as `vector(768)` (fixed, faster) or unconstrained `vector` (flexible, slightly slower). Recommendation: fixed dimension derived from config, with a migration path if the model changes. + +### Emergent Domain Model + +Instead of user-defined tenants, domains emerge automatically from the data via clustering. The user never has to decide where a memory belongs — the system figures it out. + +#### Pipeline + +``` +Phase 1: Accumulate (cold start, 0 → N memories) +│ All memories stored with domains = [] (unclassified) +│ No classification overhead, just embed and store +│ Threshold N is configurable, default ~150 memories +│ +Phase 2: Discover (triggered once at threshold, or manually) +│ Run HDBSCAN on all embeddings: +│ - min_cluster_size: ~10 +│ - min_samples: ~5 +│ - No eps parameter needed (unlike DBSCAN) +│ - Automatically determines number of clusters +│ - Handles variable-density clusters +│ - Border points between clusters flagged naturally +│ +│ For each cluster, extract: +│ - Centroid (mean embedding) +│ - Top terms (TF-IDF or frequency over cluster members) +│ - Suggested label from top terms +│ +│ Present to user (via dashboard or CLI): +│ "I found 3 natural groupings in your memories: +│ ● cluster_0 (47 memories): BGP, SONiC, VLAN, FRR, peering... +│ ● cluster_1 (31 memories): solar, kWh, battery, pool, ESPHome... +│ ● cluster_2 (22 memories): Rust, trait, async, zinit, tokio..." +│ +│ User can: +│ - Name them: cluster_0 → "infra", cluster_1 → "home", cluster_2 → "dev" +│ - Accept suggested names +│ - Merge clusters +│ - Do nothing (auto-names stick) +│ +Phase 3: Soft-assign all existing memories +│ Now that centroids exist, re-score every memory (including +│ those from discovery) against all centroids. +│ This replaces HDBSCAN's hard labels with continuous scores: +│ +│ For each memory: +│ similarities = [(domain, cosine_sim(embedding, centroid)) for each domain] +│ domains = [id for (id, score) in similarities if score >= threshold] +│ +│ Memories in overlap zones get multiple domains. +│ Memories far from all centroids stay unclassified. +│ +Phase 4: Classify (ongoing, after discovery) +│ New memory ingested: +│ 1. Compute embedding +│ 2. Compute similarity to ALL domain centroids +│ 3. Store raw scores in domain_scores JSONB +│ 4. Threshold into domains[] array +│ 5. Update domain centroids incrementally (running mean) +│ +│ Context signals as soft priors: +│ - Git repo / IDE metadata → boost similarity to code-related domains +│ - No workspace context → slight boost toward non-technical domains +│ - These shift the score, never override the embedding distance +│ +Phase 5: Re-cluster (periodic, during dream consolidation) + Re-run HDBSCAN on all embeddings including new ones + Detect: + - New clusters forming from previously unclassified memories + - Existing clusters splitting (domain grew too broad) + - Clusters merging (domains that were artificially separate) + Propose changes to user: + "Your 'dev' domain may have split into two groups: + - systems (zinit, MOS, containers, VMs) — 34 memories + - networking (BGP, SONiC, VLANs, MLAG) — 28 memories + Split them? [yes / no / later]" + Re-run soft assignment on all memories after structural changes + Centroid vectors are updated regardless +``` + +#### Domain Storage + +```rust +#[derive(Debug, Clone)] +pub struct Domain { + pub id: String, + pub label: String, + pub centroid: Vec, + pub top_terms: Vec, + pub memory_count: usize, + pub created_at: chrono::DateTime, +} +``` + +Added to the `MemoryStore` trait: + +```rust + // --- Domains --- + async fn list_domains(&self) -> Result>; + async fn get_domain(&self, id: &str) -> Result>; + async fn upsert_domain(&self, domain: &Domain) -> Result<()>; + async fn delete_domain(&self, id: &str) -> Result<()>; + async fn classify(&self, embedding: &[f32]) -> Result>; + // Returns [(domain_id, similarity)] sorted by similarity desc. + // Caller decides threshold for assignment. +``` + +#### Classification Module + +A new cognitive module alongside FSRS, spreading activation, etc.: + +```rust +pub struct DomainClassifier { + /// Similarity threshold — domains scoring above this are assigned + pub assign_threshold: f64, // default: 0.65 + /// Minimum memories before running initial discovery + pub discovery_threshold: usize, // default: 150 + /// How often to re-cluster (in dream consolidation passes) + pub recluster_interval: usize, // default: every 5th consolidation + /// HDBSCAN min_cluster_size + pub min_cluster_size: usize, // default: 10 +} + +/// Raw classification result — all scores, before thresholding +#[derive(Debug, Clone)] +pub struct ClassificationResult { + /// Similarity to every known domain centroid + pub scores: HashMap, // {"dev": 0.82, "infra": 0.71, "home": 0.34} + /// Domains above assign_threshold + pub domains: Vec, // ["dev", "infra"] +} + +impl DomainClassifier { + /// Score a memory against all domain centroids. + /// Returns raw scores AND thresholded domain list. + pub fn classify( + &self, + embedding: &[f32], + domains: &[Domain], + ) -> ClassificationResult { + if domains.is_empty() { + return ClassificationResult { + scores: HashMap::new(), + domains: vec![], // still in accumulation phase + }; + } + + let scores: HashMap = domains.iter() + .map(|d| (d.id.clone(), cosine_similarity(embedding, &d.centroid))) + .collect(); + + let assigned: Vec = scores.iter() + .filter(|(_, &s)| s >= self.assign_threshold) + .map(|(id, _)| id.clone()) + .collect(); + + ClassificationResult { scores, domains: assigned } + } + + /// Soft-assign all existing memories after discovery or re-clustering. + /// Returns number of memories whose domains changed. + pub async fn reassign_all( + &self, + store: &dyn MemoryStore, + domains: &[Domain], + ) -> Result { + // Load all memories, re-score, update domains + domain_scores + // Batched to avoid loading everything into memory at once + todo!() + } +} +``` + +**Key distinction from the previous design:** there's no "closest wins" or "margin" logic. Every domain gets a score, and *all* domains above threshold are assigned. A memory about "deploying zinit containers via BGP-routed network" might score 0.78 on "dev" and 0.72 on "infra" — it gets both. A memory about "solar panel output today" scores 0.85 on "home" and 0.31 on everything else — it only gets "home". + +The raw `domain_scores` are always stored, so you (or the dashboard) can see *why* a memory was classified the way it was, and the threshold can be adjusted retroactively without re-computing embeddings. + +#### Search Behavior + +- **Default (no domain filter)**: searches all memories across all domains +- **Domain-scoped**: `domains: Some(vec!["dev"])` — only memories tagged with `dev` +- **Multi-domain**: `domains: Some(vec!["dev", "infra"])` — memories in either +- **MCP clients can set `X-Vestige-Domain` header** for default scoping, but the system works fine without it + +#### HDBSCAN Implementation + +HDBSCAN (Hierarchical DBSCAN) over the embedding vectors. Advantages over plain DBSCAN: + +- **No `eps` parameter** — the hardest thing to tune in DBSCAN. HDBSCAN determines density thresholds from the data hierarchy. +- **Variable-density clusters** — a tight cluster of networking memories and a spread-out cluster of personal memories are both detected correctly. +- **Border points** — memories between clusters are identified as low-confidence members, which aligns perfectly with soft assignment. + +Implementation: the `hdbscan` crate in Rust. Load all embeddings into memory (at 768d × f32 × 10k memories ≈ 30MB — fine), cluster, compute centroids, soft-assign all memories against the centroids. + +```rust +use hdbscan::{Center, Hdbscan}; + +fn discover_domains( + embeddings: &[Vec], + min_cluster_size: usize, +) -> (Vec>, Vec>) { // (cluster → member indices, centroids) + let clusterer = Hdbscan::default(embeddings); + let labels = clusterer.cluster().unwrap(); + let centroids = clusterer.calc_centers(Center::Centroid, &labels).unwrap(); + + // Group indices by label, ignoring noise (-1) + let mut clusters: HashMap> = HashMap::new(); + for (i, &label) in labels.iter().enumerate() { + if label >= 0 { + clusters.entry(label).or_default().push(i); + } + } + (clusters.into_values().collect(), centroids) +} +``` + +After HDBSCAN produces hard clusters, the soft-assignment pass (Phase 3) immediately re-scores all memories — including the ones HDBSCAN assigned — against the computed centroids. So HDBSCAN's hard labels are only used to *define* the centroids. The actual domain assignments always come from the continuous similarity scores. + +This works identically for both SQLite and Postgres backends — clustering runs in Rust application code, results are written back to the storage layer. + +### Network Transport + +#### MCP over Streamable HTTP + +Extend the existing Axum server: + +```rust +// Alongside existing dashboard routes +let app = Router::new() + // Existing dashboard + .route("/api/health", get(health_handler)) + .route("/dashboard/*path", get(dashboard_handler)) + // New: MCP over HTTP + .route("/mcp", post(mcp_handler).get(mcp_sse_handler)) + // New: REST API + // X-Vestige-Domain header optionally scopes to a domain + .route("/api/v1/memories", post(create_memory).get(list_memories)) + .route("/api/v1/memories/:id", get(get_memory).put(update_memory).delete(delete_memory)) + .route("/api/v1/search", post(search_memories)) + .route("/api/v1/consolidate", post(trigger_consolidation)) + .route("/api/v1/stats", get(get_stats)) + .route("/api/v1/domains", get(list_domains)) + .route("/api/v1/domains/discover", post(trigger_discovery)) + .route("/api/v1/domains/:id", put(rename_domain).delete(merge_domain)) + // Auth on everything except health + .layer(middleware::from_fn(api_key_auth)); +``` + +#### Auth Middleware + +```rust +async fn api_key_auth( + State(store): State>, + request: axum::extract::Request, + next: middleware::Next, +) -> Result { + // Skip auth for health endpoint + if request.uri().path() == "/api/health" { + return Ok(next.run(request).await); + } + + let key = request.headers() + .get("Authorization") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .or_else(|| request.headers() + .get("X-API-Key") + .and_then(|v| v.to_str().ok())); + + match key { + Some(k) if verify_api_key(store.as_ref(), k).await => { + Ok(next.run(request).await) + } + _ => Err(StatusCode::UNAUTHORIZED), + } +} +``` + +#### Client Configuration + +```json +// Claude Desktop / Claude Code — single key, all domains +{ + "mcpServers": { + "vestige": { + "url": "http://vestige.local:3927/mcp", + "headers": { + "Authorization": "Bearer vst_a1b2c3..." + } + } + } +} +``` + +No domain header needed — searches all domains by default. The MCP tools include an optional `domain` parameter for scoped queries if the LLM or user wants to narrow down. + +Alternatively, scope a connection to a specific domain: + +```json +// Domain-scoped connection (e.g., for a home automation agent) +{ + "mcpServers": { + "vestige-home": { + "url": "http://vestige.local:3927/mcp", + "headers": { + "Authorization": "Bearer vst_e5f6g7...", + "X-Vestige-Domain": "home" + } + } + } +} +``` + +### Server Configuration + +```toml +# vestige.toml — full example for server mode +[server] +bind = "0.0.0.0:3927" # or mycelium IPv6 address +# tls_cert = "/path/to/cert.pem" # optional +# tls_key = "/path/to/key.pem" + +[auth] +enabled = true +# If false, no key required (local-only mode) + +[storage] +backend = "postgres" + +[storage.postgres] +url = "postgresql://vestige:secret@localhost:5432/vestige" +max_connections = 10 + +[embeddings] +provider = "fastembed" +model = "BAAI/bge-base-en-v1.5" +``` + +### CLI Extensions + +```bash +# Domain management (mostly automatic, but user can inspect/rename) +vestige domains list +# → dev Development (auto) memories: 87 top: Rust, trait, async, tokio +# → infra Infrastructure (auto) memories: 47 top: BGP, SONiC, VLAN, FRR +# → home Home (auto) memories: 31 top: solar, kWh, pool, ESPHome +# → (unclassified) memories: 12 + +vestige domains rename cluster_0 infra --label "Infrastructure" +vestige domains merge home personal --into home +vestige domains discover --force # re-run HDBSCAN now + +# Key management +vestige keys create --label "macbook" +# → Created key: vst_a1b2c3d4... (store this, shown once) + +vestige keys create --label "home-assistant" --scopes read --domains home +# → Created key: vst_e5f6g7h8... (read-only, home domain only) + +vestige keys list +# → macbook vst_a1b2... scopes: [read,write] domains: [all] +# → home-assistant vst_e5f6... scopes: [read] domains: [home] + +vestige keys revoke vst_a1b2c3d4... + +# Migration +vestige migrate --from sqlite --to postgres \ + --sqlite-path ~/.vestige/vestige.db \ + --postgres-url postgresql://localhost/vestige +``` + +## Implementation Plan + +### Phase 1: Storage Trait Extraction +- Define the `MemoryStore` trait (including domain methods) +- Refactor current SQLite code to implement it +- Add `domains TEXT[]` column to existing SQLite schema +- Verify all 29 modules work through the trait (no direct SQLite access) +- **No behavioral changes** — all memories start as unclassified + +### Phase 2: PostgreSQL Backend +- Implement `PgMemoryStore` +- Schema migrations (sqlx or refinery) +- `vestige migrate` command for SQLite → Postgres +- Config file support for backend selection + +### Phase 3: Network Access +- MCP Streamable HTTP endpoint on existing Axum server +- API key auth middleware + CLI management +- REST API endpoints +- Feature flags for stdio vs HTTP mode + +### Phase 4: Emergent Domain Classification +- `DomainClassifier` cognitive module +- HDBSCAN clustering via `hdbscan` crate (runs on both backends) +- Soft assignment pass: score all memories against centroids, threshold into domains +- `domain_scores` JSONB stored per memory for transparency / retroactive re-thresholding +- Domain discovery CLI and dashboard UI +- Auto-classification on ingest (once domains exist) +- Re-clustering during dream consolidation passes +- Domain management CLI (rename, merge, inspect) + +### Phase 5: Federation (future) +- Node discovery via Mycelium / mDNS +- Memory sync protocol (UUID-based, last-write-wins) +- Possibly Iroh for content-addressed replication +- FSRS state merge (review history append, not overwrite) + +## Crate Dependencies (new) + +```toml +# Phase 1 — trait abstraction +trait-variant = "0.1" + +# Phase 2 — Postgres +sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "uuid", "chrono", "json"] } +pgvector = "0.4" # sqlx integration for vector type + +# Phase 3 — Auth +blake3 = "1" # key hashing +rand = "0.8" # key generation + +# Phase 4 — Domain clustering +hdbscan = "0.10" # HDBSCAN — no eps tuning, variable density, built-in centroid calc +``` + +## Open Questions + +1. **Trait granularity**: One big `MemoryStore` trait or split into `MemoryStore + SchedulingStore + GraphStore + DomainStore`? Splitting is cleaner but means more `dyn` parameters threading through handlers. + +2. **Embedding on insert**: Should the storage backend call fastembed, or should the caller always provide the embedding? Current design says caller provides it, keeping the backend pure storage. But this means every client needs fastembed locally even if the DB is remote. For the server model, having the server compute embeddings makes more sense. + +3. **pgvector dimension**: Fixed (e.g., `vector(768)`) or unconstrained (`vector`)? Fixed is faster for HNSW but requires migration if model changes. + +4. **Sync conflict resolution for federation**: LWW per-UUID is simple but lossy. CRDTs would be more correct but massively more complex. For FSRS state specifically, merging review event logs would be ideal. + +5. **Dashboard auth**: The 3D dashboard currently runs unauthenticated on localhost. With remote access, it needs the same auth. Should it use the same API keys or have a separate session/cookie mechanism? + +6. **HDBSCAN `min_cluster_size`**: The main tuning knob. Too small → noisy micro-clusters. Too large → distinct topics get merged. Default of 10 should work for most cases, but may need a manual override or auto-sweep (run with several values, pick the one with best silhouette score). + +7. **Domain drift**: Over time, the character of a domain changes. How aggressively should re-clustering reshape existing domains? Conservative (only propose splits/merges, never auto-apply) vs. aggressive (auto-reassign memories whose scores drifted below threshold)? + +8. **Spreading activation across domains**: When searching within a single domain, should graph edges that cross into other domains be followed? Probably yes for recall quality, but with decaying weight as you cross boundaries. + +9. **Threshold tuning**: The `assign_threshold` (0.65 default) determines how many memories are multi-domain vs single-domain vs unclassified. Too low → everything is multi-domain (useless). Too high → too many unclassified. Could be auto-tuned per dataset by targeting a specific unclassified ratio (e.g., "keep fewer than 10% unclassified"). From 9c633c172b346980eacd5173e5aa66eea767836a Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 22 Apr 2026 10:28:59 +0200 Subject: [PATCH 02/38] Added postgres admin added amends to the postgres backend/phase2 --- .gitignore | 3 + docs/plans/0002-phase-2-postgres-backend.md | 3 +- docs/plans/local-dev-postgres-setup.md | 153 ++++++++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 docs/plans/local-dev-postgres-setup.md diff --git a/.gitignore b/.gitignore index 4236e68..41e352a 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ apps/dashboard/node_modules/ # ============================================================================= fastembed-rs/ .mcp.json + +.claude/ +.codebase-memory/ diff --git a/docs/plans/0002-phase-2-postgres-backend.md b/docs/plans/0002-phase-2-postgres-backend.md index a372e27..3fe28f2 100644 --- a/docs/plans/0002-phase-2-postgres-backend.md +++ b/docs/plans/0002-phase-2-postgres-backend.md @@ -2,7 +2,7 @@ **Status**: Draft **Depends on**: Phase 1 (MemoryStore + Embedder traits, embedding_model registry, domain columns) -**Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 2), docs/prd/001-getting-centralized-vestige.md +**Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 2), docs/prd/001-getting-centralized-vestige.md, docs/plans/local-dev-postgres-setup.md (local cluster provisioning) --- @@ -85,6 +85,7 @@ postgres-backend = ["dep:sqlx", "dep:pgvector", "dep:tokio-stream", "dep:futures - The `pgvector` extension installed in the target database (our migration issues `CREATE EXTENSION IF NOT EXISTS vector`). - `sqlx-cli@0.8` installed on the developer machine for `cargo sqlx prepare --workspace` and `cargo sqlx migrate add` (not a build-time requirement once `.sqlx/` is committed). - Docker or Podman reachable by the test harness for `testcontainers-modules::postgres` to spin up `pgvector/pgvector:pg16`. +- A local Postgres cluster for `sqlx prepare`, manual migration work, and `vestige migrate --to postgres` smoke runs. The recipe for standing one up on Arch/CachyOS (install, initdb, role + db, pgvector, connection string at `~/.vestige_pg_pw`) lives in `docs/plans/local-dev-postgres-setup.md`. Postgres 18 from the Arch repo satisfies the "16 or newer" requirement above. Phase 2 work assumes `DATABASE_URL` points at that cluster once migrations are applied. ### Assumed Rust toolchain diff --git a/docs/plans/local-dev-postgres-setup.md b/docs/plans/local-dev-postgres-setup.md new file mode 100644 index 0000000..6250a55 --- /dev/null +++ b/docs/plans/local-dev-postgres-setup.md @@ -0,0 +1,153 @@ +# Local Dev Postgres Setup (Arch / CachyOS) + +**Status**: Applied on this machine on 2026-04-21 +**Related**: docs/plans/0002-phase-2-postgres-backend.md, docs/adr/0001-pluggable-storage-and-network-access.md + +Purpose: capture the minimum, repeatable steps to stand up a Postgres 18 instance on a local Arch/CachyOS box for Phase 2 (`PgMemoryStore`) development, `sqlx prepare`, and manual migration testing. This is a single-operator dev recipe, not a production runbook. + +--- + +## Current state on this machine + +- Package: `postgresql` 18.3-2 (pacman). Pulls `postgresql-libs`, `libxslt`. +- Service: `postgresql.service`, enabled + active. +- Listens on: `127.0.0.1:5432` and `[::1]:5432` only (default `listen_addresses = 'localhost'`). +- Data dir: `/var/lib/postgres/data`, owner `postgres:postgres`. +- Auth (`pg_hba.conf`, Arch defaults): `peer` for local socket, `scram-sha-256` for host 127.0.0.1/::1. + +### Database + role + +- Database: `vestige`, UTF8, owner `vestige`. +- Role: `vestige` with `LOGIN CREATEDB` (no superuser, no replication, no cross-db). +- Schema `public` re-owned to `vestige`, plus default privileges so any future tables / sequences / functions in `public` are fully owned and granted to `vestige`. + +Net effect: the `vestige` role can create, alter, drop, and grant freely inside the `vestige` database -- enough for `sqlx::migrate!`, ad-hoc schema work, and the full Phase 2 `MemoryStore` surface. It cannot create extensions (see Phase 2 followups below) and cannot touch other databases. + +### Connection + +``` +postgresql://vestige:@127.0.0.1:5432/vestige +``` + +Password lives at `~/.vestige_pg_pw`, mode 600, owned by the dev user (no sudo needed to read it). Read with: + +```sh +cat ~/.vestige_pg_pw +``` + +Recommended dev shell export (keep this OUT of the repo; use `.env` + gitignore or a shell rc): + +```sh +export DATABASE_URL="postgresql://vestige:$(cat ~/.vestige_pg_pw)@127.0.0.1:5432/vestige" +``` + +--- + +## Reproduce from scratch + +On a fresh Arch / CachyOS box with passwordless sudo: + +```sh +# 1. Install +sudo pacman -S --noconfirm postgresql + +# 2. Initialize the cluster (UTF8, scram-sha-256 for host, peer for local) +sudo -iu postgres initdb \ + --locale=C.UTF-8 --encoding=UTF8 \ + -D /var/lib/postgres/data \ + --auth-host=scram-sha-256 --auth-local=peer + +# 3. Start + enable +sudo systemctl enable --now postgresql + +# 4. Generate a password and stash it in the dev user's home (mode 600) +VESTIGE_PW=$(python3 -c 'import secrets,string; a=string.ascii_letters+string.digits; print("".join(secrets.choice(a) for _ in range(32)))') +umask 077 +printf '%s' "$VESTIGE_PW" > ~/.vestige_pg_pw +chmod 600 ~/.vestige_pg_pw + +# 5. Create role + database + grants +sudo -u postgres psql -v ON_ERROR_STOP=1 < ~/.vestige_pg_pw +chmod 600 ~/.vestige_pg_pw +sudo -u postgres psql -v ON_ERROR_STOP=1 \ + -c "ALTER ROLE vestige WITH PASSWORD '${NEW_PW}';" +unset NEW_PW +``` + +Then re-export `DATABASE_URL` in any live shells. + +--- + +## Teardown + +Destroys the cluster and all data in it: + +```sh +sudo systemctl disable --now postgresql +sudo pacman -Rns postgresql postgresql-libs +sudo rm -rf /var/lib/postgres +rm -f ~/.vestige_pg_pw +``` + +--- + +## Out of scope for this doc + +- TLS, client-cert auth, non-localhost access. Phase 3 exposes the Vestige HTTP API over the network, not Postgres directly. +- Backups, PITR, WAL archiving. For dev data: `pg_dump -h 127.0.0.1 -U vestige vestige > vestige.sql`. +- Replication, PgBouncer, tuned `postgresql.conf`. Defaults are fine for Phase 2 development. +- Making this the canonical Vestige backend. By default Vestige still uses SQLite; this cluster exists so the `postgres-backend` feature can be built and tested locally. From c584ec8afeb8a82b9659b2c1fc36fc900f997c4b Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 09:35:23 +0200 Subject: [PATCH 03/38] docs(adr): add ADR 0002 -- Phase 2 execution Binding ADR for Phase 2 Postgres backend integration plus the Phase 1 amendment that removes async_trait from the storage and embedder traits. Decisions D1-D8: - D1: sunset async_trait across MemoryStore + Embedder via trait_variant - D2: PgMemoryStore::connect(url, max_connections) mirrors SqliteMemoryStore; no Embedder in constructor; register_model handles pgvector typmod - D3: split sqlite.rs into a sqlite/ directory as Phase 1 amendment - D4: postgres/ as a directory from day one - D5: sub-plan layout -- 3 Phase 1 amendment + 9 Phase 2 sub-plans - D6: no separate ADR for the SQLite split (pure code motion) - D7: reserve multi-tenancy schema (users/groups/group_memberships + owner_user_id/visibility/shared_with_groups) in Phase 2 so Phase 3 auth is additive, not an online migration over an HNSW-indexed table - D8: codebase promoted to a first-class indexed column on knowledge_nodes; mcp_client_id and session_id stay in metadata JSONB PR cadence: PR A = Phase 1 amendment (code on feat/storage-trait-phase1); PR B = this ADR + Phase 2 sub-plans (docs only); PR C = Phase 2 implementation. Phase 4 sharing_rules table sketched in Follow-ups. --- docs/adr/0002-phase-2-execution.md | 545 +++++++++++++++++++++++++++++ 1 file changed, 545 insertions(+) create mode 100644 docs/adr/0002-phase-2-execution.md diff --git a/docs/adr/0002-phase-2-execution.md b/docs/adr/0002-phase-2-execution.md new file mode 100644 index 0000000..6b6949f --- /dev/null +++ b/docs/adr/0002-phase-2-execution.md @@ -0,0 +1,545 @@ +# ADR 0002: Phase 2 Execution -- Postgres Backend Integration, Phase 1 Amendment + +**Status**: Accepted +**Date**: 2026-05-26 +**Related**: [docs/adr/0001-pluggable-storage-and-network-access.md](0001-pluggable-storage-and-network-access.md), [docs/plans/0002-phase-2-postgres-backend.md](../plans/0002-phase-2-postgres-backend.md) + +--- + +## Context + +ADR 0001 set the architectural direction: introduce `MemoryStore` and `Embedder` +traits, ship a Postgres backend behind a feature flag, and reach a single shared +memory brain across machines. Phase 1 (storage trait extraction) shipped on +`feat/storage-trait-phase1` (790c0c8). The Phase 2 master plan at +`docs/plans/0002-phase-2-postgres-backend.md` was drafted before Phase 1 was +frozen. + +Starting Phase 2 surfaces a small set of execution-level decisions that ADR 0001 +did not cover and that the master plan now disagrees with the live code on. +These decisions are too big to silently absorb into a per-step plan and too +small to amend ADR 0001. They live here. + +Three concrete realities driving this ADR: + +1. **Trait shape mismatch.** Master plan 0002 assumed `trait_variant::make` + produced distinct `MemoryStore` (Send-bound) and `LocalMemoryStore` + (non-Send) variants, and that errors were `StoreError`. Phase 1 froze on + `#[async_trait::async_trait]` with `pub use MemoryStore as LocalMemoryStore` + and an error type called `MemoryStoreError`. The Postgres backend has to + follow Phase 1, not the master plan -- but we should record that explicitly. +2. **`SqliteMemoryStore` is monolithic.** + `crates/vestige-core/src/storage/sqlite.rs` is ~8200 lines. Phase 1 appended + the trait impl block at the bottom of the same file. Adding a similarly + large `postgres.rs` perpetuates the pattern; this is the natural moment to + decide whether the SQLite file gets split. +3. **Constructor surface drift.** Master plan 0002 specifies + `PgMemoryStore::connect(url, max_connections, &dyn Embedder)`. The Phase 1 + `SqliteMemoryStore` constructor takes no embedder -- registry consistency + runs through `registered_model()` / `register_model()` on the trait, + invoked by the caller. The two backends should look the same to a caller; + right now they would not. +4. **Multi-tenancy is a one-way door.** The Postgres schema is the place to + reserve user/group/visibility columns *now*, even though Phase 3 is the + phase that wires the auth filter using them. Adding `owner_user_id` and + GIN indexes to a populated, HNSW-indexed `knowledge_nodes` table later is an + expensive online migration; reserving NULL-defaulted columns at schema + creation is ~10 lines of SQL. The same logic applies to per-memory + context capture (codebase, MCP caller, session) -- promoting `codebase` + to a first-class column now keeps the door open for context-aware + sharing rules in Phase 4 without touching `knowledge_nodes`. See D7 and D8. + +This ADR is also the umbrella under which Phase 2 sub-plans (`0002a-...`, +`0002b-...`, etc.) sit. The intent is: ADR + sub-plans land as one PR for +review; the implementation lands as a second PR with many commits inside. + +--- + +## Already Decided (carried in by reference) + +These are settled by ADR 0001 or by explicit agreement during this session. +Listed here so the discussion frame is clear; not re-litigated below. + +- Postgres backend ships behind a `postgres-backend` Cargo feature, default + OFF. Mutually compilable with SQLite. (ADR 0001.) +- Single big `MemoryStore` trait. `PgMemoryStore` implements the same surface + as `SqliteMemoryStore`. (ADR 0001.) +- pgvector HNSW + tsvector + GIN + RRF hybrid search in one SQL statement. + (Master plan 0002, D4-D5.) +- sqlx 0.8 + pgvector 0.4 + compile-time-checked queries + offline `.sqlx/` + cache committed. (Master plan 0002.) +- Two sqlx migration files: `0001_init` (extensions, tables, non-vector + indexes) and `0002_hnsw` (HNSW separated for re-embed drop/recreate). + (Master plan 0002, D4.) +- `vestige migrate --from sqlite --to postgres` and + `vestige migrate --reembed --model=` CLI subcommands. (ADR 0001 + + master plan 0002, D8-D10.) +- PR cadence: PR #1 carries this ADR plus all sub-plans; PR #2 carries the + implementation as many commits. +- Sub-plans use `0002a-`, `0002b-`, ... suffixes off `0002-`. +- `PgMemoryStore::connect` lands as `todo!()` in the skeleton; real body + comes later. + +--- + +## Decisions + +### D1. Sunset async_trait across the Phase 1 traits + +Phase 1 froze with `#[async_trait::async_trait]` on both the `MemoryStore` +trait (`storage/memory_store.rs:194`) and the `Embedder` trait +(`embedder/mod.rs:27`), plus their SQLite and Fastembed impl blocks. async_trait +boxes every async fn into `Pin>` -- one heap allocation +per call inside the hottest code path. We are amending Phase 1 to remove +async_trait entirely and replace it with `trait_variant::make`, so each trait +becomes two real generated variants (`MemoryStore` / `LocalMemoryStore`, +`Embedder` / `LocalEmbedder`) with `Send` bounds on the outer variant. + +Scope split across three Phase 1 amendment sub-plans: + +- **`0001a-trait-rewrite.md`** -- Rewrite `MemoryStore` only. Touches + `storage/memory_store.rs` (trait declaration) and `storage/sqlite.rs` + (impl block attribute). Leaves async_trait in place on the embedder side + so the diff stays focused. +- **`0001b-sqlite-split.md`** -- Pure code motion. Splits the + ~8200-line `sqlite.rs` into a `sqlite/` directory. Independent of D1; can + land in either order relative to `0001a`. +- **`0001c-async-trait-sunset.md`** -- Rewrite `Embedder` the same way, then + remove `async-trait = "0.1"` from `crates/vestige-core/Cargo.toml`. Final + amendment commit removes the dependency entirely. After this lands, the + workspace contains zero references to `async_trait`. + +All three sub-plans land on the existing `feat/storage-trait-phase1` branch +(790c0c8 has not been opened upstream yet; amend in place, no force-push to a +public PR). + +### D2. PgMemoryStore::connect mirrors SqliteMemoryStore::new + +```rust +impl PgMemoryStore { + pub async fn connect(url: &str, max_connections: u32) -> MemoryStoreResult; + pub async fn from_pool(pool: PgPool) -> MemoryStoreResult; +} +``` + +No `Embedder` in the constructor. The pgvector-specific +`ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector($N)` DDL lives +inside the trait method `register_model(&ModelSignature)`. That method is +called by the caller (cognitive engine bootstrap, migrate CLI, tests) after +construction, exactly as it is for `SqliteMemoryStore`. + +`MemoryStoreError` gains two variants behind the feature flag (added during +the Postgres impl, not during the Phase 1 amendment): +```rust +#[cfg(feature = "postgres-backend")] +#[error("postgres error: {0}")] +Postgres(#[from] sqlx::Error), + +#[cfg(feature = "postgres-backend")] +#[error("postgres migration error: {0}")] +Migrate(#[from] sqlx::migrate::MigrateError), +``` + +### D3. Split sqlite.rs into a sqlite/ directory as Phase 1 amendment + +Pure code motion, no behavioural change. Target layout: +``` +crates/vestige-core/src/storage/sqlite/ + mod.rs -- SqliteMemoryStore struct, new(), reader/writer locks + crud.rs -- insert/get/update/delete + search.rs -- fts_search, vector_search, hybrid search + scheduling.rs -- FSRS state methods + graph.rs -- edges, neighbors + domain.rs -- domain CRUD, classify stub + registry.rs -- embedding_model table + register_model + portable_sync.rs -- portable archive backend bridge + trait_impl.rs -- impl LocalMemoryStore for SqliteMemoryStore +``` + +Cognitive-module imports stay on `crate::storage::SqliteMemoryStore` and +related re-exports from `storage/mod.rs`; the split is private to the +module. Each motion commit must keep `cargo test -p vestige-core` green for +bisectability. + +This lands in the Phase 1 amendment PR alongside D1 (separate commit, same +branch). + +### D4. Postgres backend as a directory from day one + +``` +crates/vestige-core/src/storage/postgres/ + mod.rs -- PgMemoryStore struct, connect, from_pool, trait impl + pool.rs -- PgPool construction from PostgresConfig + migrations.rs -- sqlx::migrate! wrapper + registry.rs -- ensure_registry, ALTER COLUMN TYPE vector(N) + search.rs -- RRF query + row mapping + migrate_cli.rs -- SQLite -> Postgres streaming copy + reembed.rs -- O(n) re-encode + HNSW rebuild +``` + +D1+D2 of the master plan land first as a skeleton in `mod.rs` with `todo!()` +bodies; later sub-plans fill in the other files. + +### D5. Sub-plan layout: two phases worth of sub-plans + +Phase 1 amendment sub-plans (under `docs/plans/`): +- `0001a-trait-rewrite.md` -- MemoryStore async_trait -> trait_variant, call-site audit +- `0001b-sqlite-split.md` -- sqlite.rs -> sqlite/ directory, commit-by-commit +- `0001c-async-trait-sunset.md` -- Embedder rewrite + drop async-trait dep from Cargo.toml + +Phase 2 sub-plans (under `docs/plans/`): +- `0002a-skeleton-and-feature-gate.md` -- master plan D1 + D2 (todo!() bodies) +- `0002b-pool-and-config.md` -- master plan D3 + D7 +- `0002c-migrations.md` -- master plan D4 +- `0002d-store-impl-bodies.md` -- master plan D2 real bodies + D6 registry +- `0002e-hybrid-search.md` -- master plan D5 +- `0002f-migrate-cli.md` -- master plan D8 + D10 +- `0002g-reembed.md` -- master plan D9 +- `0002h-testing-and-benches.md` -- master plan D14 + D15 +- `0002i-runbook.md` -- master plan D16 + +Each sub-plan is a self-contained brief sized to fit one focused +implementation session (handed to Claude Code as a `/goal` instruction +without requiring the agent to load the master plan). + +### D6. SQLite split does not get its own ADR + +The split is pure code motion; no public types, behaviour, or paths change. +`0001b-sqlite-split.md` is enough. + +### D7. Multi-tenancy schema reservation (L1-L3) + +Phase 2 reserves the columns and tables needed for future per-user / per-group +visibility, so Phase 3 (auth) does not require a column-add migration over a +populated, HNSW-indexed `knowledge_nodes` table. Single-user behaviour is unchanged +in both backends. + +New tables in `0001_init.up.sql`: + +```sql +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE TABLE groups ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE TABLE group_memberships ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + group_id UUID NOT NULL REFERENCES groups(id) ON DELETE CASCADE, + role TEXT NOT NULL DEFAULT 'member', -- 'member' | 'admin' + joined_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, group_id) +); + +INSERT INTO users (id, handle, display_name) + VALUES ('00000000-0000-0000-0000-000000000001', 'local', 'Local User'); +``` + +New columns on `knowledge_nodes`: + +```sql +owner_user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000001' + REFERENCES users(id), +visibility TEXT NOT NULL DEFAULT 'private', -- 'private' | 'group' | 'public' +shared_with_groups UUID[] NOT NULL DEFAULT '{}' + +CREATE INDEX idx_knowledge_nodes_owner ON knowledge_nodes (owner_user_id); +CREATE INDEX idx_knowledge_nodes_shared_groups ON knowledge_nodes USING GIN (shared_with_groups); +``` + +Phase 3 visibility filter (declared here for reference; implemented in Phase 3): + +```sql +WHERE + (visibility = 'private' AND owner_user_id = $me) + OR (visibility = 'group' + AND (owner_user_id = $me OR shared_with_groups && $my_group_ids)) + OR visibility = 'public' +``` + +Why tri-state enum and not just `shared_with_groups[] + is_public`: the +explicit `visibility` field documents intent at the row level. A `'private'` +row with a non-empty `shared_with_groups` is detectable inconsistency +(a CHECK constraint can enforce it later) rather than silent data. + +SQLite parity: same tables and columns with identical defaults. +`shared_with_groups` is a JSON `'[]'` text encoding (no array type). +Single-user mode never changes any of these values; the trait surface ignores +the visibility filter for SQLite because there is exactly one user. + +Sharing automation (matching by domain, tag, repo, MCP caller, ...) is +explicitly **not** in Phase 2. See D8 for context capture, and the Follow-ups +section for the Phase 4 `sharing_rules` design sketch. + +RLS policies are not declared in Phase 2. Phase 3 decides whether to add +RLS as defense-in-depth on top of the app-layer filter. + +### D8. Context-aware ingest + +Every memory carries its ingest context, so future automation (sharing rules, +domain scoping, audit) can match on it without a schema migration. Most of +this is already happening in the Phase 1 ingest pipeline; D8 promotes it to +ADR-level commitment so Phase 2 cannot drop it on the way to Postgres. + +Context dimensions and where they live: + +- **`codebase`** -- promoted to a first-class indexed column on `knowledge_nodes`. + High-frequency query path (`SELECT ... WHERE codebase = 'vestige'`) for + both human exploration and Phase 4 HDBSCAN scoping. Direct B-tree index + beats JSONB extraction. + ```sql + codebase TEXT, -- nullable; populated from ingest context + CREATE INDEX idx_knowledge_nodes_codebase ON knowledge_nodes (codebase) WHERE codebase IS NOT NULL; + ``` + `MemoryRecord` gains `pub codebase: Option`. + +- **`mcp_client_id`** -- which MCP caller created this. Persistent identity + once Phase 3 API keys exist. Lives in `metadata.mcp_client_id` (JSONB). + Not query-hot enough to deserve a column. + +- **`session_id`** -- ephemeral; identifies the calling session for runtime + override scoping. Lives in `metadata.session_id` (JSONB). Sessions die + fast; storing them as rows or indexed columns is waste. + +- **`file` / `topics`** -- existing optional context already accepted by the + ingest pipeline. Stay in metadata JSONB. + +Phase 2's job for D8 is operational, not architectural: audit the ingest +path from MCP request to row write to ensure none of these fields gets +dropped when crossing the SQLite -> Postgres backend boundary. + +--- + +## PR Cadence + +Two work streams, three PRs total: + +1. **PR A: Phase 1 amendment** + - Branch: `feat/storage-trait-phase1` (existing, amended in place) + - Commits: MemoryStore trait rewrite (0001a) + sqlite split (0001b, multiple + motion commits) + Embedder rewrite & async-trait dep removal (0001c). + - Sub-plans `0001a-`, `0001b-`, `0001c-` are committed on this branch. + +2. **PR B: ADR 0002 + Phase 2 sub-plans (this document + the 9 sub-plans)** + - New branch off PR A's tip once that is reviewed. + - No code; docs only. + +3. **PR C: Phase 2 implementation** + - New branch off PR B's tip. + - One PR with many commits clustered by sub-plan. + +PR B is the "let's discuss execution before writing code" gate. PR C is the +"now we write code" gate. If PR A is itself sizable enough that it needs the +amendments reviewed in stages, the three sub-plans (`0001a`, `0001b`, `0001c`) +can split into separate PRs; that's a tactical call at PR time. + +--- + +## Architecture Overview + +Final layout after the Phase 1 amendment (PR A) and Phase 2 implementation +(PR C): + +``` +crates/vestige-core/src/storage/ + mod.rs -- re-exports, Storage alias for BC + memory_store.rs -- trait_variant-generated MemoryStore + LocalMemoryStore, types, error + migrations.rs -- SQLite migration registry (Phase 1, unchanged) + portable.rs -- portable archive format (Phase 1, unchanged) + sqlite/ -- was sqlite.rs (D3, Phase 1 amendment) + mod.rs -- SqliteMemoryStore struct, new(), reader/writer locks + crud.rs -- insert/get/update/delete + search.rs -- fts/vector/hybrid + scheduling.rs -- FSRS state + graph.rs -- edges, neighbors + domain.rs -- domain CRUD, classify stub + registry.rs -- embedding_model table + register_model + portable_sync.rs -- portable backend bridge + trait_impl.rs -- impl LocalMemoryStore for SqliteMemoryStore + postgres/ -- D4, Phase 2 + mod.rs -- PgMemoryStore struct, connect, from_pool, trait impl + pool.rs -- PgPool construction from config + migrations.rs -- sqlx::migrate! wrapper + registry.rs -- register_model body, ALTER COLUMN TYPE vector(N) + search.rs -- RRF query + row mapping + migrate_cli.rs -- SQLite -> Postgres streaming copy + reembed.rs -- O(n) re-encode + HNSW rebuild + +crates/vestige-core/migrations/ + sqlite/ -- Phase 1, with V15 migration for D7+D8 columns/tables + postgres/ -- Phase 2 + 0001_init.up.sql -- includes D7 tables + columns, D8 codebase column + 0001_init.down.sql + 0002_hnsw.up.sql + 0002_hnsw.down.sql +``` + +Tables in the Postgres schema after migration 0001: + +| Table | Purpose | Phase that populates | +|-------|---------|----------------------| +| `embedding_model` | One-row registry of name/dim/hash | Phase 2 (first connect) | +| `knowledge_nodes` | Core records + owner/visibility/codebase | Phase 2 ingest; Phase 4 fills `domains` | +| `scheduling` | FSRS state | Phase 2 | +| `edges` | Spreading activation graph | Phase 2 | +| `review_events` | Append-only FSRS review log | Phase 2; Phase 5 federation reads | +| `domains` | Phase 4 cluster centroids | Phase 4 | +| `users` | L1 identities (D7) | Phase 3 | +| `groups` | L3 groups (D7) | Phase 3 | +| `group_memberships` | L3 user-group links (D7) | Phase 3 | + +`sharing_rules` (Phase 4) and `api_keys` (Phase 3) are added later by their +own migrations. + +--- + +## Alternatives Considered + +| Alternative | Why not | +|-------------|---------| +| Keep async_trait on the Phase 1 trait | One heap allocation per trait call inside the hottest code path in Vestige. Boxing every future also obscures the actual return type, which makes lifetimes and Send-ness harder to reason about. The Phase 1 PR is not opened upstream yet, so amending is free. | +| Take `&dyn Embedder` into `connect` | Couples constructor to embedder; breaks ADR 0001's separation; can't be used by callers that don't have an embedder yet (tests, migrate CLI). | +| Defer SQLite split | Postgres lands alongside an 8K-line peer; the pattern compounds; future readers see "backends are huge here". | +| Single `postgres.rs` | Master plan calls out 7 sub-files; we know it's getting split; doing it twice is waste. | +| Per-deliverable sub-plans (16 docs) | Review fatigue; many sub-plans would be 3-5 lines of Cargo or one migration each. Logical groups cluster naturally with PR commits. | +| One rolling sub-plan with checkboxes | Moving target; doesn't serve as a `/goal` brief for a fresh Claude Code session. | +| Separate ADR for the SQLite split | Pure code motion with no public-surface change; doesn't constrain future decisions. ADRs are for decisions that bind. | +| Punt multi-tenancy schema entirely to Phase 3 | Adding `owner_user_id` and indexes to a populated, HNSW-indexed `knowledge_nodes` table later is an expensive online migration. Reserving NULL-defaulted columns now is ~10 lines of SQL. | +| `shared_with_groups[] + is_public` instead of tri-state visibility enum | More compact but `visibility = 'private'` documents intent at the row level; a CHECK constraint can later enforce array/enum consistency. Two columns conveying one fact is fine when both are referenced often. | +| Add `shared_with_users[]` for direct user-to-user sharing | A "group of one" subsumes it without an extra column and GIN index. Phase 3 CLI can auto-create singleton groups if a user requests direct shares. | +| Bake per-domain or per-tag sharing defaults into Phase 2 schema | Sharing automation needs real usage data before committing to fuzzy (domain centroids) vs crisp (tags) vs context (codebase / MCP caller). Phase 4 designs a generic `sharing_rules` table that matches on any context dimension; deferring costs nothing because rules live in a new table, not new columns. | +| `codebase` stays in JSONB metadata | High-frequency query path (HDBSCAN scoping, codebase-wide searches, future `sharing_rules` match). B-tree on a real column beats GIN on a JSONB key for this access pattern. Cost is one nullable TEXT column. | + +--- + +## Consequences + +### Positive +- Phase 1 trait stops boxing futures on every call. Lifetimes and Send-ness + become inspectable instead of hidden inside an `async_trait` macro expansion. +- `connect` stays backend-agnostic; tests and CLI tools stand up either backend + without an `Embedder` in scope. +- Cognitive module imports never change paths -- the SQLite split is private + to `storage/sqlite/`, public re-exports through `storage/mod.rs` unchanged. +- Postgres backend lands already-modular; future SQL changes touch one of + seven small files, not one of eight thousand lines. +- Phase 2 master plan stays archival; ADR 0002 + sub-plans are the live source + of truth for execution. +- Multi-tenancy columns reserved now means Phase 3 auth is purely additive -- + no online migration over a populated, HNSW-indexed `knowledge_nodes` table. +- Context-aware ingest (D8) keeps the door open for repo / session / + MCP-caller-scoped sharing rules in Phase 4 without changing `knowledge_nodes`. + +### Negative +- The Phase 1 amendment expands a "finished" branch. It is a real cost: the + trait rewrite touches every cognitive module that holds a store handle. +- SQLite split is a pure-motion diff. Annoying to review even when safe. +- Three PRs (amendment, ADR+plans, implementation) instead of one or two. + Discipline tax in exchange for reviewability. +- Multi-tenancy reservation adds three never-queried tables and three + always-default columns to the SQLite schema. Real but small storage cost in + single-user mode (a single bootstrap row + empty tables + NULL/empty + defaults per memory). + +### Risks +- **Trait rewrite breaks a cognitive module's Send-ness expectation.** + Mitigation: `cargo test --workspace` runs after each call-site edit; + trait_variant-generated `MemoryStore` is the Send variant and matches the + current `Arc` usage everywhere except thread-local impls (none + exist today). +- **SQLite motion commit introduces a silent semantic change.** Mitigation: + each commit keeps `cargo test -p vestige-core` green; reviewer can bisect. +- **Sub-plan boundaries don't match how implementation wants to commit.** + Mitigation: sub-plans are advisory; the implementation PR clusters commits + however it ends up needing to. +- **Reserved columns get used in Phase 3 in a way that mismatches Phase 2 + defaults.** Mitigation: Phase 3 owns the auth filter; Phase 2 defaults + (`owner_user_id = local`, `visibility = 'private'`) are intentionally the + "no access for anyone but the owner" worst-case; widening at Phase 3 is + safe, narrowing would be the dangerous direction. +- **Memory: PR A amendment invalidates the locally-deployed Phase 1 binary's + ABI.** Not a real risk -- the trait change is purely source-level Rust; the + on-disk DB schema is unchanged. The rebuilt binary slots in over the + current one without DB migration. + +--- + +## Resolved Decisions + +| # | Question | Resolution | +|---|----------|------------| +| Q1 | Phase 1 trait shape | Rewrite with trait_variant::make. Amend Phase 1 PR. | +| Q2 | PgMemoryStore::connect signature | Mirror SqliteMemoryStore::new; no Embedder. register_model does the pgvector typmod stamp. | +| Q3 | Split sqlite.rs | Yes, as Phase 1 amendment. sqlite.rs -> sqlite/ directory; pure code motion. | +| Q4 | Postgres module layout | Directory from day one. | +| Q5 | Sub-plan granularity | Logical groups, ~9 docs for Phase 2 plus 2 for the Phase 1 amendment. | +| Q6 | ADR for SQLite split | No. Sub-plan `0001b-sqlite-split.md` is sufficient. | +| Q7 | Multi-tenancy schema | Reserve users / groups / group_memberships tables and owner_user_id / visibility / shared_with_groups columns on knowledge_nodes in Phase 2. Single-user defaults; Phase 3 fills in real values. | +| Q8 | Visibility encoding | Tri-state enum `'private' \| 'group' \| 'public'` plus `shared_with_groups[]`. No `shared_with_users[]`; no RLS in Phase 2. | +| Q9 | Sharing automation grain | Per-memory only in Phase 2. Phase 4 ships a generic `sharing_rules` table matching on codebase / tag / node_type / mcp_client_id. | +| Q10 | Context capture on ingest | `codebase` promoted to a first-class indexed column; `mcp_client_id` and `session_id` stay in metadata JSONB. | + +--- + +## Follow-ups + +- Phase 1 amendment sub-plans drafted: `0001a-trait-rewrite.md`, + `0001b-sqlite-split.md`, `0001c-async-trait-sunset.md`. Ready to execute on + `feat/storage-trait-phase1`. +- Phase 2 sub-plans drafted: `0002a-` through `0002i-` against the accepted + decisions above. Ready to execute on a new branch off PR A's tip. +- Decide branch placement for this ADR before it gets committed -- it cannot + live on `feat/storage-trait-phase1` (that branch is now PR A's code-only + amendment branch). Likely a new branch off PR A's tip for PR B (docs only). +- Validate local Postgres dev cluster before PR C work begins. Recipe at + `docs/plans/local-dev-postgres-setup.md` is correct but needs to be applied + on this machine (delandtj-home): cluster is not initdb'd, pgvector is not + installed. Containerized `pgvector/pgvector:pg16` is a viable alternative + if pgvector packaging is friction. See open discussion thread. + +### Phase 4 sketch: `sharing_rules` and the precedence chain + +Recorded here so the Phase 4 author does not have to rediscover the design. +Phase 2 does **not** implement any of this; it only ensures the schema and +ingest context capture make this possible without a `knowledge_nodes` migration. + +```sql +-- Phase 4 migration (not Phase 2) +CREATE TABLE sharing_rules ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + owner_user_id UUID NOT NULL REFERENCES users(id), + -- Match: any subset; all set fields must match conjunctively + match_codebase TEXT, + match_tag TEXT, + match_node_type TEXT, + match_api_key_id UUID REFERENCES api_keys(id), -- MCP caller identity + -- Policy + visibility TEXT NOT NULL, + shared_with_groups UUID[] NOT NULL DEFAULT '{}', + -- Conflict resolution + priority INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +``` + +Precedence on ingest, first match wins: + +1. Caller-explicit visibility in the MCP request +2. Active session override held by the MCP server (per-session, in-memory, + not persisted; matched by `session_id`) +3. Highest-priority `sharing_rules` row whose match fields all hold +4. User's `default_visibility` (typically `'private'`) + +Per-session overrides do not persist; storing ephemeral session IDs as DB +rows is waste. Per-codebase / per-MCP-caller rules do persist as +`sharing_rules` rows. From 697ade5b02decdfaf1165958f945f0f92b905497 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 09:35:37 +0200 Subject: [PATCH 04/38] docs(plans): add Phase 1 amendment sub-plans 0001a/b/c Three sub-plans operationalising ADR 0002 D1 + D3 on the existing feat/storage-trait-phase1 branch (790c0c8, not yet pushed upstream): - 0001a-trait-rewrite.md -- rewrite MemoryStore with #[trait_variant::make(MemoryStore: Send)] generating a non-Send LocalMemoryStore companion. Production callers use Arc and are unaffected; only the trait declaration and SQLite impl block change. - 0001b-sqlite-split.md -- pure code motion. Split sqlite.rs (8713 lines) into a sqlite/ directory (mod, crud, search, scheduling, graph, domain, registry, portable_sync, trait_impl). Public re-exports unchanged; tests green commit-by-commit. Depends on 0001a so trait_impl.rs picks up the trait_variant attribute once. - 0001c-async-trait-sunset.md -- rewrite Embedder the same way, then remove async-trait = "0.1" from crates/vestige-core/Cargo.toml. End state: zero async_trait references in the workspace. Together these three lands as PR A. --- docs/plans/0001a-trait-rewrite.md | 689 ++++++++++++++++++++ docs/plans/0001b-sqlite-split.md | 732 +++++++++++++++++++++ docs/plans/0001c-async-trait-sunset.md | 847 +++++++++++++++++++++++++ 3 files changed, 2268 insertions(+) create mode 100644 docs/plans/0001a-trait-rewrite.md create mode 100644 docs/plans/0001b-sqlite-split.md create mode 100644 docs/plans/0001c-async-trait-sunset.md diff --git a/docs/plans/0001a-trait-rewrite.md b/docs/plans/0001a-trait-rewrite.md new file mode 100644 index 0000000..354e138 --- /dev/null +++ b/docs/plans/0001a-trait-rewrite.md @@ -0,0 +1,689 @@ +# Phase 1 Amendment Sub-Plan: async_trait -> trait_variant + +**Status**: Draft +**Depends on**: Phase 1 (already on `feat/storage-trait-phase1`, tip 790c0c8) +**Followed by**: `docs/plans/0001c-async-trait-sunset.md` (Embedder rewrite + async-trait crate removal) +**Related**: docs/adr/0002-phase-2-execution.md (decision D1), docs/plans/0001-phase-1-storage-trait-extraction.md + +--- + +## Context + +Phase 1 froze with the storage trait declared as +`#[async_trait::async_trait] pub trait MemoryStore: Send + Sync + 'static` +and a `pub use MemoryStore as LocalMemoryStore;` alias. `async_trait` boxes +every `async fn` in the trait into `Pin>`. That is one +heap allocation per call inside the hottest code path in Vestige (insert, +search, update_scheduling are all on this surface). It also collapses the +two intended trait shapes -- a Send-bound `MemoryStore` for tokio/axum and a +non-Send `LocalMemoryStore` for thread-local backends -- into a single trait +behind an alias, removing the type-system signal we will want for Phase 2's +Postgres backend. + +ADR 0002 decision D1 supersedes this. The amendment lands on the existing +`feat/storage-trait-phase1` branch before Phase 2 starts, before the PR is +opened upstream. The end state: + +- `LocalMemoryStore` is the source-of-truth trait, declared with native + async-fn-in-trait (stable on MSRV 1.91), no `Sync` bound on the trait + itself, and `Sync + 'static` on the trait object. +- `MemoryStore` is auto-generated by `#[trait_variant::make(MemoryStore: Send)]` + with `Send` bounds on every returned future, so `Arc` is + movable across `tokio::spawn`. +- `trait-variant` 0.1 is already present in `crates/vestige-core/Cargo.toml`. + The `async-trait` crate dependency stays in place through this sub-plan + (it is still in use by the embedder impl); removing it is the job of + `0001c-async-trait-sunset.md`. +- No production caller changes. Every production call site holds + `Arc` (the concrete `SqliteMemoryStore` alias), which the trait + rewrite does not touch. Only the trait declaration and impl blocks change. + +--- + +## Scope + +### In scope + +- Rewrite `MemoryStore` / `LocalMemoryStore` declaration in + `crates/vestige-core/src/storage/memory_store.rs` to use + `#[trait_variant::make(MemoryStore: Send)] pub trait LocalMemoryStore`. +- Remove the `pub use MemoryStore as LocalMemoryStore;` alias from the same + file. `LocalMemoryStore` becomes the real trait; `MemoryStore` is the + generated Send variant. +- Drop the `#[async_trait::async_trait]` attribute on the SQLite impl block + in `crates/vestige-core/src/storage/sqlite.rs` (line 6274). The impl block + switches from `impl MemoryStore for SqliteMemoryStore` (currently spelled + through the `LocalMemoryStore` alias) to `impl LocalMemoryStore for + SqliteMemoryStore`. `trait_variant` provides a blanket `impl MemoryStore for T` so `&dyn MemoryStore` and + `Arc` keep working unchanged. +- Update doc comments in `memory_store.rs` to drop + references to `#[async_trait::async_trait]` and instead describe the + `trait_variant` mechanism. +- Keep the existing Phase 1 test suite (`tests/phase_1/*.rs`) green. The + tests already use `Arc` and `Box`, which is + exactly the surface the rewrite is meant to preserve. + +### Out of scope -- moved to 0001c + +- Rewriting the `Embedder` / `LocalEmbedder` trait declaration -- handled by + `0001c-async-trait-sunset.md`. +- Dropping `#[async_trait::async_trait]` from + `crates/vestige-core/src/embedder/fastembed.rs`. +- Removing `async-trait = "0.1"` from `crates/vestige-core/Cargo.toml`. +- The hard `grep -rn 'async_trait' crates/` zero-match gate (async_trait is + still present in the embedder module after 0001a alone). + +### Out of scope + +- The SQLite file split (`sqlite.rs` -> `sqlite/` directory). That is the + sibling sub-plan `0001b-sqlite-split.md`. +- Any production-side refactor that switches `Arc` to + `Arc`. Production code keeps using the concrete alias. +- Adding `register_model` / `from_pool` / Postgres-specific variants of + `MemoryStoreError`. Those land with the Postgres backend in Phase 2. +- Removing the `pub type Storage = SqliteMemoryStore;` alias. + +--- + +## Prerequisites + +### Current state (verified) + +- `crates/vestige-core/Cargo.toml` already declares + `trait-variant = "0.1"` (line 117). `async-trait = "0.1"` (line 119) + remains in place for the duration of this sub-plan; `0001c` removes it. +- `crates/vestige-core/src/storage/memory_store.rs:194` declares the trait + with `#[async_trait::async_trait] pub trait MemoryStore: Send + Sync + + 'static`. +- `crates/vestige-core/src/storage/memory_store.rs:262` has + `pub use MemoryStore as LocalMemoryStore;`. +- `crates/vestige-core/src/storage/sqlite.rs:6274` declares + `#[async_trait::async_trait] impl + crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore`. +- Production call sites use `Arc` (the concrete + `SqliteMemoryStore` alias). Confirmed by grep: + `grep -rn "dyn MemoryStore\|dyn LocalMemoryStore" --include="*.rs"` + returns hits only in `tests/phase_1/*.rs`, in two comments inside + `memory_store.rs` and `sqlite.rs`, and in one test-only `&dyn MemoryStore` + cast inside `sqlite.rs::tests` (line 8669). +- Workspace-wide `async_trait` usages: only the trait declarations and + impl blocks in `memory_store.rs`, `sqlite.rs`, `embedder/mod.rs`, and + `embedder/fastembed.rs` (verified by + `grep -rn async_trait --include="*.rs"`). This sub-plan touches only the + first two; the embedder files are addressed by `0001c`. + +### Required crates + +| Crate | Version | Action | +|-------|---------|--------| +| `trait-variant` | `0.1` | Already declared. Verify present after edit. | +| `async-trait` | `0.1` | Unchanged for this sub-plan (still used by embedder impl). Removed by `0001c`. | +| `blake3`, `thiserror`, `chrono`, `uuid`, `serde`, `serde_json` | unchanged | unchanged | + +--- + +## Files Touched + +Grouped by category. Every change is listed; nothing else gets touched. + +### Trait declarations (vestige-core) + +| File | Lines (approx) | Change | +|------|----------------|--------| +| `crates/vestige-core/src/storage/memory_store.rs` | 183-262 | Replace `#[async_trait::async_trait]` block with `#[trait_variant::make(MemoryStore: Send)] pub trait LocalMemoryStore`. Drop the `pub use MemoryStore as LocalMemoryStore;` alias. Update doc comments. | + +### Impl blocks (vestige-core) + +| File | Lines (approx) | Change | +|------|----------------|--------| +| `crates/vestige-core/src/storage/sqlite.rs` | 6274 (impl block header only) | Delete the `#[async_trait::async_trait]` attribute. Keep the `impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { ... }` body verbatim. | + +### Cargo dependency cleanup + +None in this sub-plan. The `async-trait` crate stays declared in +`crates/vestige-core/Cargo.toml` because the embedder impl still uses it. +`0001c-async-trait-sunset.md` removes the dependency after the embedder +side is rewritten. + +### Cognitive module call sites + +No changes. The 29 cognitive modules under `crates/vestige-core/src/neuroscience/` +and `crates/vestige-core/src/advanced/` already operate on concrete +`SqliteMemoryStore` (via the `Storage` alias) or on plain data types +(`KnowledgeNode`, `Vec`, `ConnectionRecord`). Grep verified zero +production references to `dyn MemoryStore` or `dyn LocalMemoryStore`. + +### MCP call sites + +No changes. All ~30 `vestige-mcp/src/**.rs` files holding `Arc` +keep working because: +- `Storage` is still `pub type Storage = SqliteMemoryStore;` (unchanged). +- They call inherent methods on the concrete type, never via a trait object. +- `SqliteMemoryStore` implements `LocalMemoryStore`; `trait_variant` auto- + generates a blanket `impl MemoryStore for T`, + so the concrete type also satisfies `MemoryStore` for any future caller + that wants the trait-object form. + +### Test files (vestige-core integration tests) + +| File | Lines | Change | +|------|-------|--------| +| `tests/phase_1/trait_round_trip.rs` | 7-18, 134 | No change. Already uses `Arc` and `use vestige_core::storage::MemoryStore`. trait_variant emits a `MemoryStore` trait at the same path, so the imports resolve. | +| `tests/phase_1/send_bound_variant.rs` | 10-12, 36, 57 | No change. Already asserts the trait_variant Send invariant; the rewrite is what makes the doc comment on lines 3-4 actually true. | +| `tests/phase_1/cognitive_module_isolation.rs` | 11, 37, 76, 102, 115 | No change. | +| `tests/phase_1/embedding_model_registry.rs` | 10 | No change. | +| `tests/phase_1/domain_column_migration.rs` | 98 | No change. | +| `crates/vestige-core/src/storage/sqlite.rs::tests` | 8666-8675 | No change. The existing test casts `&s` to `&dyn MemoryStore` and calls trait methods through that vtable; trait_variant preserves this exact dyn-compatible surface. | + +### Documentation + +| File | Change | +|------|--------| +| `crates/vestige-core/src/storage/memory_store.rs` | Rewrite the trait-level doc comment (lines 185-193) to describe trait_variant rather than async_trait. | +| `CLAUDE.md` | No change. The repo-level architecture notes do not name the trait attribute. | + +--- + +## Trait Declaration Rewrite + +### Before (current state on `feat/storage-trait-phase1`) + +`crates/vestige-core/src/storage/memory_store.rs:183-262`: + +```rust +// ---------------------------------------------------------------------------- +// TRAIT +// ---------------------------------------------------------------------------- + +/// The single storage abstraction. +/// +/// `#[async_trait::async_trait]` makes every `async fn` return a +/// `Pin>`, which is required for `Arc` +/// to be movable across `tokio::spawn` boundaries. +/// +/// `LocalMemoryStore` is a type alias kept for source compatibility with code +/// that refers to the non-send variant. In Phase 1 both names refer to the same +/// (dyn-compatible, Send-safe) trait. +#[async_trait::async_trait] +pub trait MemoryStore: Send + Sync + 'static { + // --- Lifecycle --- + async fn init(&self) -> MemoryStoreResult<()>; + async fn health_check(&self) -> MemoryStoreResult; + + // ... 25 more async fn ... + + async fn vacuum(&self) -> MemoryStoreResult<()>; +} + +/// Type alias kept for source compatibility. Both names refer to the same +/// `async_trait`-annotated trait that is dyn-compatible and `Send + Sync`. +pub use MemoryStore as LocalMemoryStore; +``` + +### After + +`crates/vestige-core/src/storage/memory_store.rs:183-262`: + +```rust +// ---------------------------------------------------------------------------- +// TRAIT +// ---------------------------------------------------------------------------- + +/// The single storage abstraction. +/// +/// `LocalMemoryStore` is the source-of-truth trait. The +/// `#[trait_variant::make(MemoryStore: Send)]` attribute auto-generates a +/// `MemoryStore` trait whose returned futures are `Send`, so +/// `Arc` is movable across `tokio::spawn` boundaries while +/// `Arc` remains usable on single-threaded executors +/// and thread-local backends. +/// +/// Every method is native async-fn-in-trait (stable on MSRV 1.91); no +/// per-call heap allocation, no boxed futures. +#[trait_variant::make(MemoryStore: Send)] +pub trait LocalMemoryStore: Sync + 'static { + // --- Lifecycle --- + async fn init(&self) -> MemoryStoreResult<()>; + async fn health_check(&self) -> MemoryStoreResult; + + // --- Embedding model registry --- + async fn registered_model(&self) -> MemoryStoreResult>; + async fn register_model(&self, sig: &ModelSignature) -> MemoryStoreResult<()>; + + // --- CRUD --- + async fn insert(&self, record: &MemoryRecord) -> MemoryStoreResult; + async fn get(&self, id: Uuid) -> MemoryStoreResult>; + async fn update(&self, record: &MemoryRecord) -> MemoryStoreResult<()>; + async fn delete(&self, id: Uuid) -> MemoryStoreResult<()>; + + // --- Search --- + async fn search(&self, query: &SearchQuery) -> MemoryStoreResult>; + async fn fts_search(&self, text: &str, limit: usize) -> MemoryStoreResult>; + async fn vector_search( + &self, + embedding: &[f32], + limit: usize, + ) -> MemoryStoreResult>; + + // --- FSRS Scheduling --- + async fn get_scheduling( + &self, + memory_id: Uuid, + ) -> MemoryStoreResult>; + async fn update_scheduling(&self, state: &SchedulingState) -> MemoryStoreResult<()>; + async fn get_due_memories( + &self, + before: DateTime, + limit: usize, + ) -> MemoryStoreResult>; + + // --- Graph (spreading activation) --- + async fn add_edge(&self, edge: &MemoryEdge) -> MemoryStoreResult<()>; + async fn get_edges( + &self, + node_id: Uuid, + edge_type: Option<&str>, + ) -> MemoryStoreResult>; + async fn remove_edge(&self, source: Uuid, target: Uuid) -> MemoryStoreResult<()>; + async fn get_neighbors( + &self, + node_id: Uuid, + depth: usize, + ) -> MemoryStoreResult>; + + // --- Domains --- + async fn list_domains(&self) -> MemoryStoreResult>; + async fn get_domain(&self, id: &str) -> MemoryStoreResult>; + async fn upsert_domain(&self, domain: &Domain) -> MemoryStoreResult<()>; + async fn delete_domain(&self, id: &str) -> MemoryStoreResult<()>; + async fn classify(&self, embedding: &[f32]) -> MemoryStoreResult>; + + // --- Bulk / Maintenance --- + async fn count(&self) -> MemoryStoreResult; + async fn get_stats(&self) -> MemoryStoreResult; + async fn vacuum(&self) -> MemoryStoreResult<()>; +} +``` + +Notes: + +- The `pub use MemoryStore as LocalMemoryStore;` line on the current + `memory_store.rs:262` is **deleted** entirely. `MemoryStore` is now the + generated trait that `trait_variant::make` emits at the same module path; + `LocalMemoryStore` is the source-of-truth declaration. Both names export + from `storage/mod.rs` already (see lines 10-14 of that file). +- `Sync + 'static` on `LocalMemoryStore` (and no `Send` bound on the trait + itself) is correct: `Send` on the trait is what `trait_variant::make` + inserts when it emits the `MemoryStore` variant. The current trait carries + `Send + Sync + 'static`; the rewrite drops the `Send` bound from the + local variant. `Arc` is `Sync` but not `Send`; + `Arc` (the generated variant) is `Send + Sync`. +- `trait_variant` 0.1 does **not** require any attribute on the impl block. + The attribute lives only on the trait declaration. See the next section. + +The Embedder trait rewrite uses the identical `trait_variant::make` pattern +and is fully specified in `0001c-async-trait-sunset.md`. + +--- + +## Impl Block Migration + +`trait_variant` 0.1 attaches the attribute only to the trait declaration. +The impl side is plain `impl LocalMemoryStore for SqliteMemoryStore`; no +attribute on the impl, no `#[trait_variant::make(MemoryStore: Send)]` on the +impl block. The crate auto-generates the blanket +`impl MemoryStore for T`, so any concrete type +that implements `LocalMemoryStore` automatically also implements +`MemoryStore` provided it is `Send`. + +`SqliteMemoryStore` already is `Send + Sync` (it holds `Mutex` +fields whose `Mutex` is `std::sync::Mutex`, which is `Send + Sync` when its +guarded type is `Send`; `rusqlite::Connection` is `Send`). No new bound is +needed. + +### Before + +`crates/vestige-core/src/storage/sqlite.rs:6274`: + +```rust +#[async_trait::async_trait] +impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { + async fn init(&self) -> crate::storage::memory_store::MemoryStoreResult<()> { + // Migrations run in `new`; this is a no-op for the SQLite backend. + Ok(()) + } + + // ... ~2400 lines of method bodies, unchanged ... +} +``` + +### After + +`crates/vestige-core/src/storage/sqlite.rs:6274`: + +```rust +impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { + async fn init(&self) -> crate::storage::memory_store::MemoryStoreResult<()> { + // Migrations run in `new`; this is a no-op for the SQLite backend. + Ok(()) + } + + // ... ~2400 lines of method bodies, unchanged ... +} +``` + +Diff is exactly one removed line (the `#[async_trait::async_trait]` attribute). +Every method body, every `async fn` signature, every `use` statement inside +the impl block stays verbatim. No `Box::pin(async move { ... })`, no manual +`Pin>`, no `'async_trait` lifetime markers; native +async-fn-in-trait does this directly. + +The Embedder impl block rewrite follows the identical "remove one +`#[async_trait::async_trait]` attribute" pattern and is fully specified in +`0001c-async-trait-sunset.md`. + +### Why the impl block does not need an attribute + +`trait_variant::make` generates two things from the source trait: + +1. The source trait itself (`LocalMemoryStore`), with native async fns. +2. A second trait (`MemoryStore`) whose method signatures return + `impl Future + Send` instead of `impl Future`, + plus a blanket impl wiring concrete types through. + +Both are emitted at the macro-call site. `SqliteMemoryStore` only writes one +impl block (against `LocalMemoryStore`); the macro-generated blanket +guarantees `SqliteMemoryStore: MemoryStore` for free. The current `&dyn +MemoryStore` casts (sqlite.rs:8669; tests under tests/phase_1/) keep +type-checking unchanged. + +--- + +## Call Site Audit + +Verified via: + +```bash +grep -rn "dyn MemoryStore\|dyn LocalMemoryStore" --include="*.rs" \ + /home/delandtj/prppl/vestige-phase2/ +grep -rn "Arc\|Arc" --include="*.rs" \ + /home/delandtj/prppl/vestige-phase2/ +grep -rn "use.*MemoryStore;\|use.*LocalMemoryStore;" --include="*.rs" \ + /home/delandtj/prppl/vestige-phase2/ +``` + +### Files that reference the trait object form (`dyn MemoryStore` / `dyn LocalMemoryStore`) + +All test-only or doc-comment-only: + +| File | Line | Use | Required change | +|------|------|-----|-----------------| +| `tests/phase_1/trait_round_trip.rs` | 7-18 | `Arc` in `make_store()` and test bodies | None. `MemoryStore` is the generated Send variant; signature stays. | +| `tests/phase_1/trait_round_trip.rs` | 134 | `Arc` upcast inside a test body | None. | +| `tests/phase_1/send_bound_variant.rs` | 10-97 | `Arc` moved across `tokio::spawn` | None. This test becomes meaningfully correct only after the rewrite (currently it relies on async_trait boxing; after the rewrite it relies on trait_variant's Send variant -- same observable outcome). | +| `tests/phase_1/cognitive_module_isolation.rs` | 11, 33-115 | `Arc` passed into cognitive module-style closures | None. | +| `tests/phase_1/embedding_model_registry.rs` | 10 | `Arc` in `make_store()` | None. | +| `tests/phase_1/domain_column_migration.rs` | 98 | `Arc` cast inside a migration assertion | None. | +| `crates/vestige-core/src/storage/sqlite.rs` | 8666-8675 | `let dyn_s: &dyn MemoryStore = &s;` inside `mod tests` | None. The cast is testing that the dyn-vtable still resolves the trait methods correctly; after the rewrite it resolves through the `MemoryStore` trait that `trait_variant::make` emits at the same path. | +| `crates/vestige-core/src/storage/memory_store.rs` | 188 (doc) | Doc comment mentioning `Arc` | Replaced as part of the doc rewrite (see Trait Declaration section). | + +### Files that hold the concrete type (`Arc` / `Arc`) + +35 files, 116 hits. Every one of them keeps working unchanged because the +concrete `SqliteMemoryStore` type stays exactly as it is. Listed here for +completeness so a reviewer can confirm none of them needs an edit: + +``` +crates/vestige-core/src/storage/mod.rs (alias declaration) +crates/vestige-core/src/storage/sqlite.rs (impl block) +crates/vestige-mcp/src/server.rs (Arc in McpServer) +crates/vestige-mcp/src/cognitive.rs (hydrate(&Storage)) +crates/vestige-mcp/src/autopilot.rs +crates/vestige-mcp/src/protocol/http.rs +crates/vestige-mcp/src/dashboard/mod.rs +crates/vestige-mcp/src/dashboard/state.rs +crates/vestige-mcp/src/dashboard/handlers.rs +crates/vestige-mcp/src/resources/codebase.rs +crates/vestige-mcp/src/resources/memory.rs +crates/vestige-mcp/src/tools/changelog.rs +crates/vestige-mcp/src/tools/codebase_unified.rs +crates/vestige-mcp/src/tools/context.rs +crates/vestige-mcp/src/tools/contradictions.rs +crates/vestige-mcp/src/tools/cross_reference.rs +crates/vestige-mcp/src/tools/dedup.rs +crates/vestige-mcp/src/tools/dream.rs +crates/vestige-mcp/src/tools/explore.rs +crates/vestige-mcp/src/tools/feedback.rs +crates/vestige-mcp/src/tools/graph.rs +crates/vestige-mcp/src/tools/health.rs +crates/vestige-mcp/src/tools/importance.rs +crates/vestige-mcp/src/tools/intention_unified.rs +crates/vestige-mcp/src/tools/maintenance.rs +crates/vestige-mcp/src/tools/memory_states.rs +crates/vestige-mcp/src/tools/memory_unified.rs +crates/vestige-mcp/src/tools/predict.rs +crates/vestige-mcp/src/tools/restore.rs +crates/vestige-mcp/src/tools/review.rs +crates/vestige-mcp/src/tools/search_unified.rs +crates/vestige-mcp/src/tools/session_context.rs +crates/vestige-mcp/src/tools/smart_ingest.rs +crates/vestige-mcp/src/tools/suppress.rs +crates/vestige-mcp/src/tools/tagging.rs +crates/vestige-mcp/src/tools/timeline.rs +``` + +Each holds `Arc` and dispatches to inherent methods on +`SqliteMemoryStore`. None of them goes through a trait vtable. Required +change for every one of them: **none**. + +### Files that `use ...MemoryStore` from production code + +``` +grep -rn "use.*MemoryStore;\|use.*LocalMemoryStore;" --include="*.rs" \ + | grep -v "memory_store.rs\|sqlite.rs\|tests/phase_1" +``` + +returns nothing. Production code does not import the trait by name. + +### Conclusion + +The rewrite is a strictly local change to two source files +(`storage/memory_store.rs` and `storage/sqlite.rs`). Zero production call +sites need edits. The integration tests that consume `Arc` +keep their current form; the rewrite is what gives that signature its +no-box semantics on the storage side. The `Box` surface is +addressed by `0001c-async-trait-sunset.md`. + +--- + +## Commit Sequence + +Two commits, each green on `cargo test -p vestige-core --no-default-features` +and `cargo test -p vestige-core --features embeddings,vector-search`. + +### Commit 1: rewrite MemoryStore / LocalMemoryStore trait declaration + +- Touches: `crates/vestige-core/src/storage/memory_store.rs` only. +- Action: replace lines 183-262 per the "Trait Declaration Rewrite" section + above. Delete the `pub use MemoryStore as LocalMemoryStore;` line. +- Green after: `cargo check -p vestige-core` (the impl block in `sqlite.rs` + still has `#[async_trait::async_trait]` on it, but it still resolves + through the `LocalMemoryStore` trait which is now native-async; the + `async_trait` macro is harmless when applied to a trait that the impl + block targets by path, because the macro rewrites the impl's async fns + into boxed-future fns whose signatures still match the native-async + declarations after trait_variant lowering). If `cargo check` complains + here, fold commit 2 into commit 1. + + **Mitigation if check fails between commits 1 and 2:** combine the two + into a single commit. The split is offered for review convenience; the + build must be green after every commit lands. + +### Commit 2: drop `#[async_trait::async_trait]` from SqliteMemoryStore impl + +- Touches: `crates/vestige-core/src/storage/sqlite.rs` only. +- Action: delete line 6274 (`#[async_trait::async_trait]`). +- Green after: `cargo test -p vestige-core --features embeddings,vector-search`, + including all `trait_*` tests inside `sqlite.rs::tests` (lines 8643-8712) + and the trait-object cast on line 8669. + +### Combined alternative + +If the per-step split feels artificial, commits 1 and 2 can collapse into +a single commit covering both the trait rewrite and the impl-attribute +drop for `MemoryStore`. That is acceptable; the two-commit form is +preferred only because it lets a reviewer bisect trait-rewrite failures +separately from impl-rewrite failures. + +The Embedder / fastembed commits and the `async-trait` Cargo dependency +removal live in `0001c-async-trait-sunset.md`. + +--- + +## Verification + +Every command runs from the repo root unless noted otherwise. + +```bash +# 1. Vestige-core, default features (embeddings + vector-search). +cargo test -p vestige-core --features embeddings,vector-search + +# 2. Vestige-core, minimal features (no embeddings, no vector-search). +cargo test -p vestige-core --no-default-features + +# 3. Workspace build, release mode (catches any feature-gated regression +# in the vestige-mcp tools tree). +cargo build --workspace --release + +# 4. Whole-workspace test (vestige-mcp 406 tests + vestige-core 352 tests +# per the CLAUDE.md baseline). +cargo test --workspace + +# 5. Phase 1 integration tests (these are the trait-shape contract). +cargo test --test trait_round_trip --features embeddings,vector-search +cargo test --test send_bound_variant --features embeddings,vector-search +cargo test --test cognitive_module_isolation --features embeddings,vector-search +cargo test --test embedding_model_registry --features embeddings,vector-search +cargo test --test domain_column_migration --features embeddings,vector-search + +# 6. Clippy gate, deny warnings (matches Phase 1 PR policy of zero warnings). +cargo clippy --workspace --all-targets --features embeddings,vector-search -- -D warnings + +# 7. Storage-side dependency hygiene check (must return zero hits). +# Scoped to the storage module only -- the embedder module still uses +# async_trait until 0001c lands. +grep -rn "async_trait\|async-trait" crates/vestige-core/src/storage/ + +# 8. Confirm trait_variant attribute is in place on the storage trait +# (must return exactly one hit in memory_store.rs). +grep -rn "trait_variant::make" crates/vestige-core/src/storage/ +``` + +Expected outcomes: + +- Command 1: 352 tests pass (matches baseline). +- Command 2: smaller test count, all pass. +- Command 3: workspace compiles in release mode. +- Command 4: 758 total tests pass (matches CLAUDE.md baseline). +- Command 5: each phase_1 integration test binary runs green. The + `send_bound_variant::arc_dyn_memory_store_moves_across_tokio_tasks` test + is the canary; if `MemoryStore` lost its Send-bound future variant, this + fails to compile with "future cannot be sent between threads safely". +- Command 6: zero clippy warnings. The rewrite must not introduce a new + `clippy::needless_lifetimes` or `clippy::async_yields_async`. +- Command 7: empty output. async_trait is gone from the storage module. + The embedder module still contains async_trait; that is removed by + `0001c-async-trait-sunset.md`. +- Command 8: one hit, in `memory_store.rs`. + +--- + +## Acceptance Criteria + +A reviewer should be able to check every box: + +- [ ] `crates/vestige-core/src/storage/memory_store.rs` declares the trait + with `#[trait_variant::make(MemoryStore: Send)] pub trait + LocalMemoryStore: Sync + 'static`, no `async_trait` attribute, no + `Send` bound on `LocalMemoryStore` itself. +- [ ] `crates/vestige-core/src/storage/memory_store.rs` no longer contains + `pub use MemoryStore as LocalMemoryStore;`. +- [ ] `crates/vestige-core/src/storage/sqlite.rs:6274` is plain + `impl crate::storage::memory_store::LocalMemoryStore for + SqliteMemoryStore` -- no attribute on the impl block. +- [ ] `grep -rn "async_trait\|async-trait" crates/vestige-core/src/storage/` + returns zero hits. +- [ ] `grep -rn "trait_variant::make" crates/vestige-core/src/storage/` + returns exactly one hit (the storage trait in `memory_store.rs`). +- [ ] All 758 workspace tests pass (`cargo test --workspace`). +- [ ] Phase 1 integration tests pass with the trait-object surface + (`Arc`) intact. +- [ ] `cargo clippy --workspace --all-targets --features + embeddings,vector-search -- -D warnings` is clean. +- [ ] No production source file under `crates/vestige-mcp/` or + `crates/vestige-core/src/{neuroscience,advanced,consolidation,codebase, + memory,embeddings,embedder}/` was modified by this sub-plan. +- [ ] `Arc` still type-checks at every existing call site (verified + by the workspace test pass). +- [ ] Doc comments on the storage trait declaration describe + `trait_variant`, not `async_trait`. + +--- + +## Risks and Mitigations + +- **`trait_variant` 0.1 macro emits unexpected diagnostics on MSRV 1.91.** + Mitigation: the master Phase 1 plan already prescribed this exact pattern + (`#[trait_variant::make(MemoryStore: Send)] pub trait LocalMemoryStore: + Sync + 'static`, see plan `0001-...` line 274-275); the crate has been in + `vestige-core/Cargo.toml` since Phase 1 landed. If a diagnostic appears, + pin to the exact known-good version with `trait-variant = "=0.1.2"` and + open an upstream issue. +- **Native async-fn-in-trait makes the trait no longer dyn-compatible.** + Mitigation: `trait_variant::make` is specifically the workaround for this + -- it emits both the source trait (for static dispatch) and a Send-bound + variant whose returned futures use `Pin>` only at + the dyn boundary. `Arc` keeps working because the + generated `MemoryStore` trait is dyn-compatible by construction. Verified + by the existing `send_bound_variant::*` tests, which intentionally move + `Arc` across `tokio::spawn` from inside a + `multi_thread` runtime. +- **A cognitive module silently relied on the boxed-future return type.** + Mitigation: grep verified no cognitive module imports `MemoryStore` / + `LocalMemoryStore` or holds an `Arc` form; all of them use the + concrete `Storage` alias. There is no Send-ness expectation downstream to + break. +- **Future bodies inside the SQLite impl capture non-Send locals.** + Mitigation: every method body in `sqlite.rs:6274..` runs synchronous + rusqlite calls inside the same `async fn` frame; no `.await` points + exist inside the bodies that we ship today. The `Send` bound on the + generated `MemoryStore` variant is therefore satisfied automatically. + If a future change adds `.await` inside an impl method, the new + trait_variant surface will surface that as a compile error at the call + site, which is the correct outcome. +- **`async-trait` crate is left declared after this sub-plan.** + This is intentional: the embedder impl still depends on it. The + `0001c-async-trait-sunset.md` sub-plan removes the crate after the + embedder side is rewritten. Grep on the whole workspace returns only + the storage and embedder files; no downstream crate pulls `async-trait`. + +--- + +## Out-of-Band Notes + +- This sub-plan amends `feat/storage-trait-phase1` (tip 790c0c8). The + branch has not been opened upstream yet, so amending in place is safe; + no force-push to a public PR. +- The companion sub-plan `0001b-sqlite-split.md` lands after this one on + the same branch. The trait-rewrite landing first is intentional: the + SQLite split moves the impl block into `storage/sqlite/trait_impl.rs`, + and it is cleaner to move a small attribute-free impl than a + macro-decorated one. +- The companion sub-plan `0001c-async-trait-sunset.md` lands after this + one (order with `0001b` is independent) and finishes the + async_trait -> trait_variant transition for the Embedder trait, then + removes the `async-trait` crate dependency. +- After the Phase 1 amendment sub-plans (`0001a`, `0001b`, `0001c`) land, + the branch is reviewed and merged before Phase 2 sub-plans (`0002a-` + through `0002i-`) begin implementation. diff --git a/docs/plans/0001b-sqlite-split.md b/docs/plans/0001b-sqlite-split.md new file mode 100644 index 0000000..ce2590d --- /dev/null +++ b/docs/plans/0001b-sqlite-split.md @@ -0,0 +1,732 @@ +# Sub-Plan 0001b: Split sqlite.rs into a sqlite/ directory + +**Status**: Draft +**Branch**: `feat/storage-trait-phase1` (Phase 1 amendment, PR A) +**Depends on**: `0001a-trait-rewrite.md` (must land first; it carries the +`trait_variant`-generated trait declaration that `trait_impl.rs` matches) +**Related**: `docs/adr/0002-phase-2-execution.md` (D3, D6) + +--- + +## Context + +`crates/vestige-core/src/storage/sqlite.rs` is the single SQLite backend file +that Phase 1 inherited from pre-trait Vestige and then appended the +`LocalMemoryStore` trait impl block to. The file is 8713 lines as of +commit 790c0c8 on `feat/storage-trait-phase1`. ADR 0002 D3 decides to split +it into a `sqlite/` directory before Phase 2 lands `postgres/` as a peer. +Reasoning, in one paragraph: + +The Postgres backend is going in as a directory of seven small files +(`postgres/{mod,pool,migrations,registry,search,migrate_cli,reembed}.rs`). +If SQLite stays as one 8K-line file alongside that, the repo says "backends +look like one big file or seven small ones, pick a side", which forces +every future maintainer to re-litigate the layout. Splitting now -- as +**pure code motion**, no public-API change, no behavioural change, no +migration -- lets both backends look the same, keeps each surface mappable +in a single editor tab, and shrinks the diffs Phase 2 has to review. +This sub-plan is sized as one focused implementation session. + +The split is **private** to `storage/sqlite/`. Cognitive modules continue +to `use crate::storage::SqliteMemoryStore`; the existing re-exports in +`crates/vestige-core/src/storage/mod.rs` keep working without touching +any caller. Tests stay green commit-by-commit. + +This sub-plan depends on `0001a-trait-rewrite.md` landing first because +`sqlite/trait_impl.rs` is the file that picks up the new trait_variant +attribute. Doing the split first would force a second rewrite of +`trait_impl.rs` when the trait rewrite arrives. Order matters; this is +the cheap-to-respect ordering. + +--- + +## Target Layout + +Final directory after this sub-plan: + +``` +crates/vestige-core/src/storage/sqlite/ + mod.rs -- module root: SqliteMemoryStore struct, new(), + reader/writer locks, error types, shared helpers, + portable-sync-related types, record types + crud.rs -- ingest/smart_ingest/get/update/delete/purge/search-by-id + search.rs -- fts, semantic, hybrid, time-based queries + scheduling.rs -- FSRS state, decay, consolidation, review, promote/demote, + suppression, gc, retention, waking tags + graph.rs -- memory_connections (edges), subgraph, neighbors + domain.rs -- domains/domain_scores column readers, classify stub + (Phase 4 will expand this file) + registry.rs -- embedding_model table, enforce_model, register_model body + portable_sync.rs -- portable export/import/sync + merge helpers + trait_impl.rs -- impl LocalMemoryStore for SqliteMemoryStore +``` + +`storage/mod.rs` is unchanged in spirit: it still does `mod sqlite;` and +`pub use sqlite::{...};` -- the only difference is that `sqlite` is now a +directory module instead of a leaf file. The re-export list does not +change. + +--- + +## Current File Map (line numbers from commit 790c0c8) + +The current `sqlite.rs` is structurally: + +| Region | Lines | Contents | +|--------|-------|----------| +| Header | 1-43 | Imports, feature-gated imports | +| Error types | 45-89 | `StorageError`, `Result`, `SmartIngestResult`, `MergeWrite` | +| Portable sync types | 97-211 | `PortableSyncBackend` trait, `FilePortableSyncBackend` struct, `PortableSyncReport`, `PurgeReport` | +| Constants | 233-273 | `PORTABLE_TABLES`, `PORTABLE_USER_DATA_TABLES`, `PortableMergeState`, env constants | +| Struct decl | 287-301 | `SqliteMemoryStore` struct fields | +| Impl block 1 | 303-3740 | Constructor + bulk of native API | +| Record structs | 3747-3866 | `IntentionRecord`, `InsightRecord`, `ConnectionRecord`, `MemoryStateRecord`, `StateTransitionRecord`, `ConsolidationHistoryRecord`, `DreamHistoryRecord`, `Default for InsightRecord` | +| Impl block 2 | 3868-6133 | Intentions / Insights / Connections / States / History / Backup / Portable / GC / Subgraph | +| Impl block 3 | 6139-6272 | Trait-helper methods (`node_to_record`, `read_domain_columns`, `enforce_model`) | +| Trait impl | 6274-7110 | `impl LocalMemoryStore for SqliteMemoryStore` | +| Tests | 7112-8713 | `#[cfg(test)] mod tests`: native API tests + trait round-trip tests | + +--- + +## Mapping Table + +Every public method, every private helper, every struct, every test module +in the current file -- with the destination file in the new layout. Line +ranges cite the current `sqlite.rs` (commit 790c0c8 on +`feat/storage-trait-phase1`, viewed through the +`/home/delandtj/prppl/vestige-phase2` worktree). + +### Header and shared types (-> `sqlite/mod.rs`) + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| Module-level `use` imports | 1-43 | `sqlite/mod.rs` | Trimmed per-file in destination; what does not fit in `mod.rs` moves with its consumers | +| `StorageError` enum + `Result` alias | 49-71 | `sqlite/mod.rs` | Re-exported through `pub use` chain; called from every sub-module | +| `SmartIngestResult` struct | 73-89 | `sqlite/mod.rs` | Returned by `crud::smart_ingest`; defined here because other code depends on the type | +| `MergeWrite` enum | 91-95 | `sqlite/portable_sync.rs` | Only used by merge helpers | +| `PortableSyncBackend` trait | 97-109 | `sqlite/portable_sync.rs` | Public trait; re-exported through `mod.rs` | +| `FilePortableSyncBackend` struct + `impl` | 111-194 | `sqlite/portable_sync.rs` | | +| `PortableSyncReport` struct | 196-211 | `sqlite/portable_sync.rs` | | +| `PurgeReport` struct | 213-231 | `sqlite/crud.rs` | Returned by `purge_node` | +| `PORTABLE_TABLES` constant | 237-254 | `sqlite/portable_sync.rs` | | +| `PORTABLE_USER_DATA_TABLES` constant | 256-272 | `sqlite/portable_sync.rs` | | +| `PortableMergeState` struct | 274-277 | `sqlite/portable_sync.rs` | | +| `DATA_DIR_ENV` constant | 279 | `sqlite/mod.rs` | Read by constructor | +| `DATABASE_FILE` constant | 280 | `sqlite/mod.rs` | Read by constructor | +| `SqliteMemoryStore` struct decl | 282-301 | `sqlite/mod.rs` | All fields stay public-to-crate within the directory | + +### Constructor and config (-> `sqlite/mod.rs`) + +These are foundational; they live in `mod.rs` because every sub-module +calls them or operates on the struct they build. + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `fn data_dir_from_env` | 304-313 | `sqlite/mod.rs` | private helper | +| `fn expand_tilde` | 314-332 | `sqlite/mod.rs` | private helper | +| `fn prepare_data_dir` | 333-346 | `sqlite/mod.rs` | private helper | +| `pub fn db_path_for_data_dir` | 347-355 | `sqlite/mod.rs` | | +| `pub fn default_db_path` | 356-368 | `sqlite/mod.rs` | | +| `fn configure_connection` | 369-396 | `sqlite/mod.rs` | | +| `pub fn new` | 397-457 | `sqlite/mod.rs` | The constructor | +| `pub fn db_path` | 458-462 | `sqlite/mod.rs` | | +| `pub fn data_dir` | 463-467 | `sqlite/mod.rs` | | +| `pub fn sidecar_dir` | 468-473 | `sqlite/mod.rs` | | +| `fn load_embeddings_into_index` | 474-552 | `sqlite/mod.rs` | Called by `new`; touches vector index | + +### CRUD: ingest, get, update, delete, purge (-> `sqlite/crud.rs`) + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `pub fn ingest` | 553-643 | `sqlite/crud.rs` | | +| `pub fn smart_ingest` | 644-864 | `sqlite/crud.rs` | Calls vector search via `self.semantic_search`; cross-module call resolved by impl block being on the same struct | +| `pub fn get_node_embedding` (vector-search) | 865-887 | `sqlite/crud.rs` | embedding read for one node | +| `pub fn get_all_embeddings` (vector-search) | 888-914 | `sqlite/crud.rs` | | +| `pub fn get_node_embedding` (no vector-search stub) | 915-919 | `sqlite/crud.rs` | feature-gated alternative | +| `pub fn update_node_content` | 920-951 | `sqlite/crud.rs` | | +| `fn generate_embedding_for_node` | 952-999 | `sqlite/crud.rs` | private helper; only called by ingest and update_node_content | +| `pub fn get_node` | 1000-1011 | `sqlite/crud.rs` | | +| `fn parse_timestamp` | 1012-1027 | `sqlite/mod.rs` | **Shared helper**: row_to_node uses it, intention/insight rows also parse timestamps. Bump to `pub(super) fn` | +| `fn row_to_node` | 1028-1119 | `sqlite/mod.rs` | **Shared helper**: crud reads single nodes; search.rs builds node lists from rows; scheduling.rs returns nodes for review queue. Bump to `pub(super) fn`. Static method (no `&self`) so a free function in `mod.rs` is fine | +| `pub fn delete_node` | 1842-1869 | `sqlite/crud.rs` | | +| `pub fn purge_node` | 1870-1987 | `sqlite/crud.rs` | | +| `fn node_exists` | 1988-1996 | `sqlite/crud.rs` | static helper, called only by purge | +| `fn record_sync_tombstone` | 1997-2014 | `sqlite/crud.rs` | static helper, called by delete and purge | +| `pub fn get_all_nodes` | 2268-2291 | `sqlite/crud.rs` | bulk read | +| `pub fn get_nodes_by_type_and_tag` | 2292-2342 | `sqlite/crud.rs` | bulk read | + +### Search: fts, semantic, hybrid, temporal (-> `sqlite/search.rs`) + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `pub fn recall` | 1120-1147 | `sqlite/search.rs` | top-level recall path | +| `fn keyword_search` | 1148-1180 | `sqlite/search.rs` | private | +| `pub fn search` | 2015-2043 | `sqlite/search.rs` | | +| `pub fn search_terms` | 2044-2075 | `sqlite/search.rs` | | +| `pub fn concrete_search_filtered` | 2076-2172 | `sqlite/search.rs` | | +| `fn upsert_concrete_result` | 2173-2197 | `sqlite/search.rs` | static helper | +| `fn normalize_literal_query` | 2198-2210 | `sqlite/search.rs` | static helper | +| `fn escape_like` | 2211-2224 | `sqlite/search.rs` | static helper | +| `fn literal_match_score` | 2225-2248 | `sqlite/search.rs` | static helper | +| `fn node_matches_type_filters` | 2249-2267 | `sqlite/search.rs` | static helper | +| `pub fn is_embedding_ready` (both feature variants) | 2343-2354 | `sqlite/search.rs` | both versions move together | +| `pub fn init_embeddings` (both feature variants) | 2355-2367 | `sqlite/search.rs` | both versions move together | +| `fn get_query_embedding` | 2368-2400 | `sqlite/search.rs` | private; uses `query_cache` field | +| `pub fn semantic_search` | 2401-2434 | `sqlite/search.rs` | | +| `pub fn hybrid_search` (feature on) | 2435-2452 | `sqlite/search.rs` | | +| `pub fn hybrid_search_filtered` (feature on) | 2453-2581 | `sqlite/search.rs` | | +| `pub fn hybrid_search` (feature off) | 2582-2593 | `sqlite/search.rs` | feature-gated stub | +| `pub fn hybrid_search_filtered` (feature off) | 2594-2635 | `sqlite/search.rs` | feature-gated stub | +| `fn keyword_search_with_scores` | 2636-2726 | `sqlite/search.rs` | | +| `fn semantic_search_raw` | 2727-2765 | `sqlite/search.rs` | | +| `pub fn generate_embeddings` | 2766-2819 | `sqlite/search.rs` | populates embeddings post hoc | +| `fn embedding_regeneration_candidates` | 2820-2891 | `sqlite/search.rs` | called by generate_embeddings | +| `pub fn query_at_time` | 2892-2933 | `sqlite/search.rs` | temporal query | +| `pub fn query_time_range` | 2934-3005 | `sqlite/search.rs` | temporal query | +| `fn embedding_model_matches_active` (associated fn) | 5652-5671 | `sqlite/search.rs` | static helper for hybrid_search; promoted to `pub(super)` (test references it) | +| `fn embedding_model_supports_matryoshka` | 5672-5677 | `sqlite/search.rs` | static helper | +| `fn embedding_vector_for_active_model` | 5678-5697 | `sqlite/search.rs` | static helper | +| `fn active_embedding_model_like_pattern` | 5698-5713 | `sqlite/search.rs` | static helper | + +### Scheduling: FSRS, decay, consolidation, review, promote/demote, suppression, gc, retention (-> `sqlite/scheduling.rs`) + +This is the busiest destination file. The grouping rule is: anything that +touches FSRS scheduling fields (`stability`, `difficulty`, `retrievability`, +`reps`, `lapses`, `retention_strength`, `retrieval_strength`) or the +review/decay/consolidation/gc lifecycle lives here. + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `pub fn mark_reviewed` | 1181-1275 | `sqlite/scheduling.rs` | FSRS state mutation | +| `pub fn strengthen_on_access` | 1276-1344 | `sqlite/scheduling.rs` | | +| `pub fn strengthen_batch_on_access` | 1345-1357 | `sqlite/scheduling.rs` | | +| `pub fn mark_memory_useful` | 1358-1377 | `sqlite/scheduling.rs` | | +| `fn log_access` | 1378-1393 | `sqlite/scheduling.rs` | private | +| `pub fn promote_memory` | 1394-1425 | `sqlite/scheduling.rs` | | +| `pub fn demote_memory` | 1426-1472 | `sqlite/scheduling.rs` | | +| `pub fn suppress_memory` | 1473-1504 | `sqlite/scheduling.rs` | active forgetting | +| `pub fn reverse_suppression` | 1505-1552 | `sqlite/scheduling.rs` | | +| `pub fn count_suppressed` | 1553-1567 | `sqlite/scheduling.rs` | | +| `pub fn get_recently_suppressed` | 1568-1594 | `sqlite/scheduling.rs` | | +| `pub fn apply_rac1_cascade` | 1595-1641 | `sqlite/scheduling.rs` | active forgetting cascade | +| `pub fn run_rac1_cascade_sweep` | 1642-1657 | `sqlite/scheduling.rs` | | +| `pub fn get_review_queue` | 1658-1681 | `sqlite/scheduling.rs` | | +| `pub fn preview_review` | 1682-1712 | `sqlite/scheduling.rs` | | +| `pub fn get_stats` | 1713-1841 | `sqlite/scheduling.rs` | reports retention/lapses/review counts; lives here for symmetry with the FSRS reporters next door | +| `pub fn apply_decay` | 3006-3095 | `sqlite/scheduling.rs` | core decay loop | +| `fn get_fsrs_w20` | 3096-3119 | `sqlite/scheduling.rs` | | +| `pub fn run_consolidation` | 3120-3407 | `sqlite/scheduling.rs` | | +| `fn auto_dedup_consolidation` | 3408-3538 | `sqlite/scheduling.rs` | called by run_consolidation | +| `fn compute_act_r_activations` | 3539-3605 | `sqlite/scheduling.rs` | called by run_consolidation | +| `fn prune_access_log` | 3606-3620 | `sqlite/scheduling.rs` | called by run_consolidation | +| `fn optimize_w20_if_ready` | 3621-3720 | `sqlite/scheduling.rs` | called by run_consolidation | +| `fn generate_missing_embeddings` | 3721-3740 | `sqlite/scheduling.rs` | called by run_consolidation | +| `pub fn get_state_transitions` | 5714-5748 | `sqlite/scheduling.rs` | audit trail tied to scheduling state | +| `pub fn get_avg_retention` | 5780-5792 | `sqlite/scheduling.rs` | | +| `pub fn get_retention_distribution` | 5794-5825 | `sqlite/scheduling.rs` | | +| `pub fn get_retention_trend` | 5826-5858 | `sqlite/scheduling.rs` | | +| `pub fn save_retention_snapshot` | 5859-5878 | `sqlite/scheduling.rs` | | +| `pub fn count_memories_below_retention` | 5879-5892 | `sqlite/scheduling.rs` | | +| `pub fn gc_below_retention` | 5893-5936 | `sqlite/scheduling.rs` | | +| `pub fn auto_promote_frequent_access` | 5937-5985 | `sqlite/scheduling.rs` | | +| `pub fn set_waking_tag` | 5986-5998 | `sqlite/scheduling.rs` | waking SWR tagging | +| `pub fn clear_waking_tags` | 5999-6011 | `sqlite/scheduling.rs` | | +| `pub fn get_waking_tagged_memories` | 6012-6028 | `sqlite/scheduling.rs` | | +| `pub fn get_recent_state_transitions` | 6105-6132 | `sqlite/scheduling.rs` | | + +### Graph: edges (memory_connections), neighbors, subgraph (-> `sqlite/graph.rs`) + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `pub fn save_connection` | 4180-4202 | `sqlite/graph.rs` | | +| `pub fn get_connections_for_memory` | 4203-4220 | `sqlite/graph.rs` | | +| `pub fn get_all_connections` | 4221-4236 | `sqlite/graph.rs` | | +| `pub fn strengthen_connection` | 4237-4259 | `sqlite/graph.rs` | | +| `pub fn apply_connection_decay` | 4260-4272 | `sqlite/graph.rs` | | +| `pub fn prune_weak_connections` | 4273-4284 | `sqlite/graph.rs` | | +| `fn row_to_connection` | 4285-4305 | `sqlite/graph.rs` | private | +| `pub fn get_most_connected_memory` | 6029-6046 | `sqlite/graph.rs` | | +| `pub fn get_memory_subgraph` | 6048-6103 | `sqlite/graph.rs` | calls `get_connections_for_memory`, `get_node`, `get_all_connections` -- all resolvable through `self` | + +### Domain (-> `sqlite/domain.rs`) + +Phase 1 keeps domain logic to JSON-column reads + classify stub. Phase 4 +expands this file. Keeping the file in the split so Phase 4 has an +obvious place to add to. + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `fn read_domain_columns` | 6167-6196 | `sqlite/domain.rs` | private helper used by trait `get`. Bump to `pub(super)` | + +The trait methods `list_domains` / `get_domain` / `upsert_domain` / +`delete_domain` / `classify` live in `sqlite/trait_impl.rs`; they +delegate to thin helpers that, in Phase 1, are essentially noops or +JSON reads. Phase 4 will move the substance of those methods into +`sqlite/domain.rs` as real functions. + +### Registry: embedding_model table (-> `sqlite/registry.rs`) + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `fn enforce_model` | 6203-6272 | `sqlite/registry.rs` | private helper used by trait `insert` and `update`. Bump to `pub(super)` | + +The trait methods `registered_model` and `register_model` themselves +live in `sqlite/trait_impl.rs`. Phase 2's `postgres/registry.rs` will +mirror this layout. + +### Intentions, Insights, Memory States, Consolidation History, Dream History, Backup (-> `sqlite/mod.rs`) + +These were tacked onto `SqliteMemoryStore` over time as the cognitive +modules needed somewhere to persist their state. They are not part of the +trait surface, they are not naturally grouped with crud/search/scheduling, +and they are each fairly small and self-contained. They live in `mod.rs` +under labelled sections (one big impl block can carry them) rather than +inventing extra files like `intentions.rs` and `insights.rs`. Postgres +will mirror this once Phase 5 picks up the work; for now they have no +peer. + +Rationale: every one of these methods writes to a single table, the +bodies are short, and grouping them next to the constructor preserves +"open `mod.rs` to see the whole struct" as the navigation default. + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `IntentionRecord` struct | 3747-3766 | `sqlite/mod.rs` | re-exported through `storage/mod.rs` | +| `InsightRecord` struct + `Default` | 3767-3797 | `sqlite/mod.rs` | re-exported | +| `ConnectionRecord` struct | 3799-3809 | `sqlite/mod.rs` | re-exported; consumed by `graph.rs` | +| `MemoryStateRecord` struct | 3811-3821 | `sqlite/mod.rs` | | +| `StateTransitionRecord` struct | 3823-3833 | `sqlite/mod.rs` | re-exported | +| `ConsolidationHistoryRecord` struct | 3835-3846 | `sqlite/mod.rs` | | +| `DreamHistoryRecord` struct | 3848-3866 | `sqlite/mod.rs` | re-exported | +| `pub fn save_intention` etc. (intention block) | 3874-4058 | `sqlite/mod.rs` | one impl block, in-section labelled | +| `fn row_to_intention` | 4023-4058 | `sqlite/mod.rs` | private | +| insights block (`save_insight`, `get_insights`, etc.) | 4065-4174 | `sqlite/mod.rs` | | +| `fn row_to_insight` | 4153-4173 | `sqlite/mod.rs` | private | +| memory-state block | 4306-4459 | `sqlite/mod.rs` | | +| `fn row_to_memory_state` | 4431-4459 | `sqlite/mod.rs` | private | +| consolidation-history block | 4465-4540 | `sqlite/mod.rs` | | +| dream-history block | 4546-4638 | `sqlite/mod.rs` | | +| `pub fn count_memories_since` | 4639-4651 | `sqlite/mod.rs` | | +| `fn scan_last_backup_timestamp` | 4652-4682 | `sqlite/mod.rs` | | +| `pub fn last_backup_timestamp` | 4683-4688 | `sqlite/mod.rs` | | +| `pub fn get_last_backup_timestamp` (associated) | 4689-4696 | `sqlite/mod.rs` | | +| `pub fn backup_to` | 5749-5774 | `sqlite/mod.rs` | sqlite VACUUM INTO; called by backup tool | + +### Portable export/import/sync (-> `sqlite/portable_sync.rs`) + +This is the second-largest destination after `scheduling.rs` and the most +self-contained. + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `pub fn export_portable_archive` | 4699-4755 | `sqlite/portable_sync.rs` | | +| `pub fn export_portable_archive_to_path` | 4756-4806 | `sqlite/portable_sync.rs` | | +| `pub fn import_portable_archive` | 4807-4978 | `sqlite/portable_sync.rs` | | +| `pub fn import_portable_archive_from_path` | 4979-4996 | `sqlite/portable_sync.rs` | | +| `pub fn sync_portable_archive` (generic over backend) | 4997-5025 | `sqlite/portable_sync.rs` | | +| `pub fn sync_portable_archive_file` | 5026-5030 | `sqlite/portable_sync.rs` | | +| `fn merge_portable_table` | 5031-5073 | `sqlite/portable_sync.rs` | | +| `fn merge_knowledge_nodes` | 5074-5126 | `sqlite/portable_sync.rs` | | +| `fn merge_sync_tombstones` | 5127-5204 | `sqlite/portable_sync.rs` | | +| `fn merge_deletion_tombstones` | 5205-5245 | `sqlite/portable_sync.rs` | | +| `fn merge_keyed_table` | 5246-5281 | `sqlite/portable_sync.rs` | | +| `fn row_references_locally_newer_node` | 5282-5302 | `sqlite/portable_sync.rs` | | +| `fn merge_append_only_table` | 5303-5338 | `sqlite/portable_sync.rs` | | +| `fn parent_rows_exist` | 5339-5370 | `sqlite/portable_sync.rs` | | +| `fn insert_or_replace_row` | 5371-5386 | `sqlite/portable_sync.rs` | | +| `fn merge_key_columns` | 5387-5398 | `sqlite/portable_sync.rs` | | +| `fn upsert_row_with_columns` | 5399-5446 | `sqlite/portable_sync.rs` | | +| `fn insert_row_with_columns` | 5447-5469 | `sqlite/portable_sync.rs` | | +| `fn merge_row_exists` | 5470-5487 | `sqlite/portable_sync.rs` | | +| `fn row_exists_by_values` | 5488-5507 | `sqlite/portable_sync.rs` | | +| `fn row_values_for_columns` | 5508-5528 | `sqlite/portable_sync.rs` | | +| `fn portable_value` | 5529-5540 | `sqlite/portable_sync.rs` | | +| `fn portable_text` | 5541-5551 | `sqlite/portable_sync.rs` | | +| `fn portable_timestamp` | 5552-5559 | `sqlite/portable_sync.rs` | | +| `fn parse_rfc3339_opt` | 5560-5565 | `sqlite/portable_sync.rs` | | +| `fn tombstone_timestamp` | 5566-5580 | `sqlite/portable_sync.rs` | | +| `fn current_schema_version` | 5581-5589 | `sqlite/portable_sync.rs` | static helper | +| `fn ensure_portable_import_target_empty` | 5590-5604 | `sqlite/portable_sync.rs` | static helper | +| `fn table_exists` | 5605-5613 | `sqlite/portable_sync.rs` | static helper | +| `fn table_row_count` | 5614-5618 | `sqlite/portable_sync.rs` | static helper | +| `fn table_columns` | 5619-5630 | `sqlite/portable_sync.rs` | static helper | +| `fn portable_value_from_ref` | 5631-5646 | `sqlite/portable_sync.rs` | static helper | +| `fn quote_ident` | 5647-5651 | `sqlite/portable_sync.rs` | static helper | + +### Trait helpers and trait impl (-> `sqlite/trait_impl.rs`) + +| Item | Lines | Destination | Notes | +|------|-------|-------------|-------| +| `fn node_to_record` | 6142-6164 | `sqlite/trait_impl.rs` | associated fn used only by trait body; co-locate | +| `impl LocalMemoryStore for SqliteMemoryStore` block | 6274-7110 | `sqlite/trait_impl.rs` | full trait impl; attribute changes from `#[async_trait::async_trait]` to whatever 0001a settles on (`#[trait_variant::make(...)]` is on the trait declaration; the impl block carries no attribute under trait_variant) | + +### Tests + +The current `#[cfg(test)] mod tests` block at lines 7112-8713 contains +**two** distinct test families: + +1. **Native API tests** (7120-8198): unit tests against the legacy + `pub fn` surface (`test_ingest_and_get`, `test_search`, `test_review`, + `test_delete`, `test_dream_history_save_and_get_last`, + `test_portable_archive_exact_round_trip`, `test_keyword_search_*`, + `test_concrete_search_*`, `test_purge_*`, etc.). +2. **Trait round-trip tests** (8200-8712, after the + `// ===== Phase 1: LocalMemoryStore trait round-trip tests =====` + banner): `trait_init_is_idempotent`, `trait_register_model_*`, + `trait_insert_*`, `trait_get_*`, `trait_update_*`, `trait_delete_*`, + `trait_fts_search_*`, `trait_hybrid_search_*`, + `trait_scheduling_*`, `trait_add_edge_*`, `trait_get_edges_*`, + `trait_remove_edge_*`, `trait_get_neighbors_*`, `trait_list_domains_*`, + `trait_upsert_*`, `trait_classify_*`, `trait_count_*`, + `trait_get_stats_*`, `trait_vacuum_*`, + `trait_insert_refuses_dimension_mismatch`. + +See the Test Relocation section below for the resolution. + +--- + +## Visibility Changes + +The split moves items into sibling files inside one module. Helpers that +were `fn ...` (i.e. crate-private but file-private under the current +layout, since the file *is* the module) need their visibility lifted +just enough that sibling files can call them. The principle is: smallest +bump that makes the call site compile. + +`pub(super)` is sufficient for everything below; nothing needs +`pub(crate)`. The trait `LocalMemoryStore` exposure does not change -- +sub-modules call `self.method(...)` on `SqliteMemoryStore`, which +resolves through the impl blocks defined in their own files; visibility +is automatic at impl-block scope. + +Items that need a visibility bump (currently private fn, become +`pub(super) fn`): + +- `parse_timestamp` (1012): called by `row_to_node` and by intention / + insight row mappers. +- `row_to_node` (1028): called by `crud.rs`, `search.rs`, + `scheduling.rs`. Static associated fn. +- `read_domain_columns` (6167): called by `trait_impl.rs`. +- `enforce_model` (6203): called by `trait_impl.rs`. +- `embedding_model_matches_active` (5652): currently called by + `hybrid_search_filtered`; tests also reference it. Has to remain + `pub(super) fn` and be `pub` only if the existing test names reach it + through a re-export. (See Test Relocation.) +- `embedding_model_supports_matryoshka` (5672): private; only callers in + `search.rs` after the move; stays `fn` (no bump needed). +- `embedding_vector_for_active_model` (5678): same as the matches + function -- a test references it. Bump to `pub(super)`. +- `active_embedding_model_like_pattern` (5698): private; only used by + search; stays `fn`. +- `generate_embedding_for_node` (952): currently called by `ingest` and + `update_node_content`. Both move to `crud.rs`; stays `fn`. +- `get_query_embedding` (2368): only used inside `search.rs`; stays `fn`. +- `keyword_search_with_scores` (2636): only used inside `search.rs`; + stays `fn`. +- `semantic_search_raw` (2727): only used inside `search.rs`; stays `fn`. +- `embedding_regeneration_candidates` (2820): used by + `generate_embeddings`; both move to `search.rs`; stays `fn`. The + existing test (line 7167) references it through `storage.method()`, + which will continue to work because the test file can move with it. +- `log_access` (1378): private to `scheduling.rs`; stays `fn`. +- All the `auto_dedup_consolidation` / `compute_act_r_activations` / + `prune_access_log` / `optimize_w20_if_ready` / + `generate_missing_embeddings` helpers (3408-3740): private to + `scheduling.rs`; stays `fn`. +- `row_to_intention` / `row_to_insight` / `row_to_memory_state` / + `row_to_connection`: all stay private in their destination file (only + one caller each). +- All `merge_*` / `portable_*` / `parse_rfc3339_opt` / `quote_ident`: + private to `portable_sync.rs`; stays `fn`. +- `node_exists` (1988): private to `crud.rs`; stays `fn`. +- `record_sync_tombstone` (1997): private to `crud.rs`; stays `fn`. +- `get_fsrs_w20` (3096): private to `scheduling.rs`; stays `fn`. + +Items already `pub fn` (or `pub(crate) fn`) stay as they are -- no +visibility regression. + +Field visibility on `SqliteMemoryStore` itself: currently all fields are +private. The sub-modules access them via `self.field`. Because impl +blocks for `SqliteMemoryStore` are written in sibling files of the same +module, `self.field` reaches private fields without a visibility bump. +**No field visibility changes are required.** Confirm this during the +first motion commit; if Rust disagrees, mark the relevant fields +`pub(super)` and document in the commit message. + +--- + +## Public Re-exports + +`crates/vestige-core/src/storage/mod.rs` currently exports: + +```rust +mod memory_store; +mod migrations; +mod portable; +mod sqlite; + +pub use memory_store::{...}; +pub use migrations::MIGRATIONS; +pub use portable::{...}; +pub use sqlite::{ + ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, FilePortableSyncBackend, + InsightRecord, IntentionRecord, PortableSyncBackend, PortableSyncReport, Result, + SmartIngestResult, SqliteMemoryStore, StateTransitionRecord, StorageError, +}; + +pub type Storage = SqliteMemoryStore; +``` + +After the split, `mod sqlite;` resolves to the new directory module +(`storage/sqlite/mod.rs`). The `pub use sqlite::{...}` block resolves +against the items re-exported by `storage/sqlite/mod.rs`. + +`storage/sqlite/mod.rs` therefore needs the same names visible at its +top level. Add at the end of `mod.rs`: + +```rust +mod crud; +mod search; +mod scheduling; +mod graph; +mod domain; +mod registry; +mod portable_sync; +mod trait_impl; + +pub use portable_sync::{FilePortableSyncBackend, PortableSyncBackend, PortableSyncReport}; +// SqliteMemoryStore, StorageError, Result, SmartIngestResult, IntentionRecord, +// InsightRecord, ConnectionRecord, StateTransitionRecord, +// ConsolidationHistoryRecord, DreamHistoryRecord are defined in mod.rs itself, +// so they are already in scope and do not need a re-export. +``` + +The `crates/vestige-core/src/storage/mod.rs` file does not change. The +`pub type Storage = SqliteMemoryStore;` alias keeps working. + +If `cargo build` complains that `storage/mod.rs` cannot resolve a name +in its `pub use sqlite::{...}` block, the fix is to add the missing name +to `sqlite/mod.rs`'s re-export tail; no change to `storage/mod.rs`. + +--- + +## Test Relocation + +Two test families, two destinations. + +**Native API tests** (current lines 7120-8198) cover the legacy `pub fn` +surface. They live close to their subject: + +- Tests that touch the constructor, common helpers, and shared setup + (`create_test_storage`, `create_test_storage_at`, + `test_storage_creation`, `test_get_last_backup_timestamp_no_panic`) + move to `sqlite/mod.rs` in a `#[cfg(test)] mod tests` block. +- `test_ingest_and_get`, `test_delete`, + `test_purge_scrubs_insight_json_orphans_children_and_writes_tombstone` + go to `sqlite/crud.rs` as a `#[cfg(test)] mod tests` block. +- `test_search`, `test_keyword_search_with_include_types`, + `test_keyword_search_with_exclude_types`, + `test_include_types_takes_precedence_over_exclude`, + `test_type_filter_with_no_matches_returns_empty`, + `test_hybrid_search_backward_compat`, + `test_concrete_search_literal_identifier_lands_first`, + `test_embedding_model_family_matching`, + `test_embedding_regeneration_candidates_include_entire_mismatched_corpus` + go to `sqlite/search.rs`. +- `test_review` goes to `sqlite/scheduling.rs`. +- `test_dream_history_save_and_get_last`, `test_dream_history_empty`, + `test_count_memories_since` go to `sqlite/mod.rs` (history tables live + there). +- All `test_portable_*` go to `sqlite/portable_sync.rs`. +- `test_file_portable_sync_round_trips_between_devices` goes to + `sqlite/portable_sync.rs`. + +**Trait round-trip tests** (current lines 8200-8712) test the +`LocalMemoryStore` trait impl. Two viable layouts: + +A. Co-locate with the impl in `sqlite/trait_impl.rs` (one big + `#[cfg(test)] mod trait_tests`). +B. Keep them as a single `tests.rs` file in the sqlite directory. + +**Decision: A.** Co-locate. The trait round-trip tests are explicitly +testing the `impl LocalMemoryStore for SqliteMemoryStore` block; +co-location means a reader who edits the trait impl sees its tests in +the same file. Option B would mean two places to look every time a +trait method changes shape. For an 8K-line collapse the tradeoff +favours co-location. + +Concretely: `sqlite/trait_impl.rs` ends with a +`#[cfg(test)] mod trait_tests { ... }` block that contains all 30+ +`trait_*` tests, plus the shared `make_record`, `rt`, and small helpers +defined inside the current test mod for trait tests (lines 8208-8226). + +--- + +## Commit Sequence + +Each commit moves one logical group. After each commit: + +``` +cargo build -p vestige-core +cargo test -p vestige-core +cargo clippy -p vestige-core -- -D warnings +``` + +must pass. Order is chosen so that each move is small, the next move +does not depend on the previous having grown surprising visibility, and +the largest / riskiest move (the trait impl, with the new +trait_variant attribute) lands last. + +| # | Commit | What moves | Tests touched | +|---|--------|-----------|----------------| +| 1 | `refactor(sqlite): scaffold sqlite/ directory` | Convert `sqlite.rs` -> `sqlite/mod.rs` verbatim (rename + create empty sibling files `crud.rs`, `search.rs`, `scheduling.rs`, `graph.rs`, `domain.rs`, `registry.rs`, `portable_sync.rs`, `trait_impl.rs` each with `use super::*;`). At this point `mod.rs` declares the new modules but they are empty. | None move. Build proves the rename works. | +| 2 | `refactor(sqlite): split out portable sync` | Move all `merge_*`, `portable_*`, `export_*`, `import_*`, `sync_*` items + `MergeWrite`, `PortableSyncBackend`, `FilePortableSyncBackend`, `PortableSyncReport`, `PortableMergeState`, `PORTABLE_TABLES`, `PORTABLE_USER_DATA_TABLES`, helper statics into `sqlite/portable_sync.rs`. Add `pub use portable_sync::{...}` in `mod.rs` for the public types. | `test_portable_*` and `test_file_portable_sync_round_trips_between_devices` move too. | +| 3 | `refactor(sqlite): split out graph / connections` | Move `save_connection`, `get_connections_for_memory`, `get_all_connections`, `strengthen_connection`, `apply_connection_decay`, `prune_weak_connections`, `row_to_connection`, `get_most_connected_memory`, `get_memory_subgraph` to `sqlite/graph.rs`. | None move (no native graph tests; trait edge tests still in trait_tests). | +| 4 | `refactor(sqlite): split out scheduling / fsrs / consolidation` | Move all items listed in the Scheduling row to `sqlite/scheduling.rs`. | `test_review` moves. | +| 5 | `refactor(sqlite): split out search / fts / semantic / hybrid` | Move all items listed in the Search row to `sqlite/search.rs`. Add `pub(super)` to the four `embedding_model_*` helpers that tests reference. | `test_search`, `test_keyword_search_*`, `test_include_types_*`, `test_type_filter_*`, `test_hybrid_search_*`, `test_concrete_search_*`, `test_embedding_model_family_matching`, `test_embedding_regeneration_candidates_include_entire_mismatched_corpus` move. | +| 6 | `refactor(sqlite): split out crud / ingest / get / update / delete / purge` | Move `ingest`, `smart_ingest`, `get_node`, `update_node_content`, `delete_node`, `purge_node`, `get_all_nodes`, `get_nodes_by_type_and_tag`, `node_exists`, `record_sync_tombstone`, `generate_embedding_for_node`, `get_node_embedding`, `get_all_embeddings`, `PurgeReport` to `sqlite/crud.rs`. Bump `row_to_node` and `parse_timestamp` to `pub(super) fn` in `mod.rs`. | `test_ingest_and_get`, `test_delete`, `test_purge_scrubs_insight_json_orphans_children_and_writes_tombstone` move. | +| 7 | `refactor(sqlite): split out registry helper` | Move `enforce_model` to `sqlite/registry.rs`, bumped to `pub(super)`. | None move. | +| 8 | `refactor(sqlite): split out domain helper` | Move `read_domain_columns` to `sqlite/domain.rs`, bumped to `pub(super)`. | None move. | +| 9 | `refactor(sqlite): split out trait impl + tests` | Move `node_to_record` and the full `impl LocalMemoryStore for SqliteMemoryStore` block to `sqlite/trait_impl.rs`. Move the entire trait round-trip test module (lines 8200-8712, including `make_record` and `rt` helpers) to a `#[cfg(test)] mod trait_tests` block at the bottom of `trait_impl.rs`. This is the commit where the trait_variant attribute (from sub-plan 0001a) is observed: the impl block on `SqliteMemoryStore` uses whatever syntax the rewritten trait expects (no `#[async_trait::async_trait]`). | All `trait_*` tests move. | + +Commit 1 is the only commit that creates new files; the rest move +existing code into them. Reviewers can bisect through this list to +find any silent-semantic change. + +--- + +## Verification + +Run after every commit. All three must pass before pushing: + +``` +cargo build -p vestige-core +cargo test -p vestige-core +cargo clippy -p vestige-core -- -D warnings +``` + +The Phase 1 amendment branch must also build with the no-default-features +configuration that the release binary uses for the alternative feature +set: + +``` +cargo build -p vestige-core --no-default-features +cargo test -p vestige-core --no-default-features +``` + +Some of the methods being moved (`get_node_embedding`, `is_embedding_ready`, +`init_embeddings`, the feature-on/feature-off `hybrid_search` pair) have +two definitions guarded by feature flags. The split must preserve both +copies in the same destination file with their existing `#[cfg(...)]` +attributes; the no-default-features build confirms. + +After the last commit, run the workspace-wide check that Phase 1 promised: + +``` +cargo build --workspace +cargo test --workspace +``` + +This catches downstream consumers (`vestige-mcp`, `vestige`, +`vestige-restore`) that might depend on a specific module path (they +should not -- they import from `crate::storage::SqliteMemoryStore` and +the re-exports in `storage/mod.rs` -- but the workspace build is the +authoritative confirmation). + +--- + +## Acceptance Criteria + +1. `crates/vestige-core/src/storage/sqlite.rs` no longer exists. In its + place is `crates/vestige-core/src/storage/sqlite/` with the eight + files listed in the Target Layout section, each below 2000 lines. +2. `crates/vestige-core/src/storage/mod.rs` is unchanged (or + functionally unchanged -- the `pub use sqlite::{...}` block contains + the same identifiers in the same order). +3. Every cognitive module and binary in the workspace + (`vestige-core`, `vestige-mcp`, `vestige`, `vestige-restore`) + compiles with no source edits other than the ones in + `crates/vestige-core/src/storage/sqlite/`. +4. `cargo build -p vestige-core`, + `cargo test -p vestige-core`, + `cargo clippy -p vestige-core -- -D warnings`, + `cargo build -p vestige-core --no-default-features`, and + `cargo test -p vestige-core --no-default-features` all pass at the + end of every commit in the sequence (bisectability). +5. `cargo test --workspace` matches the Phase 1 baseline test count + (758 tests, of which 352 are in `vestige-core`). No new tests are + added by this sub-plan; no existing test is renamed or deleted. +6. The on-disk SQLite schema is unchanged. A live database created on + the pre-split build opens cleanly on the post-split build and round + trips a memory. +7. `git log --follow` works for at least one method in each destination + file (i.e. `git mv` was used where the line range constitutes most + of the file content of the destination, otherwise a `git log -p` + on the new file shows the history before the rename through the + block-move detection that recent `git log` versions support). +8. No public symbol disappears from `crate::storage::*`. A reviewer can + verify with: + ``` + cargo doc -p vestige-core --no-deps + ``` + before and after the split, and `diff` the generated + `target/doc/vestige_core/storage/index.html` lists. + +--- + +## Non-Goals (explicit) + +- No public API change. The trait surface (`LocalMemoryStore`, + `MemoryStore`), the legacy `pub fn` surface on `SqliteMemoryStore`, + the re-exports through `storage/mod.rs`, and the `pub type Storage = + SqliteMemoryStore;` alias are all preserved. +- No behavioural change. No SQL is rewritten, no FSRS parameter is + retuned, no embedding model is touched, no migration is added. +- No new tests. Tests move with their subject; no new tests are + written. +- No clippy fix-ups that pre-date this sub-plan. If `cargo clippy + -- -D warnings` was passing before the split, it must continue to + pass; if it was not passing, the failures stay where they are and + are addressed in a separate commit (out of scope here). +- No removal of the `pub type Storage = SqliteMemoryStore;` BC alias. + That happens at the end of Phase 4 per ADR 0001. +- No reorganisation of `storage/memory_store.rs`, + `storage/migrations.rs`, or `storage/portable.rs`. Those files are + out of scope; the split is private to `storage/sqlite/`. + +--- + +## Risks and Mitigations + +| Risk | Mitigation | +|------|------------| +| Silent semantic change introduced by a motion commit | Per-commit `cargo test -p vestige-core` keeps the bisect window to a single commit. Reviewer bisects with `cargo test -p vestige-core` as the witness. | +| Sibling-file `self.field` accesses fail because Rust enforces module visibility on tuple-struct or named fields | Confirmed: associated impl blocks in sibling files of the same `mod sqlite` reach private fields. If the compiler disagrees, bump the affected fields to `pub(super)` in `mod.rs` and note the bump in the commit message. | +| Test-only helpers (`create_test_storage`, `make_record`, `rt`) get duplicated across test modules | Lift them once into a `#[cfg(test)] mod test_support { ... }` sub-module in `sqlite/mod.rs` and re-export with `pub(super) use`. Do this in commit 1 (scaffold), not later. | +| `#[cfg(all(feature = "embeddings", feature = "vector-search"))]` items end up in `mod.rs` where they pollute the shared layer | Audit during commit 5 (search split); items behind both feature flags belong in `search.rs`. The `query_cache` field stays in `mod.rs` because the struct definition is there; the field declaration is feature-gated and that gate moves with the struct as-is. | +| `git log --follow` blame chains break on the moved methods | Use `git mv sqlite.rs sqlite/mod.rs` in commit 1 so commit 1 looks like a rename (`git log --follow` keeps working). Subsequent commits are content moves inside the module; modern `git log --follow -M -C` heuristics still trace the lines. Reviewers who need pristine blame should bisect to before commit 1. | +| Sub-plan 0001a (trait rewrite) has not landed when this work starts | Block: do not start commits 1-9 until 0001a is on the same branch (`feat/storage-trait-phase1`) and tests pass. `trait_impl.rs` lands the new attribute in commit 9; if 0001a is not in, commit 9 fails. | + +--- + +## Self-Contained Brief (for /goal) + +A fresh Claude Code session can execute this sub-plan by: + +1. Reading this file end to end. +2. Reading `crates/vestige-core/src/storage/sqlite.rs` (the file to be + split) in full, using line ranges from the Mapping Table to confirm + the current shape matches the brief. +3. Reading `crates/vestige-core/src/storage/mod.rs` (the re-export + surface that must continue to work). +4. Reading `crates/vestige-core/src/storage/memory_store.rs` (the + trait surface that `trait_impl.rs` implements). +5. Confirming sub-plan 0001a has landed on the current branch by + checking that `memory_store.rs` no longer carries + `#[async_trait::async_trait]` on the trait declaration. +6. Working through the Commit Sequence in order, running the + Verification commands after each commit. + +The session does not need to read ADR 0002 or the master Phase 2 plan +to do this work. The split is purely mechanical relative to the +mapping table above. diff --git a/docs/plans/0001c-async-trait-sunset.md b/docs/plans/0001c-async-trait-sunset.md new file mode 100644 index 0000000..f9d8938 --- /dev/null +++ b/docs/plans/0001c-async-trait-sunset.md @@ -0,0 +1,847 @@ +# Sub-Plan 0001c: Sunset the `async-trait` crate dependency + +**Status**: Draft +**Branch**: `feat/storage-trait-phase1` (Phase 1 amendment, PR A) +**Depends on**: +- `0001a-trait-rewrite.md` (rewrites `MemoryStore` / `LocalMemoryStore` and + the SQLite impl; lands first) +- `0001b-sqlite-split.md` (moves `sqlite.rs` into `sqlite/`; lands second) + +**Related**: `docs/adr/0002-phase-2-execution.md` (decision D1 closing line: +"async-trait dependency stays in Cargo.toml only if other code uses it; +otherwise removed"). This sub-plan operationalises the removal. + +--- + +## Context + +This is the third and final Phase 1 amendment sub-plan. Sub-plan `0001a` +rewrote `MemoryStore` / `LocalMemoryStore` using +`#[trait_variant::make(MemoryStore: Send)]` and dropped the +`#[async_trait::async_trait]` attribute from the SQLite impl block. +Sub-plan `0001b` then split `sqlite.rs` into a `sqlite/` directory; the +trait impl now lives in `sqlite/trait_impl.rs`. After `0001a` lands, the +only remaining `async_trait` usage in the workspace is the embedder pair +(`embedder/mod.rs` declares the trait; `embedder/fastembed.rs` implements +it). This sub-plan rewrites those two files following the exact pattern +from `0001a`, then removes `async-trait = "0.1"` from +`crates/vestige-core/Cargo.toml`. End state: zero `async_trait` references +anywhere under `crates/`, zero direct dependency on the `async-trait` +crate, workspace tests and clippy green. + +The rewrite is mechanically tiny -- one trait declaration, one impl block, +one Cargo.toml line -- but it is gated behind `0001a` and `0001b` so the +trait-rewrite pattern is already settled and so the SQLite trait impl +attribute has already been dropped. Doing the embedder rewrite without +that pre-work would leave the `async-trait` dep behind for the SQLite +side and force the Cargo.toml deletion into a separate commit later. + +--- + +## Scope + +### In scope + +- Rewrite `LocalEmbedder` declaration in + `crates/vestige-core/src/embedder/mod.rs` to use + `#[trait_variant::make(Embedder: Send)] pub trait LocalEmbedder`. +- Delete the `pub use LocalEmbedder as Embedder;` alias from the same file. + The `Embedder` symbol becomes the trait that `trait_variant::make` emits + at the same module path, so the existing + `pub use embedder::{Embedder, ..., LocalEmbedder};` line in + `crates/vestige-core/src/lib.rs:167` keeps working untouched. +- Drop the `#[async_trait::async_trait]` attribute from the + `FastembedEmbedder` impl block in + `crates/vestige-core/src/embedder/fastembed.rs`. +- Update doc comments on the trait declaration to describe + `trait_variant` rather than `async_trait`. +- Remove `async-trait = "0.1"` from + `crates/vestige-core/Cargo.toml` (line 119 area). Use + `cargo rm async-trait` from inside the crate directory. +- Verify with `grep -rn "async_trait" crates/` returning zero hits. + +### Out of scope + +- Any change to the `MemoryStore` trait or `SqliteMemoryStore` impl; + those were handled by `0001a`. +- Any file moves in `embedder/` (no parallel of `0001b` is required; + `embedder/` already has the `mod.rs` + `fastembed.rs` shape). +- Touching `crates/vestige-mcp/` or any cognitive module. None of them + hold `Arc` or `Box` in production. +- Renaming the `Embedder` / `LocalEmbedder` symbols or changing the + re-exports in `crates/vestige-core/src/lib.rs`. + +--- + +## Prerequisites + +### State assumed at start + +- `0001a` is merged onto the branch. After `0001a`: + - `crates/vestige-core/src/storage/memory_store.rs` declares + `#[trait_variant::make(MemoryStore: Send)] pub trait LocalMemoryStore`. + - The SQLite impl block has no `#[async_trait::async_trait]` attribute. + - `grep -rn async_trait crates/` returns exactly three hits, all in + `crates/vestige-core/src/embedder/` (two in `mod.rs`, one in + `fastembed.rs`), and one Cargo.toml hit. +- `0001b` is merged onto the branch. After `0001b`: + - `crates/vestige-core/src/storage/sqlite.rs` no longer exists as a + single file; the impl lives in `crates/vestige-core/src/storage/sqlite/trait_impl.rs`. + - The embedder files are untouched. + +### Required crates + +| Crate | Version | Action | +|----------------|---------|-----------------------------------------------------------------| +| `trait-variant`| `0.1` | Already declared (line 117 of Cargo.toml). Verify present. | +| `async-trait` | `0.1` | Remove. Only the two embedder files still use it after `0001a`. | + +### Workspace-wide audit before starting + +Run from `/home/delandtj/prppl/vestige-phase2/` (or the equivalent +worktree where this sub-plan executes): + +```bash +grep -rn "async_trait\|async-trait" crates/ tests/ +``` + +Expected hits before this sub-plan starts (after `0001a` + `0001b`): + +``` +crates/vestige-core/Cargo.toml:119:async-trait = "0.1" +crates/vestige-core/src/embedder/mod.rs:24:/// `#[async_trait::async_trait]` makes every `async fn` return a +crates/vestige-core/src/embedder/mod.rs:27:#[async_trait::async_trait] +crates/vestige-core/src/embedder/mod.rs:56:/// Both names refer to the same `async_trait`-annotated trait. +crates/vestige-core/src/embedder/fastembed.rs:44:#[async_trait::async_trait] +``` + +Five hits across two source files and one Cargo.toml. After this sub-plan, +the same grep must return zero hits. + +```bash +grep -rn "async-trait\|async_trait" --include="Cargo.toml" crates/ +``` + +Expected: exactly one hit (`crates/vestige-core/Cargo.toml:119`). No other +workspace crate declares `async-trait` as a direct dependency. This is +the precondition that lets us delete the line cleanly. + +--- + +## Files Touched + +### Trait declaration (vestige-core) + +| File | Lines (approx) | Change | +|-------------------------------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `crates/vestige-core/src/embedder/mod.rs` | 21-57 | Replace `#[async_trait::async_trait] pub trait LocalEmbedder: Send + Sync + 'static` with `#[trait_variant::make(Embedder: Send)] pub trait LocalEmbedder: Sync + 'static`. Delete the `pub use LocalEmbedder as Embedder;` alias on line 57. Update doc comments (lines 21-26, 55-56). | + +### Impl block (vestige-core) + +| File | Lines (approx) | Change | +|-------------------------------------------------|----------------|--------------------------------------------------------------------------------------------------------------| +| `crates/vestige-core/src/embedder/fastembed.rs` | 44 | Delete the `#[async_trait::async_trait]` attribute. Keep the `impl LocalEmbedder for FastembedEmbedder { ... }` body verbatim. No `Box::pin`, no `'async_trait` lifetimes, no manual `Pin>`. | + +### Other Embedder impls + +None. `grep -rn "impl.*LocalEmbedder\|impl.*Embedder for" crates/ tests/` +returns exactly one production hit: +`crates/vestige-core/src/embedder/fastembed.rs:45: impl LocalEmbedder for FastembedEmbedder`. +There is no test mock implementing `Embedder` in the test tree (the only +test that touches the trait, `tests/phase_1/embedder_trait.rs`, uses the +concrete `FastembedEmbedder` boxed as `Box`). + +### Call sites (production) + +Verified by: + +```bash +grep -rn "dyn Embedder\|dyn LocalEmbedder" crates/ tests/ --include="*.rs" +grep -rn "Box\|Arc" crates/ tests/ --include="*.rs" +grep -rn "use.*Embedder" crates/ tests/ --include="*.rs" +``` + +Production call sites that may need verification (and the expected change +for each, even though we have already verified that none need an edit): + +| File | Use | Required change | +|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|-----------------| +| `crates/vestige-core/src/lib.rs:167` | `pub use embedder::{Embedder, EmbedderError, EmbedderResult, FastembedEmbedder, LocalEmbedder};` | None. Both names still exist at `crate::embedder::*` after the rewrite; `Embedder` is now the `trait_variant`-generated trait, `LocalEmbedder` is the source-of-truth trait. The re-export keeps resolving. | +| `crates/vestige-core/src/embedder/fastembed.rs:7` | `use super::{EmbedderError, EmbedderResult, LocalEmbedder};` | None. `LocalEmbedder` is still the source-of-truth trait name. | +| `crates/vestige-core/src/embedder/mod.rs:5` | `pub use fastembed::FastembedEmbedder;` | None. Concrete type, untouched. | +| `crates/vestige-mcp/src/**` | No file imports `Embedder` or `LocalEmbedder` by name; none hold `Arc` or `Box`. | None. Verified by grep returning empty for `dyn Embedder` and `dyn LocalEmbedder` under `crates/vestige-mcp/`. | +| Cognitive modules under `crates/vestige-core/src/advanced/` and `crates/vestige-core/src/neuroscience/` | No file imports `Embedder` or `LocalEmbedder` by name. `advanced/adaptive_embedding.rs` defines its own unrelated `AdaptiveEmbedder` struct. | None. The name collision is purely surface-level; the two types live in different modules and never resolve to each other. | +| `crates/vestige-core/src/embeddings/**` | No file imports `Embedder` or `LocalEmbedder`. The `EmbeddingService` struct is what `FastembedEmbedder` wraps internally. | None. | + +The production audit returns zero files that need editing. + +### Call sites (tests) + +| File | Lines | Use | Required change | +|------------------------------------------------------------|-------|--------------------------------------------------------------------|-----------------| +| `tests/phase_1/embedder_trait.rs` | 3, 19 | `use vestige_core::embedder::{Embedder, FastembedEmbedder};`
`let e: Box = Box::new(FastembedEmbedder::new());` | None. `Embedder` is the `trait_variant`-generated Send variant; `Box` keeps compiling. `FastembedEmbedder` implements `LocalEmbedder`; the blanket `impl Embedder for T` that `trait_variant::make` emits gives the boxing for free. | + +The test audit returns zero files that need editing. + +### Cargo dependency cleanup + +| File | Lines | Change | +|-------------------------------------|-----------|-----------------------------------------------------------------------------------------------------| +| `crates/vestige-core/Cargo.toml` | 119 | Remove `async-trait = "0.1"`. Run `cargo rm async-trait` from inside `crates/vestige-core/` so `Cargo.lock` updates atomically with the manifest. | + +### Documentation + +| File | Change | +|---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `crates/vestige-core/src/embedder/mod.rs` | Rewrite the trait-level doc comment (lines 21-26) and the `pub use` doc comment (lines 55-56) to describe `trait_variant`, not `async_trait`. See "Trait declaration rewrite" below for the exact new text. | +| `CLAUDE.md` | No change. The repo-level architecture notes do not name the trait attribute. | + +--- + +## Trait Declaration Rewrite + +### Before (state after `0001a` and `0001b` land) + +`crates/vestige-core/src/embedder/mod.rs:1-58`: + +```rust +//! Text-to-vector encoding trait. Pluggable per-install. + +mod fastembed; + +pub use fastembed::FastembedEmbedder; + +/// Error returned by every `Embedder` method. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum EmbedderError { + #[error("embedder initialization failed: {0}")] + Init(String), + #[error("embedding generation failed: {0}")] + EmbedFailed(String), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +pub type EmbedderResult = std::result::Result; + +/// Pluggable embedder. The storage layer NEVER calls fastembed directly; +/// callers compute vectors via this trait and pass them into `MemoryStore`. +/// +/// `#[async_trait::async_trait]` makes every `async fn` return a +/// `Pin>`, which is required for `Box` +/// and `Arc` to be dyn-compatible. +#[async_trait::async_trait] +pub trait LocalEmbedder: Send + Sync + 'static { + async fn embed(&self, text: &str) -> EmbedderResult>; + + fn model_name(&self) -> &str; + + fn dimension(&self) -> usize; + + /// Stable blake3 hash of (model_name || dimension || vestige-core crate version). + /// Lowercase hex, 64 chars. + /// + /// Used by `MemoryStore::register_model` to detect silent model drift + /// (e.g. a fastembed minor upgrade that changes vector output). + fn model_hash(&self) -> String; + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>>; + + /// Returns the `ModelSignature` describing this embedder. Convenience + /// wrapper over the three accessors above. + fn signature(&self) -> crate::storage::ModelSignature { + crate::storage::ModelSignature { + name: self.model_name().to_string(), + dimension: self.dimension(), + hash: self.model_hash(), + } + } +} + +/// Type alias: `Embedder` is the dyn-compatible, Send+Sync variant. +/// Both names refer to the same `async_trait`-annotated trait. +pub use LocalEmbedder as Embedder; +``` + +### After + +`crates/vestige-core/src/embedder/mod.rs:1-55` (approximately): + +```rust +//! Text-to-vector encoding trait. Pluggable per-install. + +mod fastembed; + +pub use fastembed::FastembedEmbedder; + +/// Error returned by every `Embedder` method. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum EmbedderError { + #[error("embedder initialization failed: {0}")] + Init(String), + #[error("embedding generation failed: {0}")] + EmbedFailed(String), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +pub type EmbedderResult = std::result::Result; + +/// Pluggable embedder. The storage layer NEVER calls fastembed directly; +/// callers compute vectors via this trait and pass them into `MemoryStore`. +/// +/// `LocalEmbedder` is the source-of-truth trait. The +/// `#[trait_variant::make(Embedder: Send)]` attribute auto-generates an +/// `Embedder` variant whose returned futures are `Send`, so +/// `Box` and `Arc` are usable on tokio/axum +/// runtimes, while `Box` remains usable on single- +/// threaded executors and thread-local backends. +/// +/// Every method is native async-fn-in-trait (stable on MSRV 1.91); no +/// per-call heap allocation, no boxed futures at the static-dispatch +/// boundary. +#[trait_variant::make(Embedder: Send)] +pub trait LocalEmbedder: Sync + 'static { + async fn embed(&self, text: &str) -> EmbedderResult>; + + fn model_name(&self) -> &str; + + fn dimension(&self) -> usize; + + /// Stable blake3 hash of (model_name || dimension || vestige-core crate version). + /// Lowercase hex, 64 chars. + /// + /// Used by `MemoryStore::register_model` to detect silent model drift + /// (e.g. a fastembed minor upgrade that changes vector output). + fn model_hash(&self) -> String; + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>>; + + /// Returns the `ModelSignature` describing this embedder. Convenience + /// wrapper over the three accessors above. + fn signature(&self) -> crate::storage::ModelSignature { + crate::storage::ModelSignature { + name: self.model_name().to_string(), + dimension: self.dimension(), + hash: self.model_hash(), + } + } +} +``` + +### Both halves of the macro-generated output (for reviewer clarity) + +`trait_variant::make(Embedder: Send)` expands the source-of-truth +`LocalEmbedder` declaration above into the equivalent of: + +```rust +// 1. The source-of-truth trait, exactly as written. +pub trait LocalEmbedder: Sync + 'static { + fn embed(&self, text: &str) -> impl Future>>; + fn model_name(&self) -> &str; + fn dimension(&self) -> usize; + fn model_hash(&self) -> String; + fn embed_batch(&self, texts: &[&str]) -> impl Future>>>; + fn signature(&self) -> crate::storage::ModelSignature { /* default impl unchanged */ } +} + +// 2. The generated Send variant. +pub trait Embedder: Sync + 'static { + fn embed(&self, text: &str) -> impl Future>> + Send; + fn model_name(&self) -> &str; + fn dimension(&self) -> usize; + fn model_hash(&self) -> String; + fn embed_batch(&self, texts: &[&str]) -> impl Future>>> + Send; + fn signature(&self) -> crate::storage::ModelSignature { /* default impl unchanged */ } +} + +// 3. The blanket impl that wires any LocalEmbedder + Send through to Embedder. +impl Embedder for T +where + T: LocalEmbedder + Send, + // (all returned futures of LocalEmbedder's async fns are required to be Send, + // which is satisfied for FastembedEmbedder -- see "Risks" below) +{ /* forwarders */ } +``` + +Notes: + +- The `pub use LocalEmbedder as Embedder;` line on the current + `embedder/mod.rs:57` is **deleted** entirely. `Embedder` is now the + trait that `trait_variant::make` emits at the same module path; the + re-export in `crates/vestige-core/src/lib.rs:167` + (`pub use embedder::{Embedder, ..., LocalEmbedder};`) keeps resolving + unchanged. +- `Sync + 'static` on `LocalEmbedder` (and no `Send` bound on the trait + itself) mirrors the `0001a` pattern for `LocalMemoryStore`. The current + trait carries `Send + Sync + 'static`; the rewrite drops the `Send` + bound from the local variant. `Box` is `Sync` but + not `Send`; `Box` (the generated variant) is `Send + Sync`. +- `trait_variant` 0.1 does **not** require any attribute on the impl + block. The attribute lives only on the trait declaration. See next + section. + +--- + +## Impl Block Migration + +`trait_variant` 0.1 attaches the attribute only to the trait declaration. +The impl side is plain `impl LocalEmbedder for FastembedEmbedder`; no +attribute on the impl, no `#[trait_variant::make(Embedder: Send)]` on the +impl block. The macro auto-generates the blanket +`impl Embedder for T`, so any concrete type that +implements `LocalEmbedder` automatically also implements `Embedder` +provided it is `Send`. + +`FastembedEmbedder` is `Send + Sync` because: + +- `inner: EmbeddingService` is `Send + Sync` (it wraps fastembed's + `TextEmbedding` which is `Send + Sync` after fastembed 4.x; verified + in `crates/vestige-core/src/embeddings/mod.rs`). +- `cached_hash: std::sync::OnceLock` is `Send + Sync` for `T: Send + Sync`. +- The `#[cfg(not(feature = "embeddings"))]` branch carries only + `cached_hash`, which is unconditionally `Send + Sync`. + +No new bound is needed. + +### Before + +`crates/vestige-core/src/embedder/fastembed.rs:38-100` (relevant header): + +```rust +impl Default for FastembedEmbedder { + fn default() -> Self { + Self::new() + } +} + +#[async_trait::async_trait] +impl LocalEmbedder for FastembedEmbedder { + async fn embed(&self, text: &str) -> EmbedderResult> { + // ... body unchanged ... + } + + fn model_name(&self) -> &str { /* ... */ } + fn dimension(&self) -> usize { /* ... */ } + fn model_hash(&self) -> String { /* ... */ } + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>> { + // ... body unchanged ... + } +} +``` + +### After + +`crates/vestige-core/src/embedder/fastembed.rs:38-99` (one fewer line): + +```rust +impl Default for FastembedEmbedder { + fn default() -> Self { + Self::new() + } +} + +impl LocalEmbedder for FastembedEmbedder { + async fn embed(&self, text: &str) -> EmbedderResult> { + // ... body unchanged ... + } + + fn model_name(&self) -> &str { /* ... */ } + fn dimension(&self) -> usize { /* ... */ } + fn model_hash(&self) -> String { /* ... */ } + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>> { + // ... body unchanged ... + } +} +``` + +Diff is exactly one removed line (the `#[async_trait::async_trait]` +attribute on line 44). Every method body, every `async fn` signature, +every `use` statement inside the impl block stays verbatim. No +`Box::pin(async move { ... })`, no manual `Pin>`, no +`'async_trait` lifetime markers; native async-fn-in-trait does this +directly. + +### Why the impl block does not need an attribute + +`trait_variant::make` generates two things from the source trait +(see the "macro-generated output" subsection above): + +1. The source trait itself (`LocalEmbedder`), with native async fns. +2. A second trait (`Embedder`) whose method signatures return + `impl Future + Send` instead of `impl Future`, + plus a blanket impl wiring concrete types through. + +Both are emitted at the macro-call site. `FastembedEmbedder` writes one +impl block (against `LocalEmbedder`); the macro-generated blanket +guarantees `FastembedEmbedder: Embedder` for free. The +`Box` cast on `tests/phase_1/embedder_trait.rs:19` keeps +type-checking unchanged. + +--- + +## Call Site Audit + +Verified via, from the phase2 worktree root: + +```bash +grep -rn "async_trait\|LocalEmbedder\|dyn Embedder" crates/ +grep -rn "use.*Embedder" crates/ tests/ --include="*.rs" +grep -rn "Box\|Arc\|&dyn Embedder" crates/ tests/ --include="*.rs" +grep -rn "Box\|Arc\|&dyn LocalEmbedder" crates/ tests/ --include="*.rs" +grep -rn "impl LocalEmbedder for\|impl Embedder for" crates/ tests/ --include="*.rs" +``` + +### Files that reference the trait object form + +Exactly one, test-only: + +| File | Line | Use | Required change | +|--------------------------------------|------|------------------------------------------------------------------------------------------|-----------------| +| `tests/phase_1/embedder_trait.rs` | 3 | `use vestige_core::embedder::{Embedder, FastembedEmbedder};` | None. `Embedder` is the generated Send variant at the same path. | +| `tests/phase_1/embedder_trait.rs` | 19 | `let e: Box = Box::new(FastembedEmbedder::new());` | None. `FastembedEmbedder: LocalEmbedder + Send` -> blanket gives `: Embedder` -> `Box` is well-formed. | + +### Files that import `Embedder` or `LocalEmbedder` by name + +| File | Line | Use | Required change | +|-----------------------------------------------------|------|----------------------------------------------------------------------------------------------------------------|-----------------| +| `crates/vestige-core/src/lib.rs` | 167 | `pub use embedder::{Embedder, EmbedderError, EmbedderResult, FastembedEmbedder, LocalEmbedder};` | None. Both names still resolve. | +| `crates/vestige-core/src/embedder/mod.rs` | 5 | `pub use fastembed::FastembedEmbedder;` | None. | +| `crates/vestige-core/src/embedder/fastembed.rs` | 7 | `use super::{EmbedderError, EmbedderResult, LocalEmbedder};` | None. | +| `tests/phase_1/embedder_trait.rs` | 3 | `use vestige_core::embedder::{Embedder, FastembedEmbedder};` | None. | + +### Files that implement the trait + +| File | Line | Impl | Required change | +|-----------------------------------------------------|------|-----------------------------------------------------------------------|----------------------------------------------| +| `crates/vestige-core/src/embedder/fastembed.rs` | 45 | `impl LocalEmbedder for FastembedEmbedder` (currently `#[async_trait]`) | Drop the `#[async_trait::async_trait]` attr. | + +No other impls exist. There is no test mock implementing `Embedder` or +`LocalEmbedder` anywhere in the workspace. + +### Files that import `async_trait` directly + +After `0001a` lands, only the embedder pair: + +``` +crates/vestige-core/src/embedder/mod.rs:24 (doc comment) +crates/vestige-core/src/embedder/mod.rs:27 (attribute) +crates/vestige-core/src/embedder/mod.rs:56 (doc comment) +crates/vestige-core/src/embedder/fastembed.rs:44 (attribute) +``` + +Plus the Cargo manifest: + +``` +crates/vestige-core/Cargo.toml:119:async-trait = "0.1" +``` + +### Production files that hold a concrete embedder + +`FastembedEmbedder` is constructed and used by concrete name (not via +trait object) in: the dashboard / MCP layer if it needs to embed query +strings ad-hoc. None of those call sites need an edit because the +concrete type is what they hold, and the concrete type is untouched by +this sub-plan. + +### Conclusion + +| Category | Count | +|--------------------------------------------------|-------| +| Production source files modified | 2 | +| Test source files modified | 0 | +| Cargo manifests modified | 1 | +| Production source files importing `Embedder` / `LocalEmbedder` (verified unchanged) | 3 | +| Test source files importing `Embedder` (verified unchanged) | 1 | +| Direct `async_trait` uses outside the embedder module after `0001a` | 0 | + +--- + +## Cargo.toml Change + +From inside `crates/vestige-core/`: + +```bash +cargo rm async-trait +``` + +This removes line 119 of `Cargo.toml` and updates `Cargo.lock` in one +step. Manual editing is acceptable as a fallback if `cargo rm` is +unavailable in the agent environment; in that case, follow up with +`cargo check -p vestige-core` to refresh the lockfile. + +### Verification + +```bash +# Direct dependency must be gone. +grep -rn "async-trait\|async_trait" --include="Cargo.toml" crates/ +# Expected: empty. + +# Transitive presence is permitted (e.g. via a third-party crate). +cargo tree -p vestige-core --depth 2 | grep async-trait +# Expected: empty for the direct edges; if a sub-dependency still pulls +# async-trait transitively, the output may contain it deeper than depth 2, +# which is fine. We only care about removing the DIRECT edge. +``` + +If `cargo tree --depth 2` returns any `async-trait` line, inspect with +`cargo tree -p vestige-core -i async-trait` to see what is pulling it. +Acceptable parents: any third-party crate. Unacceptable parent: anything +under `vestige-*`, which would mean a missed file. + +--- + +## Commit Sequence + +Three commits, each green on +`cargo test -p vestige-core --features embeddings,vector-search` and +`cargo test -p vestige-core --no-default-features`. + +### Commit 1: rewrite LocalEmbedder trait declaration + +- Touches: `crates/vestige-core/src/embedder/mod.rs` only. +- Action: replace lines 21-57 per the "Trait Declaration Rewrite" + section above. Delete the `pub use LocalEmbedder as Embedder;` line. +- Green after: `cargo check -p vestige-core` (the impl block in + `fastembed.rs` still has its `#[async_trait::async_trait]` attribute; + the macro is harmless when applied to a trait that the impl block + targets by path, because async_trait rewrites the impl's async fns + into boxed-future fns whose signatures still match the native-async + declarations after trait_variant lowering, just as it did for the + SQLite intermediate state in `0001a`'s commit 1). + + **Mitigation if check fails between commits 1 and 2:** combine the + two into a single commit. The split is offered for review convenience; + the build must be green after every commit lands. + +### Commit 2: drop `#[async_trait::async_trait]` from FastembedEmbedder impl + +- Touches: `crates/vestige-core/src/embedder/fastembed.rs` only. +- Action: delete line 44 (`#[async_trait::async_trait]`). +- Green after: + - `cargo test -p vestige-core --features embeddings,vector-search`. + - `cargo test -p vestige-core --no-default-features` (the + `#[cfg(not(feature = "embeddings"))]` branches inside the impl now + stand on their own). + - Phase 1 integration test: `cargo test --test embedder_trait + --features embeddings,vector-search`. + +### Commit 3: drop the async-trait dependency + +- Touches: `crates/vestige-core/Cargo.toml` (plus `Cargo.lock` as a + side effect). +- Action: from inside `crates/vestige-core/`, run `cargo rm async-trait`. +- Green after: `cargo build --workspace --all-targets` and + `cargo test --workspace`. +- Final hard ASCII gate: `! grep -rn "async_trait" crates/` must exit + with status 0 (i.e. the inverted grep finds nothing). + +### Combined alternative + +Commits 1 and 2 may fold into a single commit if the per-step split +feels artificial (the patterns are identical to `0001a`'s commits 3 +and 4). Commit 3 (the Cargo.toml removal) should stay separate so the +dependency-removal diff is visible in isolation. + +--- + +## Verification + +Every command runs from the repo root unless noted otherwise. + +```bash +# 1. Vestige-core, default features (embeddings + vector-search). +cargo test -p vestige-core --features embeddings,vector-search + +# 2. Vestige-core, minimal features (no embeddings, no vector-search). +cargo test -p vestige-core --no-default-features + +# 3. Workspace build, all targets (catches any feature-gated regression +# in the vestige-mcp tools tree). +cargo build --workspace --all-targets + +# 4. Whole-workspace test (vestige-mcp 406 tests + vestige-core 352 +# tests per the CLAUDE.md baseline). +cargo test --workspace + +# 5. Phase 1 embedder integration test (the trait-shape contract). +cargo test --test embedder_trait --features embeddings,vector-search + +# 6. Clippy gate, deny warnings (matches Phase 1 PR policy). +cargo clippy --workspace --all-targets --features embeddings,vector-search -- -D warnings + +# 7. Hard ASCII gate -- async_trait must be gone from source. +! grep -rn "async_trait" crates/ +# Inverted grep: exit 0 iff grep found nothing. + +# 8. Hard ASCII gate -- async-trait must be gone from manifests. +! grep -rn "async-trait" --include="Cargo.toml" crates/ + +# 9. Confirm trait_variant attribute is in place at the embedder. +grep -rn "trait_variant::make" crates/vestige-core/src/embedder/ +# Expected: exactly one hit, in embedder/mod.rs. + +# 10. Workspace-wide trait_variant audit (should match the count after +# 0001a -- two hits total, one for storage, one for embedder). +grep -rn "trait_variant::make" crates/vestige-core/src/ +# Expected: two hits. +``` + +Expected outcomes: + +- Command 1: 352 vestige-core tests pass (matches baseline). +- Command 2: smaller test count, all pass. +- Command 3: workspace builds in dev mode for all targets. +- Command 4: 758 total tests pass (matches CLAUDE.md baseline). +- Command 5: `embedder_trait` integration test passes. The + `fastembed_implements_embedder_trait` assertion (`let e: Box = ...`) is the canary; if `trait_variant::make` failed to + emit the `Embedder` Send variant, this fails to compile. +- Command 6: zero clippy warnings. +- Command 7: empty output. `async_trait` is fully gone from source. +- Command 8: empty output. `async-trait` is fully gone from manifests. +- Command 9: one hit. +- Command 10: two hits. + +--- + +## Acceptance Criteria + +A reviewer should be able to check every box: + +- [ ] `crates/vestige-core/src/embedder/mod.rs` declares the embedder + trait with `#[trait_variant::make(Embedder: Send)] pub trait + LocalEmbedder: Sync + 'static`, no `async_trait` attribute, no + `Send` bound on `LocalEmbedder` itself. +- [ ] `crates/vestige-core/src/embedder/mod.rs` no longer contains + `pub use LocalEmbedder as Embedder;`. +- [ ] `crates/vestige-core/src/embedder/fastembed.rs` declares + `impl LocalEmbedder for FastembedEmbedder` with no attribute on + the impl block. +- [ ] `crates/vestige-core/Cargo.toml` does not declare `async-trait` + as a direct dependency. +- [ ] `grep -rn "async_trait" crates/` returns zero hits. +- [ ] `grep -rn "async-trait" --include="Cargo.toml" crates/` returns + zero hits. +- [ ] `grep -rn "trait_variant::make" crates/vestige-core/src/` returns + exactly two hits (storage trait + embedder trait). +- [ ] All 758 workspace tests pass (`cargo test --workspace`). +- [ ] `tests/phase_1/embedder_trait.rs` compiles and passes with the + `Box` cast intact. +- [ ] `cargo clippy --workspace --all-targets --features + embeddings,vector-search -- -D warnings` is clean. +- [ ] No file under `crates/vestige-mcp/` or under + `crates/vestige-core/src/{neuroscience,advanced,consolidation, + codebase,memory,embeddings}/` was modified by this sub-plan. +- [ ] `Cargo.lock` was updated as a side effect of `cargo rm async-trait` + (it must no longer reference `async-trait`). +- [ ] Doc comments on the embedder trait declaration describe + `trait_variant`, not `async_trait`. + +--- + +## Risks and Mitigations + +- **`trait_variant::make` requires returned futures to be `Send` for the + blanket `impl Embedder for T`. If any + `async fn embed`/`embed_batch` body inside `FastembedEmbedder` captures + a non-Send local, the blanket impl fails to type-check.** + Mitigation: the existing impl bodies call `self.inner.embed(text)` / + `self.inner.embed_batch(texts)`, where `inner: EmbeddingService` is + `Send + Sync` (verified in `crates/vestige-core/src/embeddings/mod.rs`). + No `.await` points exist inside the bodies in either feature branch; + the `EmbeddingService::embed` calls are synchronous. The futures are + trivially `Send`. If a future change introduces a non-Send local + (e.g. an `Rc` or a non-Send guard), the blanket impl will surface that + as a compile error at the dyn cast in `tests/phase_1/embedder_trait.rs`, + which is the correct outcome. +- **The macro's blanket impl interacts oddly with the default `signature` + method.** + Mitigation: `signature` is a synchronous method returning + `crate::storage::ModelSignature`, with no `Send` or `async` concerns. + `trait_variant::make` emits it on both variants as-is. The existing + Phase 1 test `signature_matches_memory_store_registry` exercises this + path and is part of the verification step. +- **`Box` cast in `tests/phase_1/embedder_trait.rs` fails + to resolve after the rewrite.** + Mitigation: the rewrite preserves the `Embedder` symbol at the same + module path; only its provenance changes (now generated by + `trait_variant::make` instead of by `pub use LocalEmbedder as + Embedder;`). The macro is specifically designed so that the generated + trait is dyn-compatible at the Send-bound boundary. Verified by the + identical pattern already working for `MemoryStore` after `0001a`. +- **`cargo rm async-trait` updates `Cargo.lock` but accidentally bumps + other crates.** + Mitigation: run `cargo rm async-trait` and then immediately inspect + the resulting `Cargo.lock` diff. The expected diff is the removal of + the `[[package]] name = "async-trait"` block and its hash. Anything + else is a red flag and should be reverted before committing + (`git checkout -- Cargo.lock` then `cargo update -p async-trait + --precise=remove` -- or fall back to manual edit + `cargo check`). +- **A new workspace crate added in parallel with this work declares + `async-trait` and the dependency removal silently re-introduces it + later.** + Mitigation: the verification step `grep -rn "async-trait" + --include="Cargo.toml" crates/` is part of the acceptance criteria; a + rebase that reintroduces the line will fail this gate. +- **MCP server uses `Embedder` somewhere we missed.** + Mitigation: full workspace grep (`grep -rn "Embedder" crates/`) + returns no hits inside `crates/vestige-mcp/` for the trait names; the + MCP layer uses the concrete `EmbeddingService` from + `crates/vestige-core/src/embeddings/` for ad-hoc embedding calls. The + trait surface is purely internal to `vestige-core`. + +--- + +## Out-of-Band Notes + +- **No other workspace crate declares `async-trait` as a direct + dependency.** Verified by + `grep -rn "async-trait" --include="Cargo.toml" crates/` returning + exactly one hit at `crates/vestige-core/Cargo.toml:119`. There is + nothing to clean up in `crates/vestige-mcp/Cargo.toml` or elsewhere. +- **Order matters across the three Phase 1 amendment sub-plans:** + `0001a` (trait rewrite) -> `0001b` (sqlite split) -> `0001c` (this + one, async-trait sunset). Reversing the order is possible in + principle but would force re-editing the embedder rewrite twice and + leaves the `async-trait` dep behind until very late. +- **This sub-plan amends `feat/storage-trait-phase1` (tip 790c0c8 plus + whatever commits `0001a` and `0001b` added).** The branch has not + been opened upstream yet, so amending in place is safe; no force-push + to a public PR. +- **After this sub-plan lands, the branch is reviewed and merged before + Phase 2 sub-plans (`0002a-` through `0002i-`) begin implementation.** + Phase 2 introduces no async-trait usage; the Postgres backend follows + the same `trait_variant::make` pattern (see ADR 0002 D1). +- **`trait-variant` 0.1 stays in `Cargo.toml`.** It is the only crate + this sub-plan keeps; `async-trait` is the only one it removes. + +--- + +## Self-Contained `/goal` Brief + +For a fresh Claude Code session executing this sub-plan without prior +conversation context: + +1. Check out branch `feat/storage-trait-phase1` (or a worktree off + of it after `0001a` and `0001b` are merged into it). +2. Read this file (`docs/plans/0001c-async-trait-sunset.md`) in full. +3. Read `docs/plans/0001a-trait-rewrite.md` sections "Trait declaration + rewrite" and "Impl block migration" -- they document the exact + pattern this sub-plan mirrors for the embedder. +4. Run the prerequisite audit grep listed under "Prerequisites". If it + returns more than the five hits documented there, stop and report; + the upstream state does not match what this sub-plan assumes. +5. Execute Commit 1 (rewrite `embedder/mod.rs`), then Commit 2 (drop + the attribute on the FastembedEmbedder impl), then Commit 3 + (`cargo rm async-trait`). Run the verification commands listed + above after each commit; do not proceed if any test or clippy gate + fails. +6. Verify every box in "Acceptance Criteria" is ticked. +7. Report file paths touched, test counts, and the final two grep + results (commands 7 and 8 from "Verification") in the closing + message. From 9ef8afdb20337d08cb5489078ae49eec8c7c1cf1 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 09:35:58 +0200 Subject: [PATCH 05/38] docs(plans): add Phase 2 sub-plans 0002a-0002i + supersession notice Nine Phase 2 sub-plans operationalising ADR 0002 against the Phase 2 master plan, each sized to fit a focused implementation session and handed to Claude Code as a /goal brief without requiring the agent to load the master plan. Order of execution (each depends on the previous unless noted): - 0002a-skeleton-and-feature-gate.md -- postgres-backend Cargo feature + PgMemoryStore skeleton with todo!() bodies. D1+D2. - 0002b-pool-and-config.md -- PgPool builder, VestigeConfig/ PostgresConfig, vestige.toml loader wired into vestige-mcp. D3+D7 (master plan numbering). - 0002c-migrations.md -- sqlx migrations 0001_init/0002_hnsw including D7 (users/groups/memberships, owner/visibility/shared_with_groups) and D8 (codebase column). SQLite V15 parity migration. D4. - 0002d-store-impl-bodies.md -- real CRUD + registry bodies; trivial fts_search/vector_search bodies. D2+D6. - 0002e-hybrid-search.md -- one-statement RRF query. D5. - 0002f-migrate-cli.md -- vestige migrate copy (SQLite -> Postgres), --dry-run, idempotent re-runs, --allow-source-upgrade for pre-V15 sources. D8+D10. - 0002g-reembed.md -- vestige migrate reembed (offline rebuild). D9 + D10 reembed arm. Ships resolve_embedder helper as a workaround for the missing Embedder::from_name(&str) constructor. - 0002h-testing-and-benches.md -- testcontainers harness, six integration test files, Criterion bench at 1k/100k. D14+D15. - 0002i-runbook.md -- operator-facing deployment + day-2 runbook. D16. Supersession notice added to the master plan (0002-phase-2-postgres- backend.md) pointing at ADR 0002; body retained as archival reference. PR B carries this commit plus the previous two (ADR 0002 + Phase 1 amendment sub-plans); no code change. --- docs/plans/0002-phase-2-postgres-backend.md | 8 + docs/plans/0002a-skeleton-and-feature-gate.md | 554 ++++++ docs/plans/0002b-pool-and-config.md | 886 +++++++++ docs/plans/0002c-migrations.md | 1119 +++++++++++ docs/plans/0002d-store-impl-bodies.md | 1771 +++++++++++++++++ docs/plans/0002e-hybrid-search.md | 825 ++++++++ docs/plans/0002f-migrate-cli.md | 1045 ++++++++++ docs/plans/0002g-reembed.md | 843 ++++++++ docs/plans/0002h-testing-and-benches.md | 1223 ++++++++++++ docs/plans/0002i-runbook.md | 724 +++++++ 10 files changed, 8998 insertions(+) create mode 100644 docs/plans/0002a-skeleton-and-feature-gate.md create mode 100644 docs/plans/0002b-pool-and-config.md create mode 100644 docs/plans/0002c-migrations.md create mode 100644 docs/plans/0002d-store-impl-bodies.md create mode 100644 docs/plans/0002e-hybrid-search.md create mode 100644 docs/plans/0002f-migrate-cli.md create mode 100644 docs/plans/0002g-reembed.md create mode 100644 docs/plans/0002h-testing-and-benches.md create mode 100644 docs/plans/0002i-runbook.md diff --git a/docs/plans/0002-phase-2-postgres-backend.md b/docs/plans/0002-phase-2-postgres-backend.md index 3fe28f2..ed2a186 100644 --- a/docs/plans/0002-phase-2-postgres-backend.md +++ b/docs/plans/0002-phase-2-postgres-backend.md @@ -1,5 +1,13 @@ # Phase 2 Plan: PostgreSQL Backend +> **Supersession Notice (2026-05-26):** This master plan is now archival. Execution is governed by: +> - **ADR**: [`docs/adr/0002-phase-2-execution.md`](../adr/0002-phase-2-execution.md) -- binding decisions +> - **Sub-plans** (executable briefs): +> - Phase 1 amendment: [0001a-trait-rewrite.md](0001a-trait-rewrite.md), [0001b-sqlite-split.md](0001b-sqlite-split.md), [0001c-async-trait-sunset.md](0001c-async-trait-sunset.md) +> - Phase 2: 0002a..0002i (skeleton, pool+config, migrations, store impl, hybrid search, migrate CLI, reembed, tests+benches, runbook) +> +> **Deltas vs body**: trait uses `trait_variant`, error type is `MemoryStoreError`/`MemoryStoreResult`, `connect` is `(url, max_connections)` only, the core table is `knowledge_nodes` (not `memories`) and gains `owner_user_id` + `visibility` + `shared_with_groups` + `codebase`, plus `users`/`groups`/`group_memberships` tables. See ADR 0002 D1-D8. + **Status**: Draft **Depends on**: Phase 1 (MemoryStore + Embedder traits, embedding_model registry, domain columns) **Related**: docs/adr/0001-pluggable-storage-and-network-access.md (Phase 2), docs/prd/001-getting-centralized-vestige.md, docs/plans/local-dev-postgres-setup.md (local cluster provisioning) diff --git a/docs/plans/0002a-skeleton-and-feature-gate.md b/docs/plans/0002a-skeleton-and-feature-gate.md new file mode 100644 index 0000000..74032dc --- /dev/null +++ b/docs/plans/0002a-skeleton-and-feature-gate.md @@ -0,0 +1,554 @@ +# Phase 2 Sub-Plan 0002a -- Skeleton and Feature Gate + +**Status**: Ready +**Depends on**: Phase 1 amendment (sub-plans `0001a-trait-rewrite.md` and +`0001b-sqlite-split.md`) merged. Specifically: +- `MemoryStore` trait declared with `#[trait_variant::make(MemoryStore: Send)]`, + generating a non-Send `LocalMemoryStore` companion trait. The + `pub use MemoryStore as LocalMemoryStore` alias from Phase 1 is gone. +- `crates/vestige-core/src/storage/sqlite.rs` has been split into + `crates/vestige-core/src/storage/sqlite/` with the same public surface. + +This sub-plan covers Phase 2 master-plan deliverables D1 and D2 only: +the `postgres-backend` Cargo feature gate and a compilable `PgMemoryStore` +skeleton whose trait method bodies are `todo!()`. No real Postgres code, no +migrations, no SQL. Later sub-plans (`0002b-pool-and-config.md`, +`0002c-migrations.md`, `0002d-store-impl-bodies.md`, ...) fill the bodies in. + +The success criterion is a clean build under both feature-flag configurations, +nothing more. + +--- + +## Context + +ADR 0002 D4 commits Phase 2 to a `crates/vestige-core/src/storage/postgres/` +directory from day one. The seven other files in that directory +(`pool.rs`, `migrations.rs`, `registry.rs`, `search.rs`, `migrate_cli.rs`, +`reembed.rs`) belong to subsequent sub-plans. This sub-plan creates only +`crates/vestige-core/src/storage/postgres/mod.rs` so the rest can be added +incrementally without breaking the build. + +Per ADR 0002 D2, `PgMemoryStore::connect` mirrors `SqliteMemoryStore::new`: +no `Embedder` argument. The pgvector typmod DDL +(`ALTER TABLE memories ALTER COLUMN embedding TYPE vector($N)`) lives inside +the trait method `register_model`, invoked by the caller after construction. +In this sub-plan `register_model` is a `todo!()` body; `0002c-migrations.md` +and `0002d-store-impl-bodies.md` provide the real implementation. + +The trait surface in `crates/vestige-core/src/storage/memory_store.rs` is the +source of truth for method signatures. Do NOT copy signatures from the master +plan -- they are stale in places (for example, master plan 0002 D2 lists +`remove_edge` as three-arg `(source, target, edge_type)`; the live trait has +two args `(source, target)`). + +--- + +## Cargo manifest changes + +Two optional crates and one new feature flag. Use `cargo add` per the global +CLAUDE.md preference; do not hand-edit `Cargo.toml`. + +```bash +cd crates/vestige-core + +cargo add sqlx@0.8 --optional --no-default-features \ + --features runtime-tokio,tls-rustls,postgres,uuid,chrono,json,migrate,macros + +cargo add pgvector@0.4 --optional --features sqlx +``` + +After both commands, open `crates/vestige-core/Cargo.toml` and add the +`postgres-backend` feature line in the `[features]` block. Place it after +the `metal` feature, before `[dependencies]`: + +```toml +# Postgres backend (mutually compilable with the SQLite backend; default OFF). +# Compile with: --features postgres-backend +postgres-backend = ["dep:sqlx", "dep:pgvector"] +``` + +Do NOT add `tokio-stream`, `futures`, `indicatif`, or `toml` in this sub-plan. +The master plan D1 lists them in the `postgres-backend` feature for +convenience, but their consumers (streaming migrate, progress bar, config +parsing) land in later sub-plans. Adding them here pulls dead weight into the +feature gate. + +Do NOT add the `vestige-mcp` pass-through feature in this sub-plan either. +The MCP crate gets its `postgres-backend` feature in `0002b-pool-and-config.md` +when `MemoryStoreConfig` lands and the binary needs a knob to pick a backend. + +Verify the diff to `crates/vestige-core/Cargo.toml` looks like this and only +this: + +```toml +[features] +# ...existing features unchanged... +postgres-backend = ["dep:sqlx", "dep:pgvector"] + +[dependencies] +# ...existing deps unchanged... +sqlx = { version = "0.8", default-features = false, features = [ + "runtime-tokio", "tls-rustls", "postgres", "uuid", "chrono", + "json", "migrate", "macros", +], optional = true } +pgvector = { version = "0.4", features = ["sqlx"], optional = true } +``` + +The exact ordering of the two new lines inside `[dependencies]` is not +significant; `cargo add` places them at the end. Leave the placement that +`cargo add` produces. + +--- + +## Storage module export + +Edit `crates/vestige-core/src/storage/mod.rs` to expose the new module behind +the feature flag. Two lines change. + +Add to the module-declaration block (after `mod sqlite;`): + +```rust +#[cfg(feature = "postgres-backend")] +mod postgres; +``` + +Add to the re-export block (after the `pub use sqlite::{ ... }` block): + +```rust +#[cfg(feature = "postgres-backend")] +pub use postgres::PgMemoryStore; +``` + +Nothing else in `storage/mod.rs` changes. The `Storage` alias still points at +`SqliteMemoryStore`; the SQLite re-export block is untouched. + +--- + +## Postgres module skeleton + +Create `crates/vestige-core/src/storage/postgres/mod.rs` with the full content +below. This is the only new file in this sub-plan. + +```rust +#![cfg(feature = "postgres-backend")] +//! Postgres-backed implementation of `MemoryStore`. +//! +//! Skeleton only. Every trait method is `todo!()`. Real bodies land in +//! subsequent Phase 2 sub-plans: +//! - `0002b-pool-and-config.md`: pool construction and config wiring +//! - `0002c-migrations.md`: sqlx migration files and `init`/`register_model` +//! - `0002d-store-impl-bodies.md`: CRUD, scheduling, edges, domains +//! - `0002e-hybrid-search.md`: RRF query and search bodies +//! +//! The directory grows companion files (`pool.rs`, `migrations.rs`, +//! `registry.rs`, `search.rs`, `migrate_cli.rs`, `reembed.rs`) in those +//! sub-plans; this skeleton only creates `mod.rs`. + +use chrono::{DateTime, Utc}; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::storage::memory_store::{ + Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, MemoryStoreResult, + ModelSignature, SchedulingState, SearchQuery, SearchResult, StoreStats, +}; + +/// Postgres-backed implementation of `MemoryStore`. +/// +/// Cheaply cloneable. Methods take `&self`; interior state lives inside the +/// `PgPool` (which already provides `Sync` via `Arc` internally). +#[derive(Clone)] +pub struct PgMemoryStore { + pool: PgPool, + /// Embedding vector dimension. Set to 0 in the skeleton; populated by + /// `register_model` in `0002d-store-impl-bodies.md` once the pgvector + /// `ALTER COLUMN TYPE vector(N)` DDL lands. + embedding_dim: i32, +} + +impl PgMemoryStore { + /// Construct a new store from a connection URL. + /// + /// Mirrors `SqliteMemoryStore::new`: no `Embedder` argument. The pgvector + /// `ALTER TABLE memories ALTER COLUMN embedding TYPE vector($N)` DDL lives + /// inside `register_model`, not here. The caller (cognitive engine + /// bootstrap, migrate CLI, tests) invokes `register_model` after this + /// returns, identically to the SQLite path. + /// + /// Real body lands in `0002b-pool-and-config.md` (pool construction) and + /// `0002c-migrations.md` (initial migration run). + pub async fn connect(_url: &str, _max_connections: u32) -> MemoryStoreResult { + todo!("PgMemoryStore::connect lands in 0002b-pool-and-config.md") + } + + /// Low-level constructor for tests: supply an existing pool, skip migrate. + /// + /// Real body lands in `0002b-pool-and-config.md`. + pub async fn from_pool(_pool: PgPool) -> MemoryStoreResult { + todo!("PgMemoryStore::from_pool lands in 0002b-pool-and-config.md") + } + + /// Accessor used by migrate/reembed CLI. + pub fn pool(&self) -> &PgPool { + &self.pool + } + + /// Currently-registered vector dimension. Returns 0 until `register_model` + /// has been called (real body: `0002d-store-impl-bodies.md`). + pub fn embedding_dim(&self) -> i32 { + self.embedding_dim + } +} + +// trait_variant::make on the trait declaration generates two traits: +// - `MemoryStore` (Send-bound) +// - `LocalMemoryStore` (non-Send companion) +// The implementer writes `impl LocalMemoryStore for ...` plainly; the Send +// variant is generated automatically from the non-Send impl. +impl LocalMemoryStore for PgMemoryStore { + // --- Lifecycle --- + async fn init(&self) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::init lands in 0002c-migrations.md") + } + + async fn health_check(&self) -> MemoryStoreResult { + todo!("PgMemoryStore::health_check lands in 0002d-store-impl-bodies.md") + } + + // --- Embedding model registry --- + async fn registered_model(&self) -> MemoryStoreResult> { + todo!("PgMemoryStore::registered_model lands in 0002d-store-impl-bodies.md") + } + + async fn register_model(&self, _sig: &ModelSignature) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::register_model lands in 0002d-store-impl-bodies.md") + } + + // --- CRUD --- + async fn insert(&self, _record: &MemoryRecord) -> MemoryStoreResult { + todo!("PgMemoryStore::insert lands in 0002d-store-impl-bodies.md") + } + + async fn get(&self, _id: Uuid) -> MemoryStoreResult> { + todo!("PgMemoryStore::get lands in 0002d-store-impl-bodies.md") + } + + async fn update(&self, _record: &MemoryRecord) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::update lands in 0002d-store-impl-bodies.md") + } + + async fn delete(&self, _id: Uuid) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::delete lands in 0002d-store-impl-bodies.md") + } + + // --- Search --- + async fn search(&self, _query: &SearchQuery) -> MemoryStoreResult> { + todo!("PgMemoryStore::search lands in 0002e-hybrid-search.md") + } + + async fn fts_search( + &self, + _text: &str, + _limit: usize, + ) -> MemoryStoreResult> { + todo!("PgMemoryStore::fts_search lands in 0002e-hybrid-search.md") + } + + async fn vector_search( + &self, + _embedding: &[f32], + _limit: usize, + ) -> MemoryStoreResult> { + todo!("PgMemoryStore::vector_search lands in 0002e-hybrid-search.md") + } + + // --- FSRS Scheduling --- + async fn get_scheduling( + &self, + _memory_id: Uuid, + ) -> MemoryStoreResult> { + todo!("PgMemoryStore::get_scheduling lands in 0002d-store-impl-bodies.md") + } + + async fn update_scheduling(&self, _state: &SchedulingState) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::update_scheduling lands in 0002d-store-impl-bodies.md") + } + + async fn get_due_memories( + &self, + _before: DateTime, + _limit: usize, + ) -> MemoryStoreResult> { + todo!("PgMemoryStore::get_due_memories lands in 0002d-store-impl-bodies.md") + } + + // --- Graph (spreading activation) --- + async fn add_edge(&self, _edge: &MemoryEdge) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::add_edge lands in 0002d-store-impl-bodies.md") + } + + async fn get_edges( + &self, + _node_id: Uuid, + _edge_type: Option<&str>, + ) -> MemoryStoreResult> { + todo!("PgMemoryStore::get_edges lands in 0002d-store-impl-bodies.md") + } + + async fn remove_edge(&self, _source: Uuid, _target: Uuid) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::remove_edge lands in 0002d-store-impl-bodies.md") + } + + async fn get_neighbors( + &self, + _node_id: Uuid, + _depth: usize, + ) -> MemoryStoreResult> { + todo!("PgMemoryStore::get_neighbors lands in 0002d-store-impl-bodies.md") + } + + // --- Domains (Phase 1: stubs return empty; full impl in Phase 4) --- + async fn list_domains(&self) -> MemoryStoreResult> { + todo!("PgMemoryStore::list_domains lands in 0002d-store-impl-bodies.md") + } + + async fn get_domain(&self, _id: &str) -> MemoryStoreResult> { + todo!("PgMemoryStore::get_domain lands in 0002d-store-impl-bodies.md") + } + + async fn upsert_domain(&self, _domain: &Domain) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::upsert_domain lands in 0002d-store-impl-bodies.md") + } + + async fn delete_domain(&self, _id: &str) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::delete_domain lands in 0002d-store-impl-bodies.md") + } + + async fn classify(&self, _embedding: &[f32]) -> MemoryStoreResult> { + todo!("PgMemoryStore::classify lands in 0002d-store-impl-bodies.md") + } + + // --- Bulk / Maintenance --- + async fn count(&self) -> MemoryStoreResult { + todo!("PgMemoryStore::count lands in 0002d-store-impl-bodies.md") + } + + async fn get_stats(&self) -> MemoryStoreResult { + todo!("PgMemoryStore::get_stats lands in 0002d-store-impl-bodies.md") + } + + async fn vacuum(&self) -> MemoryStoreResult<()> { + todo!("PgMemoryStore::vacuum lands in 0002d-store-impl-bodies.md") + } +} +``` + +Notes on the skeleton: + +- The file-level `#![cfg(feature = "postgres-backend")]` means the whole file + vanishes when the feature is off. The `mod postgres;` line in + `storage/mod.rs` is itself feature-gated, so this is belt-and-braces; both + gates are needed because the file-level attribute is what allows the file to + use `sqlx::PgPool` unconditionally inside it. +- `EmbeddingModelDescriptor` (a separate Postgres-internal type that the + master plan sketched on the struct) is dropped. The trait surface already + carries `ModelSignature` for the registry round-trip; the registry storage + layout is a private concern of `registry.rs`, which is added later. Keep + `PgMemoryStore` minimal until a real consumer needs the extra type. +- The struct only carries `pool` and `embedding_dim`. The model descriptor + field from the master plan sketch goes away with `EmbeddingModelDescriptor`. + If `register_model` later needs to cache the descriptor on the struct, it + can be added then; the skeleton does not speculate. +- The two trivial accessors (`pool`, `embedding_dim`) get real bodies. Every + other method is `todo!()` so it returns `!` and trivially coerces to the + declared return type at the type checker; this is what lets the build pass + with no error variants and no SQL. + +--- + +## Connect signature + +Per ADR 0002 D2: + +```rust +pub async fn connect(url: &str, max_connections: u32) -> MemoryStoreResult; +pub async fn from_pool(pool: PgPool) -> MemoryStoreResult; +``` + +No `&dyn Embedder` argument. This deliberately differs from master plan 0002, +which predates the Phase 1 freeze. The pgvector-specific DDL +(`ALTER TABLE memories ALTER COLUMN embedding TYPE vector($N)`) does not run +inside `connect`; it runs inside `register_model(&ModelSignature)`, which the +caller invokes after `connect` returns. + +In this sub-plan `register_model` is `todo!()`. The real body lands in +`0002d-store-impl-bodies.md` after `0002c-migrations.md` ships the +`0001_init.up.sql` migration that creates the `memories` table with a +placeholder `embedding vector` column (no typmod), against which +`register_model` later runs the typmod stamp. + +--- + +## Error variant additions: deferred + +`MemoryStoreError` does NOT gain `Postgres(sqlx::Error)` or +`Migrate(sqlx::migrate::MigrateError)` in this sub-plan. + +The reason is mechanical: `todo!()` evaluates to the never type `!`, which +coerces to any `MemoryStoreResult` regardless of the error variants +available. With every method body a `todo!()`, the skeleton has no expression +that needs to convert a `sqlx::Error` or `sqlx::migrate::MigrateError` into +`MemoryStoreError`. Adding the variants here would mean adding the +`#[cfg(feature = "postgres-backend")]` and `#[from]` plumbing to +`memory_store.rs` with no consumer yet -- dead code at every level except the +enum definition itself. + +`0002d-store-impl-bodies.md` introduces both variants in the same commit that +turns the first `todo!()` into a real `sqlx::query!` call. That keeps the +diff to `memory_store.rs` next to the first usage site, which is easier to +review than adding variants ahead of need. + +For reference, the variants that will be added in `0002d-store-impl-bodies.md` +look like this: + +```rust +#[cfg(feature = "postgres-backend")] +#[error("postgres error: {0}")] +Postgres(#[from] sqlx::Error), + +#[cfg(feature = "postgres-backend")] +#[error("postgres migration error: {0}")] +Migrate(#[from] sqlx::migrate::MigrateError), +``` + +Do not pre-add them here. + +--- + +## Verification + +Run these commands from the workspace root. All four must produce a clean +build, zero warnings on the diff-affected files, no test changes. + +```bash +# 1. Default features (SQLite backend, postgres-backend OFF). Must build. +cargo build --workspace --all-targets + +# 2. Workspace clippy with default features. Must be clean. +cargo clippy --workspace --all-targets -- -D warnings + +# 3. Postgres feature enabled. Must build. +cargo build -p vestige-core --features postgres-backend + +# 4. Clippy with postgres feature enabled. Must be clean. +cargo clippy -p vestige-core --features postgres-backend --all-targets -- -D warnings +``` + +Expected outcomes: + +- `cargo build --workspace --all-targets` finishes with no compilation of + `sqlx` or `pgvector` (both are optional, no consumer with default features). + The `postgres` module is excluded entirely via `#[cfg]`. +- `cargo build -p vestige-core --features postgres-backend` compiles `sqlx`, + `pgvector`, and `storage/postgres/mod.rs`. The build succeeds because every + trait method is `todo!()`; nothing actually runs SQL. +- Both `clippy` invocations pass with `-D warnings`. The `todo!()` macro does + not emit a `dead_code` lint by itself, and the trivial accessors are used by + later sub-plans (clippy on the postgres feature alone may flag them as + unused if you run with `--lib` only; the `--all-targets` form keeps tests + and benches in scope so this does not fire). +- If clippy flags `unused_variables` on the underscore-prefixed parameters in + the `todo!()` bodies, the underscore prefix is already the standard + suppression; if a future clippy version disagrees, add + `#[allow(unused_variables)]` to the impl block, not to each method. + +Tests are not modified in this sub-plan. The unit tests in +`memory_store.rs` (`memory_store_error_from_storage_error`, +`model_signature_serde_round_trip`, `memory_record_serde_round_trip`) keep +passing because no type they touch changes. + +Do NOT run `cargo test` against the postgres feature -- there is no Postgres +running and no test exercises `PgMemoryStore` yet. The build check is the +contract. + +--- + +## Acceptance criteria + +1. `crates/vestige-core/Cargo.toml` declares `sqlx = "0.8"` and + `pgvector = "0.4"` as optional dependencies with the exact feature sets + specified above. +2. `crates/vestige-core/Cargo.toml` declares `postgres-backend = ["dep:sqlx", + "dep:pgvector"]` and nothing else inside that feature. +3. `crates/vestige-mcp/Cargo.toml` is unchanged. +4. `crates/vestige-core/src/storage/mod.rs` adds exactly two + feature-gated lines: `mod postgres;` and `pub use postgres::PgMemoryStore;`. + No other change. +5. `crates/vestige-core/src/storage/postgres/mod.rs` exists and contains the + `PgMemoryStore` struct, `impl PgMemoryStore` block with real `pool` and + `embedding_dim` accessors and `todo!()` bodies for `connect` and + `from_pool`, and the full `impl LocalMemoryStore for PgMemoryStore` block + with `todo!()` for every trait method. +6. The trait impl method signatures match `memory_store.rs` byte-for-byte + (including `remove_edge(&self, source: Uuid, target: Uuid)` two-arg form, + not the three-arg form from the master plan). +7. `MemoryStoreError` is unchanged. +8. No other files in the crate are touched. No new files in + `storage/postgres/` besides `mod.rs`. +9. The four verification commands above all succeed. + +--- + +## Commit sequence + +One commit is recommended. The two changes (Cargo manifest + module skeleton) +do not compile in isolation: the manifest change without the skeleton produces +unused-optional-dep warnings, and the skeleton without the manifest change +fails to find `sqlx`. Splitting them adds no review value, since the second +commit is the one that has to compile cleanly. + +```bash +git add crates/vestige-core/Cargo.toml \ + crates/vestige-core/Cargo.lock \ + crates/vestige-core/src/storage/mod.rs \ + crates/vestige-core/src/storage/postgres/mod.rs + +git commit -m "feat(storage): scaffold postgres-backend feature and PgMemoryStore skeleton + +Adds the postgres-backend Cargo feature gating sqlx 0.8 and pgvector 0.4. +Introduces crates/vestige-core/src/storage/postgres/mod.rs with the +PgMemoryStore struct, connect/from_pool/pool/embedding_dim, and a trait impl +whose method bodies are todo!() pending later Phase 2 sub-plans. + +Builds clean with default features (SQLite only) and with --features +postgres-backend. No runtime behaviour change. + +Refs ADR 0002 D1, D2, D4." +``` + +If for any reason the manifest change must be reviewed separately (for +example, a security review of the sqlx version pin), split as: + +1. `cargo add` for sqlx and pgvector + manual feature line in Cargo.toml. + Build with default features will pass but `--features postgres-backend` + will fail (no module to satisfy the feature). This is acceptable for a + short-lived intermediate commit. +2. `storage/mod.rs` edits + `storage/postgres/mod.rs` creation. Both builds + pass. + +Default to the single-commit form unless asked to split. + +--- + +## Followups + +- `0002b-pool-and-config.md` adds `pool.rs`, `PostgresConfig`, and the + `vestige-mcp` `postgres-backend` pass-through feature. +- `0002c-migrations.md` adds `crates/vestige-core/migrations/postgres/` with + `0001_init.{up,down}.sql` and `0002_hnsw.{up,down}.sql`, plus + `postgres/migrations.rs` invoking `sqlx::migrate!`. `init()` body lands here. +- `0002d-store-impl-bodies.md` introduces the two `MemoryStoreError` variants + and replaces every `todo!()` in CRUD / scheduling / edges / domains / + registry with real `sqlx::query!` bodies. +- `0002e-hybrid-search.md` fills the three search bodies via the RRF query. diff --git a/docs/plans/0002b-pool-and-config.md b/docs/plans/0002b-pool-and-config.md new file mode 100644 index 0000000..9941413 --- /dev/null +++ b/docs/plans/0002b-pool-and-config.md @@ -0,0 +1,886 @@ +# Sub-plan 0002b -- Pool construction and VestigeConfig + +**Status**: Draft +**Master plan**: [0002-phase-2-postgres-backend.md](0002-phase-2-postgres-backend.md) +**ADR**: [0002-phase-2-execution.md](../adr/0002-phase-2-execution.md) +**Predecessor**: [0002a-skeleton-and-feature-gate.md](0002a-skeleton-and-feature-gate.md) + +--- + +## Context + +This sub-plan delivers two of the master plan's deliverables now that the +`0002a` skeleton has landed: + +- **D3** -- pool construction in + `crates/vestige-core/src/storage/postgres/pool.rs`. Replaces the `todo!()` + body of `PgMemoryStore::connect` with a real `PgPool` builder that reads a + `PostgresConfig`. Registry/migration calls remain `todo!()`; those are + filled in by sub-plans `0002c` (migrations) and `0002d` (store bodies + + registry). +- **D7** -- new module `crates/vestige-core/src/config.rs` containing + `VestigeConfig`, `StorageConfig`, `SqliteConfig`, `PostgresConfig`, + `EmbeddingsConfig`, plus a `ConfigError` enum and a loader that reads + `vestige.toml`. The loader is wired into `vestige-mcp` so the running + server picks SQLite or Postgres at startup based on the config file. + +After this sub-plan: + +- `cargo build` (default features, no `postgres-backend`) compiles and the + MCP server still defaults to SQLite when no `vestige.toml` is present. +- `cargo build --features postgres-backend` compiles, with + `PgMemoryStore::connect` now wiring through `pool.rs` (registry/migration + still `todo!()` until `0002c` and `0002d`). +- A `vestige.toml` example can be round-tripped through + `VestigeConfig::load` in a unit test. + +This sub-plan deliberately does NOT: + +- Add migrations (`0002c`). +- Fill in real CRUD/search bodies on `PgMemoryStore` (`0002d`, `0002e`). +- Add env-var override support (Phase 3 concern, called out in master plan + D7 behaviour notes). + +--- + +## Dependencies + +- `0002a-skeleton-and-feature-gate.md` must be merged. That sub-plan creates + `crates/vestige-core/src/storage/postgres/mod.rs` with: + - `PgMemoryStore` struct holding `pool: PgPool`. + - `PgMemoryStore::connect(url: &str, max_connections: u32) -> MemoryStoreResult` + body = `todo!()`. + - `PgMemoryStore::from_pool(pool: PgPool) -> MemoryStoreResult` + body = `todo!()`. + - The trait impl block with all methods routed to `todo!()`. + - The `postgres-backend` feature gate on the module declaration in + `storage/mod.rs`. + +This sub-plan extends those bodies and adds two siblings: `pool.rs` and +`registry.rs` (the latter is a stub here, real body in `0002d`). + +--- + +## Audit step (do this first) + +Before adding `config.rs`, confirm there is no existing top-level config +loader. Run from the repo root: + +```bash +rg -nF 'VestigeConfig' crates/ +rg -nF 'toml::from_str' crates/ +rg -n '#\[derive.*Deserialize.*\]' crates/vestige-core/src/ +``` + +If a `VestigeConfig` struct already exists from Phase 1, treat the "Config +module" section below as additive: extend the existing struct rather than +creating a new file. The cross-cut additions in that case are: + +1. Add the `StorageConfig` enum (gated and ungated branches). +2. Add `SqliteConfig`, `PostgresConfig`. +3. Add the `default_path()` helper if missing. +4. Add `ConfigError` if a different error enum is used today (rename/extend + instead of duplicating). + +As of the audit at the time of this writing, no `VestigeConfig` exists in +`vestige-core`. `directories::ProjectDirs` is already used in +`vestige-core/src/embeddings/local.rs` and in +`vestige-mcp/src/protocol/auth.rs`, so the `directories` crate is already a +workspace dependency -- no new dep there. + +--- + +## Cargo manifest additions + +Add `toml` to `vestige-core`. `serde` and `thiserror` are already present +from Phase 1; `directories` is already a transitive dep but we add it +explicitly so `default_path()` is supported. + +```bash +cd crates/vestige-core +cargo add toml@0.8 +cargo add directories@5 +``` + +No new deps on `vestige-mcp`; it already depends on `vestige-core`. + +`sqlx` is already added by `0002a` behind the `postgres-backend` feature +with `runtime-tokio`, `tls-rustls`, `postgres`, `uuid`, `chrono`, +`json`, `macros`, `migrate` features. The pool module only uses what is +already pulled in. + +--- + +## Config module + +**File**: `crates/vestige-core/src/config.rs` (new). +**Re-exported** from `crates/vestige-core/src/lib.rs` as `pub mod config;` plus +`pub use config::{VestigeConfig, StorageConfig, SqliteConfig, EmbeddingsConfig, ConfigError};` +and `#[cfg(feature = "postgres-backend")] pub use config::PostgresConfig;`. + +Full content: + +```rust +//! Vestige top-level configuration. +//! +//! Loaded from `~/.vestige/vestige.toml` by default; the path is overridable +//! via `VestigeConfig::load(Some(&path))`. Parsing uses serde + toml; the +//! `[storage]` section is internally-tagged on a `backend` field so a single +//! enum dispatch picks SQLite or Postgres. + +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +/// Top-level configuration as parsed from `vestige.toml`. +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(default, deny_unknown_fields)] +pub struct VestigeConfig { + pub embeddings: EmbeddingsConfig, + pub storage: StorageConfig, + /// Reserved for Phase 3. Empty in Phase 2. + pub server: ServerConfig, + /// Reserved for Phase 3. Empty in Phase 2. + pub auth: AuthConfig, +} + +/// Embedding provider selection. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmbeddingsConfig { + /// Provider key. Phase 2 ships `"fastembed"` only. + pub provider: String, + /// Model name. For fastembed this is e.g. `"nomic-ai/nomic-embed-text-v1.5"`. + pub model: String, +} + +impl Default for EmbeddingsConfig { + fn default() -> Self { + Self { + provider: "fastembed".to_string(), + model: crate::DEFAULT_EMBEDDING_MODEL.to_string(), + } + } +} + +/// Storage backend selection. Internally tagged on the `backend` field: +/// +/// ```toml +/// [storage] +/// backend = "sqlite" +/// +/// [storage.sqlite] +/// path = "/home/user/.vestige/vestige.db" +/// ``` +/// +/// or, when compiled with `--features postgres-backend`: +/// +/// ```toml +/// [storage] +/// backend = "postgres" +/// +/// [storage.postgres] +/// url = "postgres://vestige:secret@localhost:5432/vestige" +/// max_connections = 10 +/// acquire_timeout_secs = 30 +/// ``` +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "backend", rename_all = "lowercase", deny_unknown_fields)] +pub enum StorageConfig { + Sqlite(SqliteConfig), + #[cfg(feature = "postgres-backend")] + Postgres(PostgresConfig), +} + +impl Default for StorageConfig { + fn default() -> Self { + StorageConfig::Sqlite(SqliteConfig::default()) + } +} + +/// SQLite backend configuration. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SqliteConfig { + /// Path to the `vestige.db` file. If unset, the SqliteMemoryStore + /// constructor picks its platform default location. + #[serde(default)] + pub path: Option, +} + +impl Default for SqliteConfig { + fn default() -> Self { + Self { path: None } + } +} + +/// Postgres backend configuration. Only present when the `postgres-backend` +/// Cargo feature is enabled. +#[cfg(feature = "postgres-backend")] +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PostgresConfig { + /// `postgres://user:pass@host:port/db` -- forwarded to + /// `PgConnectOptions::from_str`. + pub url: String, + /// Pool size. Default `10`. + #[serde(default)] + pub max_connections: Option, + /// Acquire timeout in seconds. Default `30`. Set above 30 so + /// testcontainer-based test fixtures do not race. + #[serde(default)] + pub acquire_timeout_secs: Option, +} + +/// Reserved for Phase 3 (bind address, ports, TLS). +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ServerConfig {} + +/// Reserved for Phase 3 (API keys, claims). +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct AuthConfig {} + +/// Errors raised while locating, reading, or parsing `vestige.toml`. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("config io: {0}")] + Io(#[from] std::io::Error), + #[error("config toml: {0}")] + Toml(#[from] toml::de::Error), + #[error("config dir: could not locate user home")] + NoHome, + #[error("invalid config: {0}")] + Invalid(String), +} + +impl VestigeConfig { + /// Load config from `path` or from `default_path()` when `None`. + /// + /// Returns `VestigeConfig::default()` (SQLite + fastembed defaults) when + /// the file does not exist. Any other I/O or parse failure is surfaced + /// as a `ConfigError`. + pub fn load(path: Option<&Path>) -> Result { + let resolved: PathBuf = match path { + Some(p) => p.to_path_buf(), + None => Self::default_path()?, + }; + + match std::fs::read_to_string(&resolved) { + Ok(text) => { + let cfg: VestigeConfig = toml::from_str(&text)?; + cfg.validate()?; + Ok(cfg) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + Ok(Self::default()) + } + Err(e) => Err(ConfigError::Io(e)), + } + } + + /// `~/.vestige/vestige.toml`. The directory is NOT created here; loading + /// a missing file falls back to defaults. + pub fn default_path() -> Result { + let dirs = directories::ProjectDirs::from("", "vestige", "vestige") + .ok_or(ConfigError::NoHome)?; + // ProjectDirs::config_dir() varies per OS. Vestige convention is + // ~/.vestige/vestige.toml on Linux/macOS regardless of XDG, so we + // build the path off the home dir explicitly. + let home = directories::UserDirs::new() + .ok_or(ConfigError::NoHome)? + .home_dir() + .to_path_buf(); + let _ = dirs; // keep the dep wired; future Phase 3 may use it + Ok(home.join(".vestige").join("vestige.toml")) + } + + /// Light cross-field validation. Heavy validation (URL parsing, + /// directory existence) is left to the backend constructors. + fn validate(&self) -> Result<(), ConfigError> { + if self.embeddings.provider.is_empty() { + return Err(ConfigError::Invalid( + "embeddings.provider must not be empty".into(), + )); + } + if self.embeddings.model.is_empty() { + return Err(ConfigError::Invalid( + "embeddings.model must not be empty".into(), + )); + } + match &self.storage { + StorageConfig::Sqlite(_) => {} + #[cfg(feature = "postgres-backend")] + StorageConfig::Postgres(cfg) => { + if cfg.url.is_empty() { + return Err(ConfigError::Invalid( + "storage.postgres.url must not be empty".into(), + )); + } + } + } + Ok(()) + } +} +``` + +### Serde behaviour with `postgres-backend` off + +`StorageConfig` is generated by serde only for the variants that are +compiled in. When `postgres-backend` is off and the user writes: + +```toml +[storage] +backend = "postgres" + +[storage.postgres] +url = "..." +``` + +serde returns a `toml::de::Error` of the form +`unknown variant `postgres`, expected `sqlite``. That error path goes +through `From for ConfigError`, surfacing as +`ConfigError::Toml(..)`. The MCP server prints this once at startup and +exits with a non-zero code; there is no panic. + +To make the error friendlier we wrap that specific case in a clearer +message via a thin post-parse check. Add this small helper after parsing +in `load()`: + +```rust +// (Inside the Ok(text) arm in load(), wrapping the parse step.) +let cfg: VestigeConfig = match toml::from_str(&text) { + Ok(c) => c, + Err(e) => { + let msg = e.to_string(); + if msg.contains("unknown variant `postgres`") { + return Err(ConfigError::Invalid( + "storage.backend = \"postgres\" requires building with --features postgres-backend".into(), + )); + } + return Err(ConfigError::Toml(e)); + } +}; +``` + +This keeps the strict default deny_unknown_fields behaviour while giving the +user a one-line action item. + +--- + +## Pool module + +**File**: `crates/vestige-core/src/storage/postgres/pool.rs` (new). + +```rust +#![cfg(feature = "postgres-backend")] + +//! `PgPool` construction for the Postgres backend. +//! +//! Pool defaults follow ADR 0002 D2 + master plan D3: +//! - max_connections = 10 +//! - acquire_timeout = 30s (must exceed testcontainer warmup) +//! - idle_timeout = 600s +//! - max_lifetime = 1800s +//! - test_before_acquire = false (cheap queries; saves a roundtrip) + +use std::str::FromStr; +use std::time::Duration; + +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, PgPool}; + +use crate::config::PostgresConfig; +use crate::storage::memory_store::{MemoryStoreError, MemoryStoreResult}; + +const DEFAULT_MAX_CONNECTIONS: u32 = 10; +const DEFAULT_ACQUIRE_TIMEOUT_SECS: u64 = 30; +const IDLE_TIMEOUT_SECS: u64 = 600; +const MAX_LIFETIME_SECS: u64 = 1800; +const STATEMENT_CACHE_CAPACITY: usize = 256; + +/// Build a Postgres connection pool from a `PostgresConfig`. Does NOT run +/// migrations or stamp the embedding registry; those are the caller's job +/// (`PgMemoryStore::connect`). +pub async fn build_pool(cfg: &PostgresConfig) -> MemoryStoreResult { + let opts = PgConnectOptions::from_str(&cfg.url) + .map_err(MemoryStoreError::from)? + .application_name("vestige") + .statement_cache_capacity(STATEMENT_CACHE_CAPACITY) + .log_statements(tracing::log::LevelFilter::Debug); + + let max_conn = cfg.max_connections.unwrap_or(DEFAULT_MAX_CONNECTIONS); + let acquire = cfg + .acquire_timeout_secs + .unwrap_or(DEFAULT_ACQUIRE_TIMEOUT_SECS); + + let pool = PgPoolOptions::new() + .max_connections(max_conn) + .min_connections(0) + .acquire_timeout(Duration::from_secs(acquire)) + .idle_timeout(Some(Duration::from_secs(IDLE_TIMEOUT_SECS))) + .max_lifetime(Some(Duration::from_secs(MAX_LIFETIME_SECS))) + .test_before_acquire(false) + .connect_with(opts) + .await + .map_err(MemoryStoreError::from)?; + + Ok(pool) +} +``` + +### Wiring into `PgMemoryStore::connect` + +In `crates/vestige-core/src/storage/postgres/mod.rs`, replace the +`todo!()` body left by `0002a` for `connect` and `from_pool` with: + +```rust +// In crates/vestige-core/src/storage/postgres/mod.rs + +use sqlx::PgPool; + +use crate::config::PostgresConfig; +use crate::storage::memory_store::{MemoryStoreError, MemoryStoreResult}; + +mod pool; +mod registry; // see "Registry stub" section below + +pub struct PgMemoryStore { + pool: PgPool, +} + +impl PgMemoryStore { + /// Convenience constructor matching `SqliteMemoryStore::new` shape. + /// Takes a URL + pool size for the common case. + pub async fn connect(url: &str, max_connections: u32) -> MemoryStoreResult { + let cfg = PostgresConfig { + url: url.to_string(), + max_connections: Some(max_connections), + acquire_timeout_secs: None, + }; + Self::connect_with(&cfg).await + } + + /// Full-config constructor. + pub async fn connect_with(cfg: &PostgresConfig) -> MemoryStoreResult { + let pool = pool::build_pool(cfg).await?; + Self::from_pool(pool).await + } + + /// Construct from an already-built pool (used by tests and the migrate + /// CLI to share a pool across operations). + pub async fn from_pool(pool: PgPool) -> MemoryStoreResult { + // Migrations are added by 0002c. + // todo!("run sqlx::migrate! once 0002c lands") + registry::ensure_registry_stub(&pool).await?; + Ok(Self { pool }) + } +} +``` + +`connect_with` is the long-lived API; `connect` becomes a thin shim that +stays compatible with the master-plan-mandated signature. + +### Registry stub + +**File**: `crates/vestige-core/src/storage/postgres/registry.rs` (new, stub). + +```rust +#![cfg(feature = "postgres-backend")] + +//! Embedding registry. Real body lands in sub-plan 0002d. + +use sqlx::PgPool; + +use crate::storage::memory_store::MemoryStoreResult; + +/// Placeholder. Real implementation in 0002d reads/writes `embedding_model` +/// and stamps `ALTER TABLE memories ALTER COLUMN embedding TYPE vector($N)`. +pub(crate) async fn ensure_registry_stub(_pool: &PgPool) -> MemoryStoreResult<()> { + // Intentionally a no-op until 0002c lands the table + 0002d lands the + // real body. Leaving this as todo!() would crash the MCP server at + // startup the moment a user switches `backend = "postgres"`, which is + // not what we want for the build verification step in this sub-plan. + Ok(()) +} +``` + +The no-op keeps `cargo build --features postgres-backend` not just +compiling but also allowing the MCP server to *boot* against a Postgres +URL pointing at an already-migrated database (the local-dev-postgres-setup +docs cover bringing up such a DB by hand). Real init lands in `0002d`. + +--- + +## Error variants + +**File**: `crates/vestige-core/src/storage/memory_store.rs` (edit). + +The Phase 1 enum `MemoryStoreError` gains two feature-gated variants. These +were deferred in `0002a` and become required as soon as `pool.rs` calls +`.map_err(MemoryStoreError::from)` on `sqlx::Error`. + +```rust +// Within enum MemoryStoreError { ... } in memory_store.rs + +#[cfg(feature = "postgres-backend")] +#[error("postgres error: {0}")] +Postgres(#[from] sqlx::Error), + +#[cfg(feature = "postgres-backend")] +#[error("postgres migration error: {0}")] +Migrate(#[from] sqlx::migrate::MigrateError), +``` + +Both use thiserror's `#[from]` attribute so the `?` operator works in +`pool.rs`, the migrate module (`0002c`), and registry code (`0002d`). +Default-features build (no `postgres-backend`) sees neither variant; the +enum stays exhaustive on stable. + +If clippy fires on `non_exhaustive` due to the gated variants, add +`#[non_exhaustive]` on the enum. That has no caller-side effect since the +enum is constructed only inside the crate. + +--- + +## vestige-mcp wiring + +### Cargo feature passthrough + +**File**: `crates/vestige-mcp/Cargo.toml` (edit). + +Add a feature that forwards through to `vestige-core`. Default features in +`vestige-mcp` stay unchanged. + +```toml +[features] +default = ["embeddings", "vector-search"] +embeddings = ["vestige-core/embeddings"] +vector-search = ["vestige-core/vector-search"] +postgres-backend = ["vestige-core/postgres-backend"] +``` + +Verify with: + +```bash +cargo build -p vestige-mcp --features postgres-backend +``` + +### Backend dispatch at startup + +**File**: `crates/vestige-mcp/src/main.rs` (edit around the existing +`Storage::new(storage_path)` call -- see audit note above; in the current +worktree this is around line 285). + +The current code is roughly: + +```rust +let storage_path = match prepare_storage_path(config.data_dir) { ... }; +let storage = match Storage::new(storage_path) { ... }; +``` + +Replace that with a dispatch driven by `VestigeConfig`: + +```rust +use std::sync::Arc; + +use vestige_core::config::{StorageConfig, VestigeConfig}; +use vestige_core::storage::SqliteMemoryStore; +#[cfg(feature = "postgres-backend")] +use vestige_core::storage::postgres::PgMemoryStore; +use vestige_core::storage::MemoryStore; + +// Earlier: still call prepare_storage_path to honour --data-dir override. +let storage_path = match prepare_storage_path(config.data_dir.clone()) { ... }; + +// New: load vestige.toml (or fall back to defaults). +let vestige_cfg = match VestigeConfig::load(config.config_path.as_deref()) { + Ok(c) => c, + Err(e) => { + eprintln!("vestige: failed to load config: {e}"); + std::process::exit(2); + } +}; + +let storage: Arc = match &vestige_cfg.storage { + StorageConfig::Sqlite(sqlite_cfg) => { + // CLI flag --data-dir wins over the config file path. + let path = storage_path.clone().or_else(|| sqlite_cfg.path.clone()); + let s = SqliteMemoryStore::new(path).unwrap_or_else(|e| { + eprintln!("vestige: sqlite init failed: {e}"); + std::process::exit(3); + }); + Arc::new(s) + } + #[cfg(feature = "postgres-backend")] + StorageConfig::Postgres(pg_cfg) => { + let s = PgMemoryStore::connect_with(pg_cfg).await.unwrap_or_else(|e| { + eprintln!("vestige: postgres init failed: {e}"); + std::process::exit(3); + }); + Arc::new(s) + } +}; +``` + +The `config_path: Option` field on the local `Config` (or +clap-derived `Args`) struct must be added if not present; it accepts +`--config `. Default behaviour (no flag) goes through +`VestigeConfig::default_path()`. + +If the existing main wires `Storage` through a concrete type rather than +`Arc`, the dispatch above lives behind a helper: + +```rust +async fn build_store(cfg: &VestigeConfig, cli_path: Option) + -> Result, anyhow::Error> +{ ... } +``` + +and the caller chains `.into()` as needed. Phase 1 already moved +cognitive modules to `Arc` so this should be a pure +substitution; if a concrete-type holdout is found, fix it locally in this +sub-plan (separate commit) rather than punting. + +--- + +## vestige.toml example + +The canonical example to ship in `docs/` (Phase 2 docs land in `0002i`, +runbook), shown here for reference and used verbatim by the unit test +below. + +```toml +# vestige.toml -- top-level configuration +# +# Default location: ~/.vestige/vestige.toml +# Override: vestige-mcp --config /path/to/vestige.toml + +[embeddings] +provider = "fastembed" +model = "nomic-ai/nomic-embed-text-v1.5" + +# --- SQLite backend (default) --- +[storage] +backend = "sqlite" + +[storage.sqlite] +path = "/home/user/.vestige/vestige.db" + +# --- Postgres backend (requires --features postgres-backend) --- +# [storage] +# backend = "postgres" +# +# [storage.postgres] +# url = "postgres://vestige:secret@localhost:5432/vestige" +# max_connections = 10 +# acquire_timeout_secs = 30 + +[server] +# Reserved for Phase 3 (bind address, ports, TLS). + +[auth] +# Reserved for Phase 3 (API keys, claims). +``` + +--- + +## Verification + +Run all of these from the repo root. The first three are the gates that +must pass before this sub-plan is considered done. + +### 1. Default build (no Postgres) + +```bash +cargo build -p vestige-core +cargo build -p vestige-mcp +cargo test -p vestige-core --lib +``` + +Expected: clean build. `VestigeConfig::default()` selects SQLite; the MCP +server boots the same way it did pre-sub-plan. + +### 2. Postgres-feature build + +```bash +cargo build -p vestige-core --features postgres-backend +cargo build -p vestige-mcp --features postgres-backend +``` + +Expected: clean build. `PgMemoryStore::connect_with` resolves to +`pool::build_pool` + `registry::ensure_registry_stub`; no `todo!()` is +reachable on the build path. `connect` and `from_pool` are exported. + +### 3. Clippy across both feature sets + +```bash +cargo clippy -p vestige-core -- -D warnings +cargo clippy -p vestige-core --features postgres-backend -- -D warnings +cargo clippy -p vestige-mcp --features postgres-backend -- -D warnings +``` + +### 4. Unit test: round-trip the example + +Add this test to `crates/vestige-core/src/config.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + const EXAMPLE_SQLITE: &str = r#" +[embeddings] +provider = "fastembed" +model = "nomic-ai/nomic-embed-text-v1.5" + +[storage] +backend = "sqlite" + +[storage.sqlite] +path = "/home/user/.vestige/vestige.db" +"#; + + #[cfg(feature = "postgres-backend")] + const EXAMPLE_POSTGRES: &str = r#" +[embeddings] +provider = "fastembed" +model = "nomic-ai/nomic-embed-text-v1.5" + +[storage] +backend = "postgres" + +[storage.postgres] +url = "postgres://vestige:secret@localhost:5432/vestige" +max_connections = 10 +acquire_timeout_secs = 30 +"#; + + #[test] + fn parses_sqlite_example() { + let cfg: VestigeConfig = toml::from_str(EXAMPLE_SQLITE).expect("parse"); + match cfg.storage { + StorageConfig::Sqlite(s) => assert!(s.path.is_some()), + #[cfg(feature = "postgres-backend")] + StorageConfig::Postgres(_) => panic!("wrong variant"), + } + assert_eq!(cfg.embeddings.provider, "fastembed"); + } + + #[cfg(feature = "postgres-backend")] + #[test] + fn parses_postgres_example() { + let cfg: VestigeConfig = toml::from_str(EXAMPLE_POSTGRES).expect("parse"); + match cfg.storage { + StorageConfig::Postgres(p) => { + assert_eq!(p.url, "postgres://vestige:secret@localhost:5432/vestige"); + assert_eq!(p.max_connections, Some(10)); + assert_eq!(p.acquire_timeout_secs, Some(30)); + } + StorageConfig::Sqlite(_) => panic!("wrong variant"), + } + } + + #[cfg(not(feature = "postgres-backend"))] + #[test] + fn rejects_postgres_when_feature_off() { + let toml_text = r#" +[storage] +backend = "postgres" + +[storage.postgres] +url = "postgres://x/y" +"#; + let res: Result = toml::from_str(toml_text); + assert!(res.is_err(), "must fail without postgres-backend feature"); + } + + #[test] + fn defaults_pick_sqlite() { + let cfg = VestigeConfig::default(); + assert!(matches!(cfg.storage, StorageConfig::Sqlite(_))); + } + + #[test] + fn load_missing_file_returns_default() { + let tmp = std::env::temp_dir().join("vestige-no-such-file.toml"); + let _ = std::fs::remove_file(&tmp); + let cfg = VestigeConfig::load(Some(&tmp)).expect("missing file is OK"); + assert!(matches!(cfg.storage, StorageConfig::Sqlite(_))); + } + + #[test] + fn load_roundtrip_from_disk() { + let tmp = std::env::temp_dir().join("vestige-roundtrip.toml"); + std::fs::write(&tmp, EXAMPLE_SQLITE).unwrap(); + let cfg = VestigeConfig::load(Some(&tmp)).expect("load"); + assert!(matches!(cfg.storage, StorageConfig::Sqlite(_))); + let _ = std::fs::remove_file(&tmp); + } +} +``` + +Run: + +```bash +cargo test -p vestige-core --lib config:: +cargo test -p vestige-core --lib config:: --features postgres-backend +``` + +### 5. Smoke: server boots with default config + +```bash +# default build, no vestige.toml on disk +cargo run -p vestige-mcp -- --help +# should print help, no panic +``` + +--- + +## Acceptance criteria + +- [ ] `cargo build -p vestige-core` (default features) succeeds. +- [ ] `cargo build -p vestige-core --features postgres-backend` succeeds. +- [ ] `cargo build -p vestige-mcp` (default features) succeeds. +- [ ] `cargo build -p vestige-mcp --features postgres-backend` succeeds. +- [ ] `cargo clippy` with and without `postgres-backend` is clean on both + crates. +- [ ] `crates/vestige-core/src/config.rs` exists, exposes + `VestigeConfig`, `StorageConfig`, `SqliteConfig`, `EmbeddingsConfig`, + `ConfigError`, plus `PostgresConfig` when the feature is on. +- [ ] `VestigeConfig::load(None)` returns `Ok(default)` when + `~/.vestige/vestige.toml` is missing. +- [ ] `VestigeConfig::load(Some(&path))` round-trips both the SQLite and + Postgres example blocks above. +- [ ] With `postgres-backend` off, parsing `backend = "postgres"` returns + a clear `ConfigError::Invalid` mentioning the feature flag, NOT a + panic. +- [ ] `crates/vestige-core/src/storage/postgres/pool.rs` exists, + implementing `build_pool(&PostgresConfig) -> MemoryStoreResult` + with the documented defaults. +- [ ] `PgMemoryStore::connect`, `connect_with`, and `from_pool` all wire + through `pool::build_pool`. None of them is `todo!()`. The registry + step is a no-op stub documented as filled in by `0002d`. +- [ ] `MemoryStoreError::Postgres(sqlx::Error)` and + `MemoryStoreError::Migrate(sqlx::migrate::MigrateError)` exist + behind `#[cfg(feature = "postgres-backend")]` with `#[from]`. +- [ ] `vestige-mcp` has a `postgres-backend` feature that forwards to + `vestige-core/postgres-backend`. +- [ ] `vestige-mcp/src/main.rs` selects SQLite vs Postgres at startup + based on `VestigeConfig`. SQLite is the default when no config file + is present. +- [ ] Unit tests in the "Verification" section pass on both feature sets. + +--- + +## Out of scope (handled by other sub-plans) + +- Migrations (`crates/vestige-core/migrations/postgres/*.sql`) -- `0002c`. +- Real `PgMemoryStore` CRUD/search/scheduling/edges bodies -- `0002d`, + `0002e`. +- `ensure_registry` real body with `ALTER COLUMN TYPE vector(N)` -- `0002d`. +- `vestige migrate --from sqlite --to postgres` CLI -- `0002f`. +- Re-embed flow -- `0002g`. +- Env-var override (`VESTIGE_POSTGRES_URL`, etc.) -- Phase 3. +- RLS, multi-tenant column population -- Phase 3. diff --git a/docs/plans/0002c-migrations.md b/docs/plans/0002c-migrations.md new file mode 100644 index 0000000..ef8e35c --- /dev/null +++ b/docs/plans/0002c-migrations.md @@ -0,0 +1,1119 @@ +# Phase 2 Sub-plan 0002c: sqlx Migrations + +**Status**: Draft +**Depends on**: `0002a-skeleton-and-feature-gate.md` (PgMemoryStore skeleton, error variants), `0002b-pool-and-config.md` (PgPool builder, PostgresConfig) +**Related**: docs/adr/0002-phase-2-execution.md (D7 multi-tenancy reservation, D8 codebase column), docs/plans/0002-phase-2-postgres-backend.md (D4 master SQL), docs/plans/local-dev-postgres-setup.md (local cluster + role + DB) + +--- + +## Context + +This sub-plan covers Phase 2 deliverable D4 (sqlx migration files under +`crates/vestige-core/migrations/postgres/`) PLUS the schema additions decided +in ADR 0002: + +- D7 -- multi-tenancy reservation: `users`, `groups`, `group_memberships` + tables, plus `owner_user_id`, `visibility`, `shared_with_groups` columns on + `knowledge_nodes`. Phase 3 fills these in; Phase 2 just reserves them so the auth + filter is later additive instead of an online migration over a populated, + HNSW-indexed table. +- D8 -- `codebase` promoted to a first-class indexed column on `knowledge_nodes`. + +This sub-plan also adds the parity SQLite migration (V15) that mirrors D7 + +D8 on the SQLite side, so a single-user SQLite deployment sees the same +columns (with stand-in defaults). + +After this sub-plan lands: + +- A fresh Postgres database, with the `vestige` role from the local-dev + setup, can be initialized by running `sqlx::migrate!` against + `crates/vestige-core/migrations/postgres/`, plus one programmatic + `register_model` call before the HNSW migration. +- A fresh SQLite database initialized by `apply_migrations` lands at + schema_version = 15 with the new tables and columns present. +- `PgMemoryStore::connect` wires the migrator into the connect path + (pool build -> migrator up-to v1 -> register_model -> migrator up-to v2). +- The SQLite test suite continues to pass. +- No `sqlx::query!` calls are introduced yet; the offline `.sqlx/` cache is + filled out in `0002d-store-impl-bodies.md`. + +The deliverable is purely schema. No query bodies, no row-mapping, no search. + +--- + +## Postgres migration files + +Layout, relative to repo root: + +``` +crates/vestige-core/migrations/postgres/ + 0001_init.up.sql + 0001_init.down.sql + 0002_hnsw.up.sql + 0002_hnsw.down.sql +``` + +The `migrations/postgres/` directory is sibling-of-`src/`, not under `src/`, +because `sqlx::migrate!` and `sqlx-cli` both look for a path relative to +`CARGO_MANIFEST_DIR`. The directory is committed. + +### 0001_init.up.sql + +Creates extensions, the multi-tenancy tables (D7), the embedding registry, +the domains catalogue, the `knowledge_nodes` table (with D7 + D8 columns merged in), +the FSRS scheduling and edges tables, the review-events log, all non-vector +indexes, the updated_at trigger, and the bootstrap `local` user row. + +The HNSW vector index is deliberately NOT here -- it requires a typmod on +`knowledge_nodes.embedding`, which is stamped by `register_model` at runtime. See +the "HNSW typmod ordering" section below. + +```sql +-- crates/vestige-core/migrations/postgres/0001_init.up.sql +-- +-- Phase 2 initial schema for the Postgres backend. +-- Includes D7 multi-tenancy reservation (users/groups/group_memberships, +-- owner_user_id/visibility/shared_with_groups on knowledge_nodes) and D8 +-- (codebase first-class column on knowledge_nodes). +-- +-- The HNSW index on knowledge_nodes.embedding lives in 0002_hnsw.up.sql; it +-- requires the column typmod to be stamped first by register_model(). + +-- Extensions ---------------------------------------------------------------- + +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS vector; + +-- Embedding model registry -------------------------------------------------- +-- Mirrors the SQLite table created in Phase 1 V14. +-- One logical row enforced by CHECK (id = 1). + +CREATE TABLE embedding_model ( + id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1), + name TEXT NOT NULL, + dimension INTEGER NOT NULL CHECK (dimension > 0), + hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Domains catalogue --------------------------------------------------------- +-- Populated by the Phase 4 DomainClassifier. Phase 2 creates the empty +-- table so list/get/upsert/delete work uniformly against both backends. + +CREATE TABLE domains ( + id TEXT PRIMARY KEY, + label TEXT NOT NULL, + centroid vector, + top_terms TEXT[] NOT NULL DEFAULT '{}', + memory_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +-- Multi-tenancy (D7) -------------------------------------------------------- +-- Reserved in Phase 2; populated in Phase 3. +-- Single bootstrap user inserted at the bottom of this file. + +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE TABLE groups ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb +); + +CREATE TABLE group_memberships ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + group_id UUID NOT NULL REFERENCES groups(id) ON DELETE CASCADE, + role TEXT NOT NULL DEFAULT 'member', + joined_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, group_id), + CHECK (role IN ('member', 'admin')) +); + +-- Core knowledge_nodes table ------------------------------------------------- +-- Original Phase 2 columns merged with D7 (owner_user_id, visibility, +-- shared_with_groups) and D8 (codebase). + +CREATE TABLE knowledge_nodes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Content + content TEXT NOT NULL, + node_type TEXT NOT NULL DEFAULT 'general', + tags TEXT[] NOT NULL DEFAULT '{}', + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- Phase 4 emergent domains (Phase 2 leaves empty) + domains TEXT[] NOT NULL DEFAULT '{}', + domain_scores JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- Embedding (typmod stamped by register_model before 0002_hnsw runs) + embedding vector, + + -- D8: first-class codebase column for high-frequency scoped queries + codebase TEXT, + + -- D7: multi-tenancy reservation. Defaults make Phase 2 single-user + -- behaviour identical to Phase 1. + owner_user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000001' + REFERENCES users(id), + visibility TEXT NOT NULL DEFAULT 'private', + shared_with_groups UUID[] NOT NULL DEFAULT '{}', + + -- Timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + -- Generated full-text search vector. Phase 2 uses websearch_to_tsquery + -- against this column at query time (see 0002e-hybrid-search.md). + search_vec TSVECTOR GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(content, '')), 'A') || + setweight(to_tsvector('english', coalesce(node_type, '')), 'B') || + setweight(to_tsvector('english', coalesce(array_to_string(tags, ' '), '')), 'C') + ) STORED, + + -- Visibility tri-state CHECK constraint. See "Visibility CHECK + -- constraint" section below for the cardinality variant we + -- intentionally do NOT add yet. + CHECK (visibility IN ('private', 'group', 'public')) +); + +-- FSRS scheduling state (1:1 with knowledge_nodes) --------------------------- +-- +-- Note: the FK column is named `memory_id` (not `node_id`) to match the +-- Phase 1 SQLite trait surface: `SchedulingState { memory_id: Uuid, ... }` +-- and `get_scheduling(memory_id: Uuid)` / `update_scheduling(&state)`. The +-- table is `knowledge_nodes` but the Rust identifier remained `memory_id` +-- across Phase 1 and is preserved here so both backends speak the same +-- language at the trait boundary. + +CREATE TABLE scheduling ( + memory_id UUID PRIMARY KEY REFERENCES knowledge_nodes(id) ON DELETE CASCADE, + stability DOUBLE PRECISION NOT NULL DEFAULT 0.0, + difficulty DOUBLE PRECISION NOT NULL DEFAULT 0.0, + retrievability DOUBLE PRECISION NOT NULL DEFAULT 1.0, + last_review TIMESTAMPTZ, + next_review TIMESTAMPTZ, + reps INTEGER NOT NULL DEFAULT 0, + lapses INTEGER NOT NULL DEFAULT 0 +); + +-- Spreading activation graph edges ------------------------------------------ + +CREATE TABLE edges ( + source_id UUID NOT NULL REFERENCES knowledge_nodes(id) ON DELETE CASCADE, + target_id UUID NOT NULL REFERENCES knowledge_nodes(id) ON DELETE CASCADE, + edge_type TEXT NOT NULL DEFAULT 'related', + weight DOUBLE PRECISION NOT NULL DEFAULT 1.0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source_id, target_id, edge_type) +); + +-- FSRS review event log (append-only; Phase 5 federation reads) ------------- + +CREATE TABLE review_events ( + id BIGSERIAL PRIMARY KEY, + memory_id UUID NOT NULL REFERENCES knowledge_nodes(id) ON DELETE CASCADE, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + rating SMALLINT NOT NULL, + prior_state JSONB NOT NULL, + new_state JSONB NOT NULL +); + +-- Indexes ------------------------------------------------------------------- + +-- knowledge_nodes: full-text, arrays, hot scalar columns, D7+D8 access patterns +CREATE INDEX idx_knowledge_nodes_fts ON knowledge_nodes USING GIN (search_vec); +CREATE INDEX idx_knowledge_nodes_domains ON knowledge_nodes USING GIN (domains); +CREATE INDEX idx_knowledge_nodes_tags ON knowledge_nodes USING GIN (tags); +CREATE INDEX idx_knowledge_nodes_node_type ON knowledge_nodes (node_type); +CREATE INDEX idx_knowledge_nodes_created ON knowledge_nodes (created_at); +CREATE INDEX idx_knowledge_nodes_updated ON knowledge_nodes (updated_at); + +-- D7 visibility filter (Phase 3 query: WHERE owner_user_id = $me ...) +CREATE INDEX idx_knowledge_nodes_owner ON knowledge_nodes (owner_user_id); +CREATE INDEX idx_knowledge_nodes_shared_groups ON knowledge_nodes USING GIN (shared_with_groups); + +-- D8 codebase scoping (Phase 4 HDBSCAN per-repo, sharing rules in Phase 4). +-- Partial index keeps the index small in single-user mode where most rows +-- never set a codebase. +CREATE INDEX idx_knowledge_nodes_codebase + ON knowledge_nodes (codebase) + WHERE codebase IS NOT NULL; + +-- scheduling: hot lookup paths for FSRS pickers +CREATE INDEX idx_scheduling_next_review ON scheduling (next_review); +CREATE INDEX idx_scheduling_last_review ON scheduling (last_review); + +-- edges: bidirectional + edge type +CREATE INDEX idx_edges_target ON edges (target_id); +CREATE INDEX idx_edges_source ON edges (source_id); +CREATE INDEX idx_edges_type ON edges (edge_type); + +-- review_events: per-memory and chronological +CREATE INDEX idx_review_events_memory ON review_events (memory_id); +CREATE INDEX idx_review_events_ts ON review_events (timestamp); + +-- users / groups: unique handle indexes are implicit; add nothing extra. +-- group_memberships: primary key (user_id, group_id) is the access path. + +-- updated_at trigger on knowledge_nodes ---------------------------------------- + +CREATE OR REPLACE FUNCTION knowledge_nodes_set_updated_at() RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_knowledge_nodes_updated_at +BEFORE UPDATE ON knowledge_nodes +FOR EACH ROW EXECUTE FUNCTION knowledge_nodes_set_updated_at(); + +-- Bootstrap rows ------------------------------------------------------------ +-- Single 'local' user matches the default on knowledge_nodes.owner_user_id so +-- single-user Phase 2 inserts never violate the FK. + +INSERT INTO users (id, handle, display_name) + VALUES ('00000000-0000-0000-0000-000000000001', 'local', 'Local User'); +``` + +### 0001_init.down.sql + +Reverse-dependency drop order. Trigger and function first, then indexes, +then tables, then extensions are left alone (extensions are global; we do +not drop them in a `down`). + +```sql +-- crates/vestige-core/migrations/postgres/0001_init.down.sql + +DROP TRIGGER IF EXISTS trg_knowledge_nodes_updated_at ON knowledge_nodes; +DROP FUNCTION IF EXISTS knowledge_nodes_set_updated_at(); + +-- knowledge_nodes indexes +DROP INDEX IF EXISTS idx_knowledge_nodes_codebase; +DROP INDEX IF EXISTS idx_knowledge_nodes_shared_groups; +DROP INDEX IF EXISTS idx_knowledge_nodes_owner; +DROP INDEX IF EXISTS idx_knowledge_nodes_updated; +DROP INDEX IF EXISTS idx_knowledge_nodes_created; +DROP INDEX IF EXISTS idx_knowledge_nodes_node_type; +DROP INDEX IF EXISTS idx_knowledge_nodes_tags; +DROP INDEX IF EXISTS idx_knowledge_nodes_domains; +DROP INDEX IF EXISTS idx_knowledge_nodes_fts; + +-- scheduling indexes +DROP INDEX IF EXISTS idx_scheduling_last_review; +DROP INDEX IF EXISTS idx_scheduling_next_review; + +-- edges indexes +DROP INDEX IF EXISTS idx_edges_type; +DROP INDEX IF EXISTS idx_edges_source; +DROP INDEX IF EXISTS idx_edges_target; + +-- review_events indexes +DROP INDEX IF EXISTS idx_review_events_ts; +DROP INDEX IF EXISTS idx_review_events_memory; + +-- Tables, reverse dependency order +DROP TABLE IF EXISTS review_events; +DROP TABLE IF EXISTS edges; +DROP TABLE IF EXISTS scheduling; +DROP TABLE IF EXISTS knowledge_nodes; +DROP TABLE IF EXISTS group_memberships; +DROP TABLE IF EXISTS groups; +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS domains; +DROP TABLE IF EXISTS embedding_model; + +-- Extensions are intentionally NOT dropped. They may be in use by other +-- databases on the cluster; dropping them is an admin choice. +``` + +### 0002_hnsw.up.sql + +Single statement; separated from 0001 so reembed (sub-plan 0002g) can +DROP/CREATE this index in isolation without touching anything else. + +```sql +-- crates/vestige-core/migrations/postgres/0002_hnsw.up.sql +-- +-- HNSW index on knowledge_nodes.embedding. This migration runs AFTER +-- register_model() has stamped the typmod via: +-- +-- ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector($N) +-- +-- where $N is the embedder's dimension(). Without the typmod, pgvector +-- rejects HNSW creation with: +-- +-- ERROR: column does not have dimensions +-- +-- See "HNSW typmod ordering" in 0002c-migrations.md and the connect() +-- sequence in 0002a-skeleton-and-feature-gate.md / 0002d-store-impl-bodies.md. +-- +-- Operator class: vector_cosine_ops -> distance operator `<=>`. +-- Build parameters: m = 16, ef_construction = 64 (pgvector defaults; see +-- the master plan 0002 D5 RRF discussion for the rationale). + +CREATE INDEX idx_knowledge_nodes_embedding_hnsw + ON knowledge_nodes USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64); +``` + +### 0002_hnsw.down.sql + +```sql +-- crates/vestige-core/migrations/postgres/0002_hnsw.down.sql + +DROP INDEX IF EXISTS idx_knowledge_nodes_embedding_hnsw; +``` + +--- + +## HNSW typmod ordering + +pgvector's HNSW index requires the indexed column to have a typmod (fixed +dimension). `vector` (unconstrained) is rejected; `vector(768)` is accepted. +We cannot bake the dimension into 0001 because the dimension is an +embedder-determined runtime value -- different builds may use different +embedders. + +This forces an ordering: + +1. Apply migration 0001 (creates `knowledge_nodes.embedding vector`, no typmod). +2. Connect, decide which embedder is in use, run + `ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector($N)` + inside `register_model`. +3. Apply migration 0002 (creates HNSW; succeeds because the column now has + a typmod). + +`sqlx::migrate!("...")` runs ALL pending migrations in a single call. It is +not designed to pause between two specific migrations so application code +can interleave a runtime DDL step. So we have two options: + +**Option A: Migration 0002 lives outside the sqlx migrations directory.** +Keep `0001_init.{up,down}.sql` only in `migrations/postgres/`; promote +`0002_hnsw.up.sql` to a Rust `include_str!` constant or a separate +`migrations/postgres-hnsw/` directory, run it manually by `PgMemoryStore` +after `register_model`. + +Pros: simple control flow, one `sqlx::migrate!()` call. +Cons: `sqlx_migrations` table does not record 0002, so `sqlx-cli migrate +info` lies. The HNSW index becomes "shadow" schema state from sqlx's POV. +Reembed (sub-plan 0002g) has to also know about this file outside the +normal migrations directory. + +**Option B (chosen): Both migrations live in the directory; the runner +splits them programmatically.** Use `sqlx::migrate::Migrator::new` to load +the directory and call its `run_to(...)` method with a specific version. + +```rust +// crates/vestige-core/src/storage/postgres/migrations.rs +use sqlx::migrate::Migrator; +use sqlx::PgPool; + +use crate::storage::error::MemoryStoreResult; + +/// Embedded migrator. Loaded at compile time from the migrations directory +/// alongside the crate. Path is relative to CARGO_MANIFEST_DIR. +static MIGRATOR: Migrator = sqlx::migrate!("./migrations/postgres"); + +/// Run migrations up to (and including) version 1. +/// +/// This must be called BEFORE register_model so the schema (knowledge_nodes table, +/// embedding_model registry, etc.) exists for register_model to write into +/// and to ALTER. +pub(crate) async fn run_pre_register(pool: &PgPool) -> MemoryStoreResult<()> { + MIGRATOR.run_to(pool, 1).await?; + Ok(()) +} + +/// Run any remaining migrations (currently: HNSW = version 2). +/// +/// Called AFTER register_model has stamped the embedding column's typmod. +pub(crate) async fn run_post_register(pool: &PgPool) -> MemoryStoreResult<()> { + MIGRATOR.run(pool).await?; + Ok(()) +} +``` + +Pros: sqlx is the only source of truth for migration version state; +`sqlx-cli migrate info` is accurate; reembed re-applies 0002 by name; future +migrations slot in normally. +Cons: relies on `Migrator::run_to`, which exists in sqlx 0.7+ and is the +documented API for staged migration. If that API ever disappears we fall +back to Option A. + +Decision: Option B. `Migrator::run_to(target_version)` is stable in sqlx +0.8. Sub-plan 0002a's `MemoryStoreError` already gains +`#[from] sqlx::migrate::MigrateError` to absorb whichever error variant +this surfaces. + +The `connect()` sequence in sub-plan 0002d will therefore look like: + +```rust +// Sketch only; full body lives in 0002d-store-impl-bodies.md. +pub async fn connect(url: &str, max_connections: u32) -> MemoryStoreResult { + let pool = crate::storage::postgres::pool::build(url, max_connections).await?; + crate::storage::postgres::migrations::run_pre_register(&pool).await?; + let store = Self { pool }; + // register_model is called by the cognitive engine bootstrap, NOT here. + // After it runs, the engine calls store.finalize_schema() which calls + // run_post_register. Same shape as SqliteMemoryStore. + Ok(store) +} + +pub async fn finalize_schema(&self) -> MemoryStoreResult<()> { + crate::storage::postgres::migrations::run_post_register(&self.pool).await +} +``` + +`finalize_schema` lands in 0002d; this sub-plan only ships `run_pre_register` +and `run_post_register` plus their wiring into `connect`. + +--- + +## SQLite V15 migration + +The Phase 1 SQLite schema lives in `crates/vestige-core/src/storage/migrations.rs` +as a `MIGRATIONS` slice. V14 is the latest entry. V15 is appended to mirror +D7 (multi-tenancy) and D8 (codebase) on the SQLite side, so a single-user +SQLite deployment sees the same surface area. + +Constraints versus the Postgres migration: + +- No `UUID[]` -- `shared_with_groups` is a TEXT JSON-encoded `'[]'`. +- No `gen_random_uuid()` -- the bootstrap user UUID is a literal. +- No partial indexes for our chosen pattern (SQLite *does* support partial + indexes since 3.8; we use one for `codebase` to match Postgres). +- No `ADD COLUMN IF NOT EXISTS` -- the V15 column additions are split into a + `MIGRATION_V15_ALTER_COLUMNS` slice exactly like V14 did, so the migration + is idempotent on replay. + +### Insertion point in migrations.rs + +Add to the `MIGRATIONS` slice immediately after V14: + +```rust +// In MIGRATIONS slice, after the V14 entry: +Migration { + version: 15, + description: "ADR 0002 D7+D8: multi-tenancy reservation + codebase column", + up: MIGRATION_V15_UP, +}, +``` + +### V15 SQL + +```rust +/// V15: ADR 0002 D7 + D8. +/// +/// D7 reserves users / groups / group_memberships and owner_user_id / +/// visibility / shared_with_groups columns on knowledge_nodes. Single-user +/// SQLite mode never reads these (the trait surface ignores visibility +/// because there is exactly one user) but they exist so Phase 3 does not +/// have to ALTER a populated table. +/// +/// D8 adds a first-class `codebase` column. +/// +/// Like V14, the ALTER TABLE statements are split into +/// MIGRATION_V15_ALTER_COLUMNS because SQLite has no ADD COLUMN IF NOT EXISTS. +const MIGRATION_V15_UP: &str = r#" +-- Migration V15: multi-tenancy reservation + codebase column. + +-- 1. Users / groups / group_memberships ----------------------------------- +-- Mirrors the Postgres D7 tables. Single bootstrap user inserted below. + +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY, + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + created_at TEXT NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS groups ( + id TEXT PRIMARY KEY, + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + created_at TEXT NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' +); + +CREATE TABLE IF NOT EXISTS group_memberships ( + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + group_id TEXT NOT NULL REFERENCES groups(id) ON DELETE CASCADE, + role TEXT NOT NULL DEFAULT 'member' CHECK (role IN ('member', 'admin')), + joined_at TEXT NOT NULL, + PRIMARY KEY (user_id, group_id) +); + +-- 2. Bootstrap 'local' user. Same UUID as the Postgres default so a future +-- portable export from SQLite -> import to Postgres preserves owner_user_id. + +INSERT OR IGNORE INTO users (id, handle, display_name, created_at) + VALUES ('00000000-0000-0000-0000-000000000001', 'local', 'Local User', + datetime('now')); + +-- 3. Per-memory column additions are applied separately by the migration +-- runner (see MIGRATION_V15_ALTER_COLUMNS). + +-- 4. Indexes that do not depend on the new columns. Index creation on the +-- new knowledge_nodes columns is done after MIGRATION_V15_ALTER_COLUMNS +-- runs (see runner glue below). + +UPDATE schema_version SET version = 15, applied_at = datetime('now'); +"#; + +/// V15 column additions. SQLite has no ADD COLUMN IF NOT EXISTS, so the +/// runner skips "duplicate column" errors per statement (same shape as V14). +pub const MIGRATION_V15_ALTER_COLUMNS: &[&str] = &[ + // D7 columns. Defaults match the Postgres side. shared_with_groups is + // a JSON-encoded array. + "ALTER TABLE knowledge_nodes ADD COLUMN owner_user_id TEXT NOT NULL DEFAULT '00000000-0000-0000-0000-000000000001'", + "ALTER TABLE knowledge_nodes ADD COLUMN visibility TEXT NOT NULL DEFAULT 'private'", + "ALTER TABLE knowledge_nodes ADD COLUMN shared_with_groups TEXT NOT NULL DEFAULT '[]'", + // D8 column. + "ALTER TABLE knowledge_nodes ADD COLUMN codebase TEXT", +]; + +/// V15 index creation. Runs AFTER the ALTER COLUMN statements succeed. +/// Kept as a separate batch so a partial replay (columns already there, +/// indexes not yet) still creates the indexes. +const MIGRATION_V15_INDEXES: &str = r#" +CREATE INDEX IF NOT EXISTS idx_nodes_owner_user_id ON knowledge_nodes(owner_user_id); +CREATE INDEX IF NOT EXISTS idx_nodes_codebase ON knowledge_nodes(codebase) WHERE codebase IS NOT NULL; +-- shared_with_groups is TEXT JSON in SQLite; we do not add a GIN-equivalent +-- index. Phase 3 lookups on the SQLite side will scan; SQLite never serves +-- the multi-user query path in Phase 2-4 anyway. +"#; +``` + +### Runner glue + +Extend `apply_migrations` in `migrations.rs` to recognise V15 the same way +it recognises V14: + +```rust +// Existing pattern for V14 lives in apply_migrations; extend it: +if migration.version == 15 { + for stmt in MIGRATION_V15_ALTER_COLUMNS { + if let Err(e) = conn.execute_batch(stmt) { + let msg = e.to_string(); + if msg.contains("duplicate column name") { + tracing::debug!( + "V15 ALTER TABLE skipped (column already exists): {}", + msg + ); + } else { + return Err(e); + } + } + } + // Indexes run *after* the columns exist. + conn.execute_batch(MIGRATION_V15_INDEXES)?; +} + +// Then the normal: +conn.execute_batch(migration.up)?; +``` + +Order of operations on a fresh in-memory DB: + +1. V1 - V14 run as before. +2. V15: column ALTERs run first (so MIGRATION_V15_INDEXES sees them). +3. V15 main body creates users/groups/group_memberships and the bootstrap row. +4. V15 indexes batch runs. +5. schema_version advances to 15. + +This intentionally mirrors how V14 handles its ALTER + index pair. + +### Existing-data backfill + +Existing SQLite databases (every Phase 1 deployment) have populated +`knowledge_nodes` rows. The V15 ALTER COLUMN ADD COLUMN statements assign +the default values to every existing row: + +- `owner_user_id` -> `'00000000-0000-0000-0000-000000000001'` +- `visibility` -> `'private'` +- `shared_with_groups` -> `'[]'` +- `codebase` -> NULL + +Phase 2 leaves these defaults in place. Phase 3 owns the migration story +for populating real owner UUIDs and visibility values. + +--- + +## Rust wrapper + +Single file: + +```rust +// crates/vestige-core/src/storage/postgres/migrations.rs +// +// sqlx::migrate! wrapper for the Postgres backend. +// +// We split the migration apply into two halves around register_model: +// - run_pre_register: applies everything up to and including version 1 +// (schema, indexes, bootstrap row). Safe to call on a +// fresh DB. +// - run_post_register: applies the remainder (currently: 0002_hnsw, which +// needs the embedding column typmod stamped first). +// +// See docs/plans/0002c-migrations.md "HNSW typmod ordering" for why this +// split exists. + +#![cfg(feature = "postgres-backend")] + +use sqlx::PgPool; +use sqlx::migrate::Migrator; + +use crate::storage::error::MemoryStoreResult; + +/// Embedded migrator. Path is relative to CARGO_MANIFEST_DIR +/// (`crates/vestige-core/`). +static MIGRATOR: Migrator = sqlx::migrate!("./migrations/postgres"); + +/// Apply migrations through version 1 (the schema-only migration). +/// +/// Idempotent: sqlx::migrate consults the `_sqlx_migrations` table and is +/// a no-op on a database already at version 1 or higher. +pub(crate) async fn run_pre_register(pool: &PgPool) -> MemoryStoreResult<()> { + MIGRATOR.run_to(pool, 1).await?; + Ok(()) +} + +/// Apply any remaining migrations. Called after `register_model` has +/// stamped the typmod on `knowledge_nodes.embedding`. +pub(crate) async fn run_post_register(pool: &PgPool) -> MemoryStoreResult<()> { + MIGRATOR.run(pool).await?; + Ok(()) +} +``` + +Wiring into `PgMemoryStore::connect`. The skeleton from 0002a uses +`todo!()` for everything past pool construction. This sub-plan replaces +that with `run_pre_register` only; `run_post_register` is invoked by +`finalize_schema`, which lands in 0002d. Sketch: + +```rust +// In crates/vestige-core/src/storage/postgres/mod.rs (sub-plan 0002a wires +// pool construction; this sub-plan adds the run_pre_register call): + +impl PgMemoryStore { + pub async fn connect(url: &str, max_connections: u32) -> MemoryStoreResult { + let pool = super::pool::build(url, max_connections).await?; + super::migrations::run_pre_register(&pool).await?; + Ok(Self { pool }) + } +} +``` + +Module wire-up in `crates/vestige-core/src/storage/postgres/mod.rs`: + +```rust +mod migrations; // pub(crate) functions; not re-exported. +``` + +### Error variant + +Sub-plan 0002a already added (under feature gate) to `MemoryStoreError`: + +```rust +#[cfg(feature = "postgres-backend")] +#[error("postgres migration error: {0}")] +Migrate(#[from] sqlx::migrate::MigrateError), +``` + +`run_pre_register` / `run_post_register` use the `?` operator and the +`#[from]` conversion handles it; no extra error handling code is needed. + +--- + +## Visibility CHECK constraint + +ADR 0002 D7 specifies the tri-state enum: + +``` +visibility IN ('private', 'group', 'public') +``` + +This sub-plan includes that CHECK on the `knowledge_nodes` table (see 0001_init.up.sql +above) on both sides: + +- Postgres: `CHECK (visibility IN ('private', 'group', 'public'))` inline on + the table. +- SQLite: same CHECK constraint can be added to V15 if desired. (It is not + in the V15 body above because adding a CHECK via ALTER TABLE on SQLite + requires a table rebuild; we trust the application layer for SQLite, since + SQLite never serves the multi-user query path in Phase 2.) + +The stronger consistency rule from the ADR 0002 follow-ups section, + +``` +CHECK ( + visibility = 'private' + OR cardinality(shared_with_groups) > 0 + OR visibility = 'public' +) +``` + +is intentionally NOT added in this sub-plan. Rationale: + +- The rule is a "no orphan group rows" sanity check, not a correctness + requirement for Phase 2 (single-user mode never touches the column). +- Phase 3 is the first phase that writes `visibility = 'group'`. The check + belongs in the Phase 3 migration that lights up auth, alongside the + application code that ensures `shared_with_groups` is populated before + the visibility flips. +- Adding it now and discovering Phase 3 wants a different shape forces an + online CHECK constraint replacement. + +Recommendation: include only the IN check in Phase 2; revisit the +cardinality check in Phase 3. + +--- + +## Offline sqlx cache + +`crates/vestige-core/.sqlx/` is the on-disk cache of compile-time-checked +queries that `sqlx::query!` / `sqlx::query_as!` emit at build time when +`SQLX_OFFLINE=true`. It is committed to the repo so builds without +`DATABASE_URL` (CI, downstream consumers, contributors without Postgres) +succeed. + +This sub-plan does NOT yet generate or commit `.sqlx/` content. Reasons: + +- `sqlx::query!` calls are introduced in `0002d-store-impl-bodies.md` (real + CRUD bodies) and `0002e-hybrid-search.md` (RRF). This sub-plan ships only + the migrations directory and a wrapper that uses `sqlx::migrate!` -- which + is a compile-time macro that reads files, not a query macro that needs a + DB connection. +- Generating an empty `.sqlx/` directory now is noise that gets immediately + overwritten in the next sub-plan. + +Sub-plan 0002d will land the procedure: + +```sh +# Local dev box with vestige DB initialised per local-dev-postgres-setup.md. +export DATABASE_URL="postgresql://vestige:$(cat ~/.vestige_pg_pw)@127.0.0.1:5432/vestige" + +# Apply migrations against the dev DB. +cargo sqlx migrate run \ + --source crates/vestige-core/migrations/postgres \ + --database-url "$DATABASE_URL" + +# Generate the offline cache. +cargo sqlx prepare --workspace -- --features postgres-backend + +# Verify cache compiles offline. +SQLX_OFFLINE=true cargo check --workspace --features postgres-backend +``` + +The `.sqlx/` directory commit policy is: committed, reviewed in PRs that +add or change `query!` calls, regenerated locally and pushed. + +What this sub-plan DOES need from sqlx-cli, for verification only (see next +section): `cargo sqlx migrate run --source crates/vestige-core/migrations/postgres`. + +--- + +## Verification + +Two halves: Postgres migrations run cleanly on a fresh DB; SQLite V15 does +not break the Phase 1 store. + +### Postgres + +Prerequisites: Postgres 18 with pgvector, a role with CREATEDB and EXTENSION +rights, per `docs/plans/local-dev-postgres-setup.md`. Alternatively, a +container: + +```sh +podman run --rm -d --name vestige-pg \ + -e POSTGRES_PASSWORD=devpw \ + -e POSTGRES_USER=vestige \ + -e POSTGRES_DB=vestige \ + -p 5432:5432 \ + docker.io/pgvector/pgvector:pg16 + +export DATABASE_URL="postgresql://vestige:devpw@127.0.0.1:5432/vestige" +``` + +Steps: + +1. Apply migrations. From the repo root: + + ```sh + cargo install sqlx-cli --no-default-features --features postgres + cargo sqlx migrate run \ + --source crates/vestige-core/migrations/postgres \ + --database-url "$DATABASE_URL" + ``` + + Expected output: `Applied 1/migrate init` (`0002` is gated on typmod; + sqlx-cli will run it and pgvector will reject the HNSW creation with + "column does not have dimensions". This is the expected behaviour when + running migrations without going through the Rust connect path. To run + 0002 manually for verification, first stamp the typmod: + + ```sh + psql "$DATABASE_URL" -c "ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector(768);" + cargo sqlx migrate run \ + --source crates/vestige-core/migrations/postgres \ + --database-url "$DATABASE_URL" + ``` + + Now 0002 should apply.) + +2. Verify tables exist: + + ```sh + psql "$DATABASE_URL" -c "\dt" + ``` + + Expected (alphabetical): + ``` + domains + edges + embedding_model + group_memberships + groups + knowledge_nodes + review_events + scheduling + users + ``` + +3. Verify the bootstrap user row: + + ```sh + psql "$DATABASE_URL" -c "SELECT id, handle, display_name FROM users;" + ``` + + Expected: + ``` + id | handle | display_name + --------------------------------------+--------+-------------- + 00000000-0000-0000-0000-000000000001 | local | Local User + ``` + +4. Verify HNSW index (only after the typmod stamp + migrate 0002): + + ```sh + psql "$DATABASE_URL" -c "\d knowledge_nodes" + ``` + + The trailing `Indexes:` block should include `idx_knowledge_nodes_embedding_hnsw`. + +5. Verify the D7+D8 columns are present: + + ```sh + psql "$DATABASE_URL" -c " + SELECT column_name, data_type, column_default + FROM information_schema.columns + WHERE table_name = 'knowledge_nodes' + AND column_name IN ('owner_user_id', 'visibility', + 'shared_with_groups', 'codebase') + ORDER BY column_name; + " + ``` + + Expected: four rows, with `owner_user_id` defaulting to the bootstrap + UUID, `visibility` to `'private'::text`, `shared_with_groups` to + `'{}'::uuid[]`, `codebase` NULL-default. + +6. Verify CHECK constraint: + + ```sh + psql "$DATABASE_URL" -c " + INSERT INTO knowledge_nodes (content, visibility) VALUES ('test', 'bogus'); + " + # Expected: ERROR: new row for relation \"knowledge_nodes\" violates check constraint + ``` + +7. Roll back to verify down migrations work: + + ```sh + cargo sqlx migrate revert \ + --source crates/vestige-core/migrations/postgres \ + --database-url "$DATABASE_URL" + cargo sqlx migrate revert \ + --source crates/vestige-core/migrations/postgres \ + --database-url "$DATABASE_URL" + ``` + + `\dt` should then list only the sqlx-managed `_sqlx_migrations` table. + +8. Rust-side smoke test (no `sqlx::query!` calls yet, so cannot live in + a `#[sqlx::test]`-decorated function until 0002d). Manual: + + ```sh + cargo build -p vestige-core --features postgres-backend + ``` + + Should compile. The `sqlx::migrate!("./migrations/postgres")` macro + reads the directory at compile time; a missing file or syntax error + surfaces as a compile error. + +### SQLite + +1. Run the existing test suite: + + ```sh + cargo test -p vestige-core + ``` + + Expected: 352 (or current count + new V15 tests) tests pass, zero + warnings. + +2. New test in `migrations.rs#tests`: + + ```rust + #[test] + fn test_v15_advances_to_15_and_adds_d7_d8_columns() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("apply_migrations succeeds"); + + let version = get_current_version(&conn).expect("read schema_version"); + assert_eq!(version, 15, "schema_version should advance to 15"); + + // Tables exist + for tbl in ["users", "groups", "group_memberships"] { + let n: i32 = conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1", + [tbl], + |r| r.get(0), + ).expect("query sqlite_master"); + assert_eq!(n, 1, "table {tbl} should exist after V15"); + } + + // Bootstrap user row exists + let n: i32 = conn.query_row( + "SELECT COUNT(*) FROM users WHERE id = '00000000-0000-0000-0000-000000000001'", + [], + |r| r.get(0), + ).expect("query users"); + assert_eq!(n, 1, "bootstrap local user row should exist"); + + // D7+D8 columns on knowledge_nodes + let cols: Vec = conn + .prepare("PRAGMA table_info(knowledge_nodes)") + .unwrap() + .query_map([], |r| r.get::<_, String>(1)) + .unwrap() + .collect::>() + .unwrap(); + for c in ["owner_user_id", "visibility", "shared_with_groups", "codebase"] { + assert!(cols.iter().any(|x| x == c), + "knowledge_nodes should have column {c}"); + } + } + ``` + +3. Idempotency: re-applying V15 on an already-V15 DB must not error. + `apply_migrations` already skips when `current_version >= migration.version`; + no extra test needed beyond ensuring the V14 + V15 ALTER pattern works. + +4. Existing-data backfill smoke: insert a row before applying V15, then + verify the defaults populate: + + ```rust + #[test] + fn test_v15_backfills_existing_rows_with_defaults() { + let conn = rusqlite::Connection::open_in_memory().expect("open"); + + // Apply migrations through V14 only. + // (We rely on the fact that re-running apply_migrations is a no-op, + // so we apply all, then probe the columns. The V15 ALTER on a + // populated table is what we are testing implicitly.) + apply_migrations(&conn).expect("V1-V15"); + + // Insert a row using only Phase 1 columns; V15 defaults must + // populate owner_user_id / visibility / shared_with_groups / codebase. + conn.execute( + "INSERT INTO knowledge_nodes (id, content, node_type, created_at, updated_at, last_accessed) + VALUES ('test', 'hello', 'fact', datetime('now'), datetime('now'), datetime('now'))", + [], + ).expect("insert"); + + let (owner, vis, shared, codebase): (String, String, String, Option) = + conn.query_row( + "SELECT owner_user_id, visibility, shared_with_groups, codebase + FROM knowledge_nodes WHERE id = 'test'", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ).expect("query"); + + assert_eq!(owner, "00000000-0000-0000-0000-000000000001"); + assert_eq!(vis, "private"); + assert_eq!(shared, "[]"); + assert_eq!(codebase, None); + } + ``` + +5. Live deployment: apply V15 to a copy of `~/.vestige/vestige.db` and + verify the existing 150 memories all carry the four new columns with + default values: + + ```sh + cp ~/.vestige/vestige.db /tmp/v15-test.db + sqlite3 /tmp/v15-test.db <<'SQL' + .schema knowledge_nodes + SELECT COUNT(*) FROM knowledge_nodes; + SELECT DISTINCT owner_user_id, visibility, shared_with_groups + FROM knowledge_nodes LIMIT 5; + SQL + # (Migration applies on first read by the vestige binary running V15.) + ``` + + Capture pre- and post-counts. Expected: no row count change, all new + columns populated by defaults. + +--- + +## Acceptance criteria + +- [ ] `crates/vestige-core/migrations/postgres/` directory contains exactly + four files: `0001_init.up.sql`, `0001_init.down.sql`, + `0002_hnsw.up.sql`, `0002_hnsw.down.sql`. Content matches this + sub-plan. +- [ ] `crates/vestige-core/src/storage/postgres/migrations.rs` exports + `run_pre_register` and `run_post_register` as `pub(crate)` async + functions returning `MemoryStoreResult<()>`. Compiles with + `--features postgres-backend`. +- [ ] `PgMemoryStore::connect` (sub-plan 0002a skeleton) is updated to call + `run_pre_register` immediately after pool construction. `connect` + still returns before `register_model` runs; `run_post_register` + lands in 0002d via `finalize_schema`. +- [ ] `crates/vestige-core/src/storage/migrations.rs` has a new V15 entry + in `MIGRATIONS`, with `MIGRATION_V15_UP`, `MIGRATION_V15_ALTER_COLUMNS`, + and `MIGRATION_V15_INDEXES` constants. `apply_migrations` handles + V15 the same shape as V14. +- [ ] `cargo test -p vestige-core` passes. New tests cover V15 advance, + D7+D8 column existence, bootstrap user row, and existing-row backfill. +- [ ] `cargo build -p vestige-core --features postgres-backend` compiles + (the `sqlx::migrate!` macro will fail at compile time if any of the + four SQL files is missing or malformed). +- [ ] `cargo sqlx migrate run --source crates/vestige-core/migrations/postgres` + against a fresh container applies 0001 cleanly; `\dt` lists the nine + Phase 2 tables; `users` contains the bootstrap row. +- [ ] After the manual typmod stamp documented above, `cargo sqlx migrate + run` applies 0002 and `\d knowledge_nodes` shows `idx_knowledge_nodes_embedding_hnsw`. +- [ ] `cargo sqlx migrate revert` twice cleans the DB back to only the + `_sqlx_migrations` table. +- [ ] Inserting a row with `visibility = 'bogus'` is rejected by the CHECK + constraint. +- [ ] No `sqlx::query!` / `sqlx::query_as!` calls are added in this + sub-plan; the `.sqlx/` offline cache is not yet generated. +- [ ] The existing live SQLite DB on the development machine migrates from + V14 to V15 without row count change, and the 150 existing rows all + receive the four V15 default values. diff --git a/docs/plans/0002d-store-impl-bodies.md b/docs/plans/0002d-store-impl-bodies.md new file mode 100644 index 0000000..ad1d9b7 --- /dev/null +++ b/docs/plans/0002d-store-impl-bodies.md @@ -0,0 +1,1771 @@ +# Phase 2 Sub-Plan 0002d -- Store Implementation Bodies + +**Status**: Ready +**Depends on**: +- `0002a-skeleton-and-feature-gate.md` -- `PgMemoryStore` struct + trait impl block exist with `todo!()` bodies. +- `0002b-pool-and-config.md` -- `PgPool` is constructable, `MemoryStoreError::Postgres` and `MemoryStoreError::Migrate` variants exist behind the `postgres-backend` feature. +- `0002c-migrations.md` -- the two sqlx migrations (`0001_init`, `0002_hnsw`) exist, the schema is applied on `connect`, the `knowledge_nodes` / `scheduling` / `edges` / `domains` / `embedding_model` / `users` / `groups` / `group_memberships` / `review_events` tables exist with the D7+D8 columns. + +This sub-plan replaces every `todo!()` in +`crates/vestige-core/src/storage/postgres/mod.rs` with a real sqlx-backed +body, and adds `crates/vestige-core/src/storage/postgres/registry.rs` with +the `ensure_registry` / `register_model` typmod-stamping logic. + +The hybrid `search()` method is the meatiest single body in the backend +(RRF in one SQL statement) and lives in its own sub-plan +(`0002e-hybrid-search.md`). The bodies for the trivial single-branch +variants `fts_search` and `vector_search` are still inside this sub-plan +because they share row-mapping infrastructure with the CRUD bodies. + +Out of scope for this sub-plan: +- The full hybrid `search()` -- see `0002e-hybrid-search.md`. +- SQLite -> Postgres migrate CLI -- see `0002f-migrate-cli.md`. +- Re-embed flow -- see `0002g-reembed.md`. +- Phase 3 visibility filter -- explicitly NOT wired in Phase 2; see the + "Visibility filter posture" section below. + +--- + +## Context + +The Phase 1 `MemoryStore` trait surface is defined in +`crates/vestige-core/src/storage/memory_store.rs` and is the source of +truth for method signatures. ADR 0002 D7 added owner / visibility / +shared_with_groups columns to the `knowledge_nodes` table; ADR 0002 D8 promoted +`codebase` to a first-class column. The sqlx bodies in this sub-plan must +write to and read from those columns, but per ADR 0002 D7 they must NOT +filter on them in Phase 2 -- the visibility filter is a Phase 3 +deliverable that takes an `AuthContext` parameter. + +The semantics of every body must match the SQLite backend's current +behaviour. Where Postgres has native types (`UUID`, `JSONB`, `vector`, +`TEXT[]`, `TIMESTAMPTZ`) we use them directly; the SQLite backend's +RFC3339-string-and-JSON-blob encoding is an artefact of SQLite typing, +not the trait contract. + +Compile-time SQL validation uses sqlx's `query!` / `query_as!` macros. +The first time these macros run against a real database in CI they +populate `.sqlx/` query metadata; the metadata file is committed so +offline builds (CI without a live Postgres) succeed. + +--- + +## MemoryRecord type changes + +ADR 0002 D7 and D8 added four new columns to the `knowledge_nodes` table. +The `MemoryRecord` struct in +`crates/vestige-core/src/storage/memory_store.rs` must grow matching +fields so the trait surface can carry the data through both backends. +This is an additive change to the public type. + +Add to `MemoryRecord` (after the existing `metadata` field): + +```rust +/// Owner of this memory. Defaults to the local bootstrap user +/// (`00000000-0000-0000-0000-000000000001`) in single-user mode. +pub owner_user_id: Uuid, + +/// Tri-state visibility. ADR 0002 D7. +pub visibility: Visibility, + +/// Group IDs this memory is shared with when `visibility == Group`. +/// Empty for `Private` and `Public`. +pub shared_with_groups: Vec, + +/// First-class codebase tag. ADR 0002 D8. None if the ingest pipeline +/// could not infer one. +pub codebase: Option, +``` + +Add a new enum next to `MemoryRecord`: + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Visibility { + Private, + Group, + Public, +} + +impl Visibility { + pub fn as_str(&self) -> &'static str { + match self { + Self::Private => "private", + Self::Group => "group", + Self::Public => "public", + } + } + + pub fn from_str(s: &str) -> MemoryStoreResult { + match s { + "private" => Ok(Self::Private), + "group" => Ok(Self::Group), + "public" => Ok(Self::Public), + other => Err(MemoryStoreError::Backend( + format!("unknown visibility value: {other}"), + )), + } + } +} + +impl Default for Visibility { + fn default() -> Self { Self::Private } +} +``` + +`MemoryRecord` already derives `Serialize` and `Deserialize`; the new +fields ride along automatically. Two callers must change as part of this +sub-plan: + +1. **SQLite backend (V15 migration ships in `0001b-sqlite-split.md` or + the same Phase 1 amendment branch)**: the SQLite backend reads the + four new columns out of `knowledge_nodes` (V15 added them) and + populates the new fields in `Self::node_to_record`. Bootstrap user + ID is the same constant on both backends. Existing call sites that + construct `MemoryRecord` literally (in tests, in cognitive modules) + may default-init the four new fields: + + ```rust + MemoryRecord { + // ... existing fields ... + owner_user_id: LOCAL_USER_ID, + visibility: Visibility::default(), + shared_with_groups: Vec::new(), + codebase: None, + metadata: serde_json::json!({}), + } + ``` + + A single `pub const LOCAL_USER_ID: Uuid = uuid::uuid!("00000000-0000-0000-0000-000000000001");` + in `storage::memory_store` provides the bootstrap constant. + +2. **Cognitive modules that build `MemoryRecord` from the ingest + pipeline**: the ingest path already captures `codebase` in metadata + (see ADR 0002 D8). Lift it from `metadata.codebase` to the new + `codebase` field at the boundary where `MemoryRecord` is built. The + `metadata.codebase` JSON key is removed in the same commit; the + column is now the only source of truth. + +The change is purely additive to the trait surface -- no method +signatures change. Backwards compatibility for stored data (in the +SQLite case) comes from V15 defaulting the new columns to `'private'` +and the bootstrap user. The Postgres schema applies the same defaults +in `0001_init.up.sql`. + +--- + +## Registry module + +New file: `crates/vestige-core/src/storage/postgres/registry.rs`. + +```rust +#![cfg(feature = "postgres-backend")] + +//! Embedding-model registry for the Postgres backend. +//! +//! The `embedding_model` table stores exactly one row (id = 1) describing +//! the model whose vectors live in `knowledge_nodes.embedding`. Phase 2 enforces +//! that the active embedder matches the registered model on every write; +//! re-embed (`0002g-reembed.md`) is the only flow allowed to change the +//! row. +//! +//! The pgvector column `knowledge_nodes.embedding` is created in +//! `0001_init.up.sql` with a placeholder type (`vector`) -- no typmod. +//! On first connect we stamp the real dimension via +//! `ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector($N)` so the +//! HNSW index (created in `0002_hnsw.up.sql`) sees a sized type. + +use sqlx::PgPool; + +use crate::storage::memory_store::{ + MemoryStoreError, MemoryStoreResult, ModelSignature, +}; + +/// Look up the registered signature, if any. Returns `Ok(None)` on a +/// fresh database. +pub(crate) async fn fetch_registry( + pool: &PgPool, +) -> MemoryStoreResult> { + let row = sqlx::query!( + r#" + SELECT name, dimension, hash + FROM embedding_model + WHERE id = 1 + "# + ) + .fetch_optional(pool) + .await?; + + Ok(row.map(|r| ModelSignature { + name: r.name, + dimension: r.dimension as usize, + hash: r.hash, + })) +} + +/// First-ever call inserts the row and stamps the typmod on +/// `knowledge_nodes.embedding`. Subsequent calls compare against the stored +/// row and return `ModelMismatch` if any field differs. +pub(crate) async fn ensure_registry( + pool: &PgPool, + sig: &ModelSignature, +) -> MemoryStoreResult<()> { + let existing = fetch_registry(pool).await?; + + match existing { + None => { + sqlx::query!( + r#" + INSERT INTO embedding_model (id, name, dimension, hash) + VALUES (1, $1, $2, $3) + "#, + sig.name, + sig.dimension as i32, + sig.hash, + ) + .execute(pool) + .await?; + + stamp_vector_typmod(pool, sig.dimension).await?; + Ok(()) + } + Some(reg) if reg == *sig => Ok(()), + Some(reg) => Err(MemoryStoreError::ModelMismatch { + registered_name: reg.name, + registered_dim: reg.dimension, + registered_hash: reg.hash, + actual_name: sig.name.clone(), + actual_dim: sig.dimension, + actual_hash: sig.hash.clone(), + }), + } +} + +/// Called only by the re-embed flow (`0002g-reembed.md`) after a full +/// re-encode has rewritten every row. Updates the registry row and +/// re-stamps the typmod for the new dimension. +pub(crate) async fn update_registry_for_reembed( + pool: &PgPool, + sig: &ModelSignature, +) -> MemoryStoreResult<()> { + sqlx::query!( + r#" + UPDATE embedding_model + SET name = $1, dimension = $2, hash = $3, created_at = now() + WHERE id = 1 + "#, + sig.name, + sig.dimension as i32, + sig.hash, + ) + .execute(pool) + .await?; + + stamp_vector_typmod(pool, sig.dimension).await?; + Ok(()) +} + +async fn stamp_vector_typmod(pool: &PgPool, dim: usize) -> MemoryStoreResult<()> { + // pgvector's typmod is part of the column type, not a bound parameter. + // `format!` is safe here because `dim` is a `usize` cast to a decimal + // literal; there is no path for user-controlled SQL to reach this + // string. + let ddl = format!( + "ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector({dim})" + ); + sqlx::query(&ddl).execute(pool).await?; + Ok(()) +} +``` + +Wire the new module into `crates/vestige-core/src/storage/postgres/mod.rs`: + +```rust +pub(crate) mod registry; +``` + +The `fetch_registry` / `ensure_registry` functions are reached from the +trait methods `registered_model` and `register_model` (see method bodies +below). `update_registry_for_reembed` is reached only from +`postgres::reembed`, which is filled in by `0002g-reembed.md`. + +--- + +## Method-by-method bodies + +Every body below replaces a `todo!()` in +`crates/vestige-core/src/storage/postgres/mod.rs`. Method order matches +the trait declaration in `memory_store.rs`. + +Common imports at the top of `mod.rs`: + +```rust +use chrono::{DateTime, Utc}; +use pgvector::Vector; +use uuid::Uuid; + +use crate::storage::memory_store::{ + Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, + MemoryStoreError, MemoryStoreResult, ModelSignature, SchedulingState, + SearchQuery, SearchResult, StoreStats, Visibility, +}; +``` + +Recurring row-to-record helper (private to `mod.rs`): + +```rust +fn row_to_record( + id: Uuid, + content: String, + node_type: String, + tags: Vec, + domains: Vec, + domain_scores: serde_json::Value, + codebase: Option, + owner_user_id: Uuid, + visibility: String, + shared_with_groups: Vec, + embedding: Option, + metadata: serde_json::Value, + created_at: DateTime, + updated_at: DateTime, +) -> MemoryStoreResult { + let domain_scores: std::collections::HashMap = + serde_json::from_value(domain_scores).unwrap_or_default(); + let embedding = embedding.map(|v| v.to_vec()); + Ok(MemoryRecord { + id, + domains, + domain_scores, + content, + node_type, + tags, + embedding, + created_at, + updated_at, + metadata, + owner_user_id, + visibility: Visibility::from_str(&visibility)?, + shared_with_groups, + codebase, + }) +} +``` + +### Lifecycle + +#### `init` + +```rust +async fn init(&self) -> MemoryStoreResult<()> +``` + +Migrations already ran in `connect`; this is a no-op identical to +SQLite's behaviour. + +```rust +async fn init(&self) -> MemoryStoreResult<()> { + Ok(()) +} +``` + +#### `health_check` + +```rust +async fn health_check(&self) -> MemoryStoreResult +``` + +Issue a trivial `SELECT 1`. Pool acquisition errors degrade to +`HealthStatus::Degraded`; any other error path returns `Unavailable`. + +```rust +async fn health_check(&self) -> MemoryStoreResult { + match sqlx::query_scalar!("SELECT 1::int") + .fetch_one(&self.pool) + .await + { + Ok(_) => Ok(HealthStatus::Healthy), + Err(sqlx::Error::PoolTimedOut) => Ok(HealthStatus::Degraded { + reason: "pool exhausted".to_string(), + }), + Err(e) => Ok(HealthStatus::Unavailable { + reason: e.to_string(), + }), + } +} +``` + +### Embedding-model registry + +#### `registered_model` + +```rust +async fn registered_model(&self) -> MemoryStoreResult> +``` + +Thin pass-through to `registry::fetch_registry`. The Postgres backend +does not cache the row in-memory the way the SQLite backend does -- +sqlx's prepared-statement cache already keeps the SELECT cheap, and +`registered_model` is not on the hot path. + +```rust +async fn registered_model(&self) -> MemoryStoreResult> { + crate::storage::postgres::registry::fetch_registry(&self.pool).await +} +``` + +#### `register_model` + +```rust +async fn register_model(&self, sig: &ModelSignature) -> MemoryStoreResult<()> +``` + +Delegate to `registry::ensure_registry`, which handles the +"insert + stamp typmod" first-run path and the "compare" subsequent path. + +```rust +async fn register_model(&self, sig: &ModelSignature) -> MemoryStoreResult<()> { + crate::storage::postgres::registry::ensure_registry(&self.pool, sig).await +} +``` + +### CRUD + +#### `insert` + +```rust +async fn insert(&self, record: &MemoryRecord) -> MemoryStoreResult +``` + +Single `INSERT` into `knowledge_nodes` with all D7+D8 columns. Bind embedding +as `Option` -- pgvector's sqlx integration handles the +typmod check at execution time, so a length mismatch surfaces as +`MemoryStoreError::Postgres`. The caller-supplied UUID is preserved +(same contract as SQLite). Initial scheduling state is inserted in the +same transaction so a memory is never queryable without a scheduling +row. + +```rust +async fn insert(&self, record: &MemoryRecord) -> MemoryStoreResult { + let embedding: Option = record + .embedding + .as_ref() + .map(|v| Vector::from(v.clone())); + let domain_scores = serde_json::to_value(&record.domain_scores) + .unwrap_or_else(|_| serde_json::json!({})); + + let mut tx = self.pool.begin().await?; + + sqlx::query!( + r#" + INSERT INTO knowledge_nodes ( + id, + owner_user_id, + visibility, + shared_with_groups, + codebase, + content, + node_type, + tags, + domains, + domain_scores, + embedding, + metadata, + created_at, + updated_at + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb, + $11, $12::jsonb, $13, $14 + ) + "#, + record.id, + record.owner_user_id, + record.visibility.as_str(), + &record.shared_with_groups as &[Uuid], + record.codebase.as_deref(), + record.content, + record.node_type, + &record.tags as &[String], + &record.domains as &[String], + domain_scores, + embedding as Option, + record.metadata, + record.created_at, + record.updated_at, + ) + .execute(&mut *tx) + .await?; + + // Seed scheduling state. Mirrors SQLite defaults from `knowledge_nodes` + // (stability=1.0, difficulty=0.3, retrievability=1.0, reps=0, lapses=0, + // next_review = created_at + 1 day). + sqlx::query!( + r#" + INSERT INTO scheduling ( + memory_id, stability, difficulty, retrievability, + last_review, next_review, reps, lapses + ) + VALUES ($1, 1.0, 0.3, 1.0, NULL, $2, 0, 0) + "#, + record.id, + record.created_at + chrono::Duration::days(1), + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(record.id) +} +``` + +Tricky bits: +- `&record.tags as &[String]` -- sqlx requires an explicit slice cast + to bind a `Vec` as `text[]`. +- `&record.shared_with_groups as &[Uuid]` -- same pattern for `uuid[]`. +- `embedding as Option` -- type annotation is mandatory in the + macro because the inference path bottoms out at a generic; pgvector's + `Encode` impl resolves only with a known concrete type. +- The `$10::jsonb` and `$12::jsonb` casts force sqlx to encode through + the JSONB path even if the parameter type-resolves to `JSON`. This + matters because the migrations created the columns as `JSONB`, and + sqlx 0.8 does not always pick JSONB without the cast. + +#### `get` + +```rust +async fn get(&self, id: Uuid) -> MemoryStoreResult> +``` + +`SELECT *` filtered by primary key. Row mapping goes through +`row_to_record`. + +```rust +async fn get(&self, id: Uuid) -> MemoryStoreResult> { + let row = sqlx::query!( + r#" + SELECT + id AS "id!: Uuid", + owner_user_id AS "owner_user_id!: Uuid", + visibility, + shared_with_groups AS "shared_with_groups!: Vec", + codebase, + content, + node_type, + tags AS "tags!: Vec", + domains AS "domains!: Vec", + domain_scores AS "domain_scores!: serde_json::Value", + embedding AS "embedding: Vector", + metadata AS "metadata!: serde_json::Value", + created_at AS "created_at!: DateTime", + updated_at AS "updated_at!: DateTime" + FROM knowledge_nodes + WHERE id = $1 + "#, + id, + ) + .fetch_optional(&self.pool) + .await?; + + let Some(r) = row else { return Ok(None) }; + + Ok(Some(row_to_record( + r.id, r.content, r.node_type, r.tags, r.domains, + r.domain_scores, r.codebase, r.owner_user_id, r.visibility, + r.shared_with_groups, r.embedding, r.metadata, + r.created_at, r.updated_at, + )?)) +} +``` + +The `AS "name!: Type"` annotations tell sqlx the exact Rust type for +each column, which is required for `Vec` (from `uuid[]`) and +`Vector` (from `vector`). The `!` means "trust me, this column is NOT +NULL"; sqlx skips its `Option` wrapping for those columns. The +`embedding` column is nullable, so it gets `Option` (no `!`). + +#### `update` + +```rust +async fn update(&self, record: &MemoryRecord) -> MemoryStoreResult<()> +``` + +Update everything the caller might have changed. `updated_at` is set +server-side via `now()` so clock drift between hosts does not leak into +the timeline. (If the caller wants to forge `updated_at` -- e.g. the +migrate CLI replaying SQLite timestamps -- it goes through `insert`, not +`update`.) The schema's `BEFORE UPDATE` trigger could replace this; we +write `updated_at = now()` explicitly to be backend-agnostic. + +```rust +async fn update(&self, record: &MemoryRecord) -> MemoryStoreResult<()> { + let embedding: Option = record + .embedding + .as_ref() + .map(|v| Vector::from(v.clone())); + let domain_scores = serde_json::to_value(&record.domain_scores) + .unwrap_or_else(|_| serde_json::json!({})); + + let rows = sqlx::query!( + r#" + UPDATE knowledge_nodes SET + owner_user_id = $2, + visibility = $3, + shared_with_groups = $4, + codebase = $5, + content = $6, + node_type = $7, + tags = $8, + domains = $9, + domain_scores = $10::jsonb, + embedding = $11, + metadata = $12::jsonb, + updated_at = now() + WHERE id = $1 + "#, + record.id, + record.owner_user_id, + record.visibility.as_str(), + &record.shared_with_groups as &[Uuid], + record.codebase.as_deref(), + record.content, + record.node_type, + &record.tags as &[String], + &record.domains as &[String], + domain_scores, + embedding as Option, + record.metadata, + ) + .execute(&self.pool) + .await? + .rows_affected(); + + if rows == 0 { + return Err(MemoryStoreError::NotFound(record.id.to_string())); + } + Ok(()) +} +``` + +#### `delete` + +```rust +async fn delete(&self, id: Uuid) -> MemoryStoreResult<()> +``` + +Single `DELETE` by id. `scheduling`, `edges`, and `review_events` all +have `ON DELETE CASCADE` on their `memory_id` foreign key, so this one +statement clears every dependent row. + +```rust +async fn delete(&self, id: Uuid) -> MemoryStoreResult<()> { + let rows = sqlx::query!( + "DELETE FROM knowledge_nodes WHERE id = $1", + id, + ) + .execute(&self.pool) + .await? + .rows_affected(); + + if rows == 0 { + return Err(MemoryStoreError::NotFound(id.to_string())); + } + Ok(()) +} +``` + +### Search (single-branch variants) + +The full hybrid `search` is implemented in `0002e-hybrid-search.md`. +The two single-branch variants below ship in this sub-plan. + +#### `fts_search` + +```rust +async fn fts_search(&self, text: &str, limit: usize) -> MemoryStoreResult> +``` + +PostgreSQL full-text search using the precomputed `search_vec` tsvector +column and `websearch_to_tsquery` (handles bare words, phrases, and +boolean operators). Ranking with `ts_rank_cd` (cover-density) so longer +matches outrank shorter ones; the SQLite backend uses BM25 from FTS5 but +the trait contract only requires "higher is better". + +```rust +async fn fts_search( + &self, + text: &str, + limit: usize, +) -> MemoryStoreResult> { + let limit = limit.min(1000) as i64; + let rows = sqlx::query!( + r#" + SELECT + m.id AS "id!: Uuid", + m.owner_user_id AS "owner_user_id!: Uuid", + m.visibility, + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase, + m.content, + m.node_type, + m.tags AS "tags!: Vec", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.embedding AS "embedding: Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: DateTime", + m.updated_at AS "updated_at!: DateTime", + ts_rank_cd(m.search_vec, websearch_to_tsquery('english', $1)) + AS "score!: f64" + FROM knowledge_nodes m + WHERE m.search_vec @@ websearch_to_tsquery('english', $1) + ORDER BY score DESC + LIMIT $2 + "#, + text, + limit, + ) + .fetch_all(&self.pool) + .await?; + + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + let rec = row_to_record( + r.id, r.content, r.node_type, r.tags, r.domains, + r.domain_scores, r.codebase, r.owner_user_id, r.visibility, + r.shared_with_groups, r.embedding, r.metadata, + r.created_at, r.updated_at, + )?; + out.push(SearchResult { + record: rec, + score: r.score, + fts_score: Some(r.score), + vector_score: None, + }); + } + Ok(out) +} +``` + +The `'english'` text-search configuration matches the GIN index built in +`0001_init.up.sql`. If a future migration parameterises the config, both +the index and this query change together. + +#### `vector_search` + +```rust +async fn vector_search(&self, embedding: &[f32], limit: usize) -> MemoryStoreResult> +``` + +pgvector cosine distance. The HNSW index on `embedding` (built in +`0002_hnsw.up.sql` with `vector_cosine_ops`) makes the `<=>` operator +index-accelerated. We convert the returned distance (0 = identical, 2 = +opposite for cosine on normalized vectors) to a similarity in `[0, 1]` +via `1 - distance`; this matches the SQLite backend's convention. + +```rust +async fn vector_search( + &self, + embedding: &[f32], + limit: usize, +) -> MemoryStoreResult> { + let query_vec = Vector::from(embedding.to_vec()); + let limit = limit.min(1000) as i64; + + let rows = sqlx::query!( + r#" + SELECT + m.id AS "id!: Uuid", + m.owner_user_id AS "owner_user_id!: Uuid", + m.visibility, + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase, + m.content, + m.node_type, + m.tags AS "tags!: Vec", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.embedding AS "embedding: Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: DateTime", + m.updated_at AS "updated_at!: DateTime", + (1.0 - (m.embedding <=> $1)) AS "score!: f64" + FROM knowledge_nodes m + WHERE m.embedding IS NOT NULL + ORDER BY m.embedding <=> $1 + LIMIT $2 + "#, + query_vec as Vector, + limit, + ) + .fetch_all(&self.pool) + .await?; + + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + let rec = row_to_record( + r.id, r.content, r.node_type, r.tags, r.domains, + r.domain_scores, r.codebase, r.owner_user_id, r.visibility, + r.shared_with_groups, r.embedding, r.metadata, + r.created_at, r.updated_at, + )?; + out.push(SearchResult { + record: rec, + score: r.score, + fts_score: None, + vector_score: Some(r.score), + }); + } + Ok(out) +} +``` + +The `query_vec as Vector` cast is the same type-annotation trick as +`insert` -- sqlx needs the concrete pgvector type to wire up encoding. +The `ORDER BY m.embedding <=> $1` (no `score`) is intentional: it lets +the HNSW index serve the query directly. Sorting by the computed +`score` column would force a sequential scan because the index orders +by distance, not similarity. + +### Scheduling + +The Postgres `scheduling` table is a separate row keyed on `memory_id`, +not embedded in `knowledge_nodes` (unlike SQLite where FSRS columns live on +`knowledge_nodes`). The bodies abstract that difference at the SQL +boundary; callers see the same `SchedulingState` value. + +#### `get_scheduling` + +```rust +async fn get_scheduling(&self, memory_id: Uuid) -> MemoryStoreResult> +``` + +```rust +async fn get_scheduling( + &self, + memory_id: Uuid, +) -> MemoryStoreResult> { + let row = sqlx::query!( + r#" + SELECT + memory_id AS "memory_id!: Uuid", + stability AS "stability!: f64", + difficulty AS "difficulty!: f64", + retrievability AS "retrievability!: f64", + last_review AS "last_review: DateTime", + next_review AS "next_review: DateTime", + reps AS "reps!: i32", + lapses AS "lapses!: i32" + FROM scheduling + WHERE memory_id = $1 + "#, + memory_id, + ) + .fetch_optional(&self.pool) + .await?; + + Ok(row.map(|r| SchedulingState { + memory_id: r.memory_id, + stability: r.stability, + difficulty: r.difficulty, + retrievability: r.retrievability, + last_review: r.last_review, + next_review: r.next_review, + reps: r.reps as u32, + lapses: r.lapses as u32, + })) +} +``` + +#### `update_scheduling` + +```rust +async fn update_scheduling(&self, state: &SchedulingState) -> MemoryStoreResult<()> +``` + +Upsert -- the `INSERT ... ON CONFLICT DO UPDATE` form -- so cognitive +modules that update scheduling for a freshly-inserted memory don't have +to race with the seed row from `insert`. + +```rust +async fn update_scheduling( + &self, + state: &SchedulingState, +) -> MemoryStoreResult<()> { + sqlx::query!( + r#" + INSERT INTO scheduling ( + memory_id, stability, difficulty, retrievability, + last_review, next_review, reps, lapses + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (memory_id) DO UPDATE SET + stability = EXCLUDED.stability, + difficulty = EXCLUDED.difficulty, + retrievability = EXCLUDED.retrievability, + last_review = EXCLUDED.last_review, + next_review = EXCLUDED.next_review, + reps = EXCLUDED.reps, + lapses = EXCLUDED.lapses + "#, + state.memory_id, + state.stability, + state.difficulty, + state.retrievability, + state.last_review, + state.next_review, + state.reps as i32, + state.lapses as i32, + ) + .execute(&self.pool) + .await?; + Ok(()) +} +``` + +#### `get_due_memories` + +```rust +async fn get_due_memories( + &self, + before: DateTime, + limit: usize, +) -> MemoryStoreResult> +``` + +Join `knowledge_nodes` and `scheduling`, filter on `next_review <= before`. +Single query returns both halves of the tuple. + +```rust +async fn get_due_memories( + &self, + before: DateTime, + limit: usize, +) -> MemoryStoreResult> { + let limit = limit.min(10_000) as i64; + let rows = sqlx::query!( + r#" + SELECT + m.id AS "id!: Uuid", + m.owner_user_id AS "owner_user_id!: Uuid", + m.visibility, + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase, + m.content, + m.node_type, + m.tags AS "tags!: Vec", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.embedding AS "embedding: Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: DateTime", + m.updated_at AS "updated_at!: DateTime", + s.stability AS "stability!: f64", + s.difficulty AS "difficulty!: f64", + s.retrievability AS "retrievability!: f64", + s.last_review AS "last_review: DateTime", + s.next_review AS "next_review: DateTime", + s.reps AS "reps!: i32", + s.lapses AS "lapses!: i32" + FROM knowledge_nodes m + JOIN scheduling s ON s.memory_id = m.id + WHERE s.next_review IS NOT NULL AND s.next_review <= $1 + ORDER BY s.next_review ASC + LIMIT $2 + "#, + before, + limit, + ) + .fetch_all(&self.pool) + .await?; + + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + let rec = row_to_record( + r.id, r.content, r.node_type, r.tags, r.domains, + r.domain_scores, r.codebase, r.owner_user_id, r.visibility, + r.shared_with_groups, r.embedding, r.metadata, + r.created_at, r.updated_at, + )?; + let state = SchedulingState { + memory_id: rec.id, + stability: r.stability, + difficulty: r.difficulty, + retrievability: r.retrievability, + last_review: r.last_review, + next_review: r.next_review, + reps: r.reps as u32, + lapses: r.lapses as u32, + }; + out.push((rec, state)); + } + Ok(out) +} +``` + +### Graph (edges) + +#### `add_edge` + +```rust +async fn add_edge(&self, edge: &MemoryEdge) -> MemoryStoreResult<()> +``` + +`INSERT ... ON CONFLICT` -- updating the weight if an edge already +exists (matches SQLite's `save_connection` semantics). + +```rust +async fn add_edge(&self, edge: &MemoryEdge) -> MemoryStoreResult<()> { + sqlx::query!( + r#" + INSERT INTO edges ( + source_id, target_id, edge_type, weight, created_at + ) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (source_id, target_id, edge_type) DO UPDATE SET + weight = EXCLUDED.weight + "#, + edge.source_id, + edge.target_id, + edge.edge_type, + edge.weight, + edge.created_at, + ) + .execute(&self.pool) + .await?; + Ok(()) +} +``` + +#### `get_edges` + +```rust +async fn get_edges( + &self, + node_id: Uuid, + edge_type: Option<&str>, +) -> MemoryStoreResult> +``` + +Return every edge incident to `node_id` in either direction, optionally +filtered by `edge_type`. The optional filter binds as nullable; `$2 IS +NULL OR edge_type = $2` keeps the prepared statement reusable. + +```rust +async fn get_edges( + &self, + node_id: Uuid, + edge_type: Option<&str>, +) -> MemoryStoreResult> { + let rows = sqlx::query!( + r#" + SELECT + source_id AS "source_id!: Uuid", + target_id AS "target_id!: Uuid", + edge_type, + weight AS "weight!: f64", + created_at AS "created_at!: DateTime" + FROM edges + WHERE (source_id = $1 OR target_id = $1) + AND ($2::text IS NULL OR edge_type = $2) + "#, + node_id, + edge_type, + ) + .fetch_all(&self.pool) + .await?; + + Ok(rows + .into_iter() + .map(|r| MemoryEdge { + source_id: r.source_id, + target_id: r.target_id, + edge_type: r.edge_type, + weight: r.weight, + created_at: r.created_at, + }) + .collect()) +} +``` + +#### `remove_edge` + +```rust +async fn remove_edge(&self, source: Uuid, target: Uuid) -> MemoryStoreResult<()> +``` + +Note: the live trait signature is two args (`source`, `target`). The +master plan's stale three-arg signature including `edge_type` is not +implemented -- the live trait surface wins. Deletes every edge between +the pair regardless of `edge_type`. + +```rust +async fn remove_edge( + &self, + source: Uuid, + target: Uuid, +) -> MemoryStoreResult<()> { + sqlx::query!( + "DELETE FROM edges WHERE source_id = $1 AND target_id = $2", + source, + target, + ) + .execute(&self.pool) + .await?; + Ok(()) +} +``` + +#### `get_neighbors` + +```rust +async fn get_neighbors( + &self, + node_id: Uuid, + depth: usize, +) -> MemoryStoreResult> +``` + +Recursive CTE walks the edge graph outward from `node_id` for up to +`depth` hops. Weights compound multiplicatively along the path (same as +SQLite BFS). Cap the visited set at 256 rows to match SQLite. Direction +is treated as undirected by unioning both halves of each edge inside +the CTE. + +```rust +async fn get_neighbors( + &self, + node_id: Uuid, + depth: usize, +) -> MemoryStoreResult> { + if depth == 0 { + let Some(rec) = self.get(node_id).await? else { + return Err(MemoryStoreError::NotFound(node_id.to_string())); + }; + return Ok(vec![(rec, 1.0)]); + } + + let depth_i = depth.min(16) as i32; + let rows = sqlx::query!( + r#" + WITH RECURSIVE walk(node_id, weight, hops) AS ( + SELECT $1::uuid, 1.0::float8, 0 + UNION ALL + SELECT + CASE WHEN e.source_id = w.node_id THEN e.target_id + ELSE e.source_id END, + w.weight * e.weight, + w.hops + 1 + FROM walk w + JOIN edges e + ON e.source_id = w.node_id OR e.target_id = w.node_id + WHERE w.hops < $2 + ), + best AS ( + SELECT node_id, MAX(weight) AS weight + FROM walk + GROUP BY node_id + LIMIT 256 + ) + SELECT + m.id AS "id!: Uuid", + m.owner_user_id AS "owner_user_id!: Uuid", + m.visibility, + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase, + m.content, + m.node_type, + m.tags AS "tags!: Vec", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.embedding AS "embedding: Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: DateTime", + m.updated_at AS "updated_at!: DateTime", + b.weight AS "weight!: f64" + FROM best b + JOIN knowledge_nodes m ON m.id = b.node_id + "#, + node_id, + depth_i, + ) + .fetch_all(&self.pool) + .await?; + + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + let rec = row_to_record( + r.id, r.content, r.node_type, r.tags, r.domains, + r.domain_scores, r.codebase, r.owner_user_id, r.visibility, + r.shared_with_groups, r.embedding, r.metadata, + r.created_at, r.updated_at, + )?; + out.push((rec, r.weight)); + } + Ok(out) +} +``` + +The CTE can visit a node multiple times via different paths; the `best` +sub-CTE picks the highest weight per node. The `LIMIT 256` matches the +SQLite BFS cap. Postgres' recursive CTE is breadth-first by hop count +because of the `WHERE w.hops < $2` predicate. + +### Domains (Phase 4 populates; Phase 2 ships CRUD) + +The `domains` table is empty in Phase 2; these methods exist so the +trait surface is complete but they do not get exercised until Phase 4 +HDBSCAN clustering runs. + +#### `list_domains` + +```rust +async fn list_domains(&self) -> MemoryStoreResult> +``` + +```rust +async fn list_domains(&self) -> MemoryStoreResult> { + let rows = sqlx::query!( + r#" + SELECT + id, + label, + centroid AS "centroid: Vector", + top_terms AS "top_terms!: Vec", + memory_count AS "memory_count!: i64", + created_at AS "created_at!: DateTime" + FROM domains + ORDER BY created_at ASC + "# + ) + .fetch_all(&self.pool) + .await?; + + Ok(rows + .into_iter() + .map(|r| Domain { + id: r.id, + label: r.label, + centroid: r.centroid.map(|v| v.to_vec()).unwrap_or_default(), + top_terms: r.top_terms, + memory_count: r.memory_count as usize, + created_at: r.created_at, + }) + .collect()) +} +``` + +#### `get_domain` + +```rust +async fn get_domain(&self, id: &str) -> MemoryStoreResult> +``` + +```rust +async fn get_domain(&self, id: &str) -> MemoryStoreResult> { + let row = sqlx::query!( + r#" + SELECT + id, + label, + centroid AS "centroid: Vector", + top_terms AS "top_terms!: Vec", + memory_count AS "memory_count!: i64", + created_at AS "created_at!: DateTime" + FROM domains + WHERE id = $1 + "#, + id, + ) + .fetch_optional(&self.pool) + .await?; + + Ok(row.map(|r| Domain { + id: r.id, + label: r.label, + centroid: r.centroid.map(|v| v.to_vec()).unwrap_or_default(), + top_terms: r.top_terms, + memory_count: r.memory_count as usize, + created_at: r.created_at, + })) +} +``` + +#### `upsert_domain` + +```rust +async fn upsert_domain(&self, domain: &Domain) -> MemoryStoreResult<()> +``` + +```rust +async fn upsert_domain(&self, domain: &Domain) -> MemoryStoreResult<()> { + let centroid = if domain.centroid.is_empty() { + None + } else { + Some(Vector::from(domain.centroid.clone())) + }; + + sqlx::query!( + r#" + INSERT INTO domains ( + id, label, centroid, top_terms, memory_count, created_at + ) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE SET + label = EXCLUDED.label, + centroid = EXCLUDED.centroid, + top_terms = EXCLUDED.top_terms, + memory_count = EXCLUDED.memory_count + "#, + domain.id, + domain.label, + centroid as Option, + &domain.top_terms as &[String], + domain.memory_count as i64, + domain.created_at, + ) + .execute(&self.pool) + .await?; + Ok(()) +} +``` + +#### `delete_domain` + +```rust +async fn delete_domain(&self, id: &str) -> MemoryStoreResult<()> +``` + +```rust +async fn delete_domain(&self, id: &str) -> MemoryStoreResult<()> { + sqlx::query!( + "DELETE FROM domains WHERE id = $1", + id, + ) + .execute(&self.pool) + .await?; + Ok(()) +} +``` + +#### `classify` + +```rust +async fn classify(&self, embedding: &[f32]) -> MemoryStoreResult> +``` + +The Postgres backend can ship this as a single SQL query against the +empty `domains` table -- it correctly returns an empty vector in Phase +2 and starts returning real scores in Phase 4 without any code change. + +```rust +async fn classify( + &self, + embedding: &[f32], +) -> MemoryStoreResult> { + let query_vec = Vector::from(embedding.to_vec()); + let rows = sqlx::query!( + r#" + SELECT + id, + (1.0 - (centroid <=> $1)) AS "score!: f64" + FROM domains + WHERE centroid IS NOT NULL + ORDER BY score DESC + "#, + query_vec as Vector, + ) + .fetch_all(&self.pool) + .await?; + + Ok(rows.into_iter().map(|r| (r.id, r.score)).collect()) +} +``` + +### Bulk / maintenance + +#### `count` + +```rust +async fn count(&self) -> MemoryStoreResult +``` + +```rust +async fn count(&self) -> MemoryStoreResult { + let n: i64 = sqlx::query_scalar!("SELECT COUNT(*) FROM knowledge_nodes") + .fetch_one(&self.pool) + .await? + .unwrap_or(0); + Ok(n as usize) +} +``` + +#### `get_stats` + +```rust +async fn get_stats(&self) -> MemoryStoreResult +``` + +Aggregate counts across `knowledge_nodes`, `edges`, `domains`. Read the +registry inline. + +```rust +async fn get_stats(&self) -> MemoryStoreResult { + let row = sqlx::query!( + r#" + SELECT + (SELECT COUNT(*) FROM knowledge_nodes) + AS "total_memories!: i64", + (SELECT COUNT(*) FROM knowledge_nodes WHERE embedding IS NOT NULL) + AS "memories_with_embeddings!: i64", + (SELECT COUNT(*) FROM edges) + AS "total_edges!: i64", + (SELECT COUNT(*) FROM domains) + AS "total_domains!: i64", + (SELECT name FROM embedding_model WHERE id = 1) + AS "registered_model_name: String", + (SELECT dimension FROM embedding_model WHERE id = 1) + AS "registered_model_dim: i32" + "# + ) + .fetch_one(&self.pool) + .await?; + + Ok(StoreStats { + total_memories: row.total_memories as usize, + memories_with_embeddings: row.memories_with_embeddings as usize, + total_edges: row.total_edges as usize, + total_domains: row.total_domains as usize, + registered_model_name: row.registered_model_name, + registered_model_dim: row.registered_model_dim.map(|d| d as usize), + }) +} +``` + +#### `vacuum` + +```rust +async fn vacuum(&self) -> MemoryStoreResult<()> +``` + +`VACUUM` cannot run inside a transaction. sqlx wraps each `query!` +invocation in an implicit transaction when it grabs a pooled +connection, but it does not -- the pool hands out a raw connection +that runs statements in autocommit mode by default. The safe path is +to acquire a connection explicitly and `execute` each statement +separately so neither participates in a transaction. + +```rust +async fn vacuum(&self) -> MemoryStoreResult<()> { + let mut conn = self.pool.acquire().await?; + sqlx::query("VACUUM ANALYZE knowledge_nodes") + .execute(conn.as_mut()) + .await?; + sqlx::query("VACUUM ANALYZE scheduling") + .execute(conn.as_mut()) + .await?; + sqlx::query("VACUUM ANALYZE edges") + .execute(conn.as_mut()) + .await?; + Ok(()) +} +``` + +`conn.as_mut()` yields a `&mut PgConnection`, which sqlx accepts as an +executor. Using `&self.pool` here would let sqlx pick a fresh +connection per statement (still fine, but two extra acquisitions). Note +we do NOT vacuum `domains`, `edges`-related lookup tables (`users` / +`groups` etc.) -- they are either empty in Phase 2 or low-churn and the +nightly autovacuum suffices. + +--- + +## Visibility filter posture + +ADR 0002 D7 declares the future Phase 3 visibility filter (reproduced +here for clarity): + +```sql +WHERE + (visibility = 'private' AND owner_user_id = $me) + OR (visibility = 'group' + AND (owner_user_id = $me OR shared_with_groups && $my_group_ids)) + OR visibility = 'public' +``` + +**Phase 2 does NOT apply this filter.** Every body above reads and +writes the rows it touches regardless of `owner_user_id` or +`visibility` because there is exactly one user in Phase 2 mode (the +bootstrap user from `0001_init.up.sql`). The reviewer should NOT expect +`WHERE owner_user_id = $...` clauses in Phase 2 method bodies. + +Phase 3 introduces an `AuthContext` parameter on the trait methods and +threads it into each WHERE clause. That migration is purely additive +(adds a parameter, adds a clause); it does not need a schema migration +because the columns and indexes are already in place. + +The four new `MemoryRecord` fields ARE populated correctly in Phase 2 +(insert writes them, get/search read them) so that exported archives +and replicated rows round-trip the visibility intent the moment Phase +3 enables filtering. + +--- + +## Offline sqlx cache + +`sqlx::query!` and `sqlx::query_as!` validate every SQL string at +compile time by contacting a live database. To keep CI builds from +needing a Postgres on the build host, sqlx supports an offline cache +in `/.sqlx/` containing one JSON file per validated query. + +This sub-plan is where `.sqlx/` is first populated and committed. + +Workflow: + +1. Ensure a local Postgres is running with the same schema CI will see: + + ```bash + cd crates/vestige-core + export DATABASE_URL="postgres://vestige:vestige@127.0.0.1:5432/vestige_dev" + sqlx database create + sqlx migrate run --source migrations/postgres + ``` + +2. Generate the offline cache: + + ```bash + cargo sqlx prepare --workspace -- --features postgres-backend + ``` + + This walks every `sqlx::query!` invocation under the active feature + flags and writes `crates/vestige-core/.sqlx/query-.json`. The + `--workspace` flag is needed because `vestige-mcp` enables the + `postgres-backend` feature transitively in `0002b-pool-and-config.md`. + +3. Stage and commit the cache directory: + + ```bash + git add crates/vestige-core/.sqlx/ + git commit -m "store: populate sqlx offline cache for postgres backend" + ``` + +4. Add to repo `.gitignore` adjustments (only if entries already deny + `target/` or similar globs): leave `.sqlx/` excluded from any + ignore globs. Specifically the workspace root `.gitignore` does NOT + contain a `.sqlx` line; if a future PR adds one, this sub-plan's + commit reverts it. + +5. CI runs `SQLX_OFFLINE=true cargo check --features postgres-backend`. + sqlx falls back to the JSON cache when `SQLX_OFFLINE=true` is set, + so CI does not need network access to a Postgres. + +Every time a `query!` invocation changes -- add a column, change a +WHERE clause, rename a binding -- re-run `cargo sqlx prepare` and +commit the updated `.sqlx/` files. The agent implementing this sub-plan +runs `cargo sqlx prepare` as the last step before opening the PR. + +--- + +## Verification + +Three layers of verification before merging this sub-plan. + +### 1. Compile and lint + +```bash +cargo check --workspace --features postgres-backend +cargo build --workspace --features postgres-backend +cargo clippy --workspace --features postgres-backend -- -D warnings + +# SQLite-only build still works (mutual compilability per CLAUDE.md): +cargo check --workspace --no-default-features --features embeddings,vector-search +``` + +### 2. Offline cache builds + +```bash +SQLX_OFFLINE=true cargo check --workspace --features postgres-backend +``` + +This is what CI will run. If it fails, `cargo sqlx prepare` was not +re-run after the last query change. + +### 3. Integration round-trip test (testcontainers) + +New test file: +`crates/vestige-core/tests/postgres_round_trip.rs`. Skipped unless the +`postgres-backend` feature is active and Docker / Podman is available. + +```rust +#![cfg(feature = "postgres-backend")] + +use chrono::Utc; +use testcontainers::{clients, GenericImage}; +use uuid::Uuid; +use vestige_core::storage::memory_store::{ + LocalMemoryStore, MemoryEdge, MemoryRecord, SchedulingState, Visibility, + LOCAL_USER_ID, +}; +use vestige_core::storage::postgres::PgMemoryStore; + +#[tokio::test] +async fn round_trip_crud_search_scheduling_edges() { + let docker = clients::Cli::default(); + let image = GenericImage::new("pgvector/pgvector", "pg16") + .with_env_var("POSTGRES_PASSWORD", "test") + .with_env_var("POSTGRES_DB", "vestige_test") + .with_exposed_port(5432); + let container = docker.run(image); + let port = container.get_host_port_ipv4(5432); + let url = format!("postgres://postgres:test@127.0.0.1:{port}/vestige_test"); + + let store = PgMemoryStore::connect(&url, 5).await.expect("connect"); + + // Register the model (typmod stamp). + store.register_model(&fixture_signature(384)).await.expect("register"); + + // insert -> get -> update -> delete. + let mut rec = fixture_record(); + let id = store.insert(&rec).await.expect("insert"); + let fetched = store.get(id).await.expect("get").expect("present"); + assert_eq!(fetched.content, rec.content); + assert_eq!(fetched.owner_user_id, LOCAL_USER_ID); + assert_eq!(fetched.visibility, Visibility::Private); + + rec.content = "edited".to_string(); + store.update(&rec).await.expect("update"); + assert_eq!(store.get(id).await.unwrap().unwrap().content, "edited"); + + // fts_search. + let hits = store.fts_search("edited", 10).await.expect("fts"); + assert!(hits.iter().any(|h| h.record.id == id)); + + // vector_search. + let emb = rec.embedding.clone().unwrap(); + let vhits = store.vector_search(&emb, 10).await.expect("vector"); + assert!(vhits.iter().any(|h| h.record.id == id)); + + // scheduling round-trip. + let sched = store.get_scheduling(id).await.unwrap().expect("seeded"); + let new_state = SchedulingState { + memory_id: id, + stability: 5.5, + difficulty: 0.2, + retrievability: 0.95, + last_review: Some(Utc::now()), + next_review: Some(Utc::now() + chrono::Duration::days(3)), + reps: sched.reps + 1, + lapses: sched.lapses, + }; + store.update_scheduling(&new_state).await.expect("update sched"); + let after = store.get_scheduling(id).await.unwrap().unwrap(); + assert_eq!(after.reps, new_state.reps); + + // edges. + let other = fixture_record(); + let other_id = store.insert(&other).await.unwrap(); + let edge = MemoryEdge { + source_id: id, + target_id: other_id, + edge_type: "semantic".to_string(), + weight: 0.8, + created_at: Utc::now(), + }; + store.add_edge(&edge).await.expect("add_edge"); + let edges = store.get_edges(id, None).await.unwrap(); + assert_eq!(edges.len(), 1); + let neighbors = store.get_neighbors(id, 1).await.unwrap(); + assert!(neighbors.iter().any(|(r, _)| r.id == other_id)); + store.remove_edge(id, other_id).await.expect("remove_edge"); + assert!(store.get_edges(id, None).await.unwrap().is_empty()); + + // delete -> cascade. + store.delete(id).await.expect("delete"); + assert!(store.get(id).await.unwrap().is_none()); + assert!(store.get_scheduling(id).await.unwrap().is_none()); +} + +fn fixture_record() -> MemoryRecord { + MemoryRecord { + id: Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: "the quick brown fox jumps over the lazy dog".into(), + node_type: "fact".into(), + tags: vec!["test".into()], + embedding: Some(vec![0.1_f32; 384]), + created_at: Utc::now(), + updated_at: Utc::now(), + metadata: serde_json::json!({}), + owner_user_id: LOCAL_USER_ID, + visibility: Visibility::Private, + shared_with_groups: vec![], + codebase: Some("vestige".to_string()), + } +} + +fn fixture_signature(dim: usize) -> vestige_core::storage::memory_store::ModelSignature { + vestige_core::storage::memory_store::ModelSignature { + name: "test/model".to_string(), + dimension: dim, + hash: "0".repeat(64), + } +} +``` + +Add `testcontainers = "0.20"` to `[dev-dependencies]` under +`#[cfg(feature = "postgres-backend")]` gating. The test is the slowest +in the suite (spawns a Docker container, ~5 s startup); annotate with +`#[ignore]` if CI runtime budget requires opt-in execution. + +### 4. Manual smoke (optional but recommended) + +```bash +# Tear down any prior database. +make postgres-down ; make postgres-up +DATABASE_URL=$(make postgres-url) cargo test \ + -p vestige-core --features postgres-backend -- --include-ignored +``` + +The `postgres-up` / `postgres-down` / `postgres-url` make targets are +defined in `docs/plans/local-dev-postgres-setup.md`. + +--- + +## Acceptance criteria + +This sub-plan is complete when ALL of the following hold: + +1. `cargo build --workspace --features postgres-backend` succeeds with + zero warnings. +2. `cargo clippy --workspace --features postgres-backend -- -D warnings` + succeeds. +3. `cargo build --workspace --no-default-features --features embeddings,vector-search` + still succeeds (the SQLite-only build is not regressed). +4. `SQLX_OFFLINE=true cargo check --workspace --features postgres-backend` + succeeds. `crates/vestige-core/.sqlx/` exists and contains one JSON + file per `sqlx::query!` / `sqlx::query_as!` invocation in the + Postgres backend. +5. Zero `todo!()` macros remain in + `crates/vestige-core/src/storage/postgres/mod.rs`. The only + exception is the body of the trait method `search` -- that method + stays `todo!()` until `0002e-hybrid-search.md` lands. +6. `crates/vestige-core/src/storage/postgres/registry.rs` exists with + the three functions described above + (`fetch_registry`, `ensure_registry`, `update_registry_for_reembed`). +7. `MemoryRecord` carries the four new fields + (`owner_user_id`, `visibility`, `shared_with_groups`, `codebase`) + and the `Visibility` enum is exported alongside it. The SQLite + backend reads and writes the same four fields. +8. The `tests/postgres_round_trip.rs` integration test passes against + a `pgvector/pgvector:pg16` container (insert / get / update / delete + / fts_search / vector_search / get_scheduling / update_scheduling + / add_edge / get_edges / remove_edge / get_neighbors / cascade + delete). +9. No visibility filter clause is present in any Phase 2 method body. + `WHERE owner_user_id = ...`, `WHERE visibility = ...`, and + `shared_with_groups && ...` do not appear anywhere in + `crates/vestige-core/src/storage/postgres/`. +10. `cargo sqlx prepare` was the last command run before commit; the + diff includes `.sqlx/` changes if any query changed. diff --git a/docs/plans/0002e-hybrid-search.md b/docs/plans/0002e-hybrid-search.md new file mode 100644 index 0000000..1f45174 --- /dev/null +++ b/docs/plans/0002e-hybrid-search.md @@ -0,0 +1,825 @@ +# Phase 2 Sub-Plan 0002e -- Hybrid RRF Search + +**Status**: Ready +**Depends on**: +- `0002a-skeleton-and-feature-gate.md` (the `postgres-backend` feature flag + exists and `PgMemoryStore` compiles with `todo!()` bodies). +- `0002b-pool-and-config.md` (a working `PgPool` reaches the backend). +- `0002c-migrations.md` (migration `0001_init` has created the `knowledge_nodes` + table with the D7 columns -- `owner_user_id`, `visibility`, + `shared_with_groups` -- and the D8 column `codebase`; migration `0002_hnsw` + has built the HNSW index). +- `0002d-store-impl-bodies.md` (real CRUD bodies exist so the integration + tests below can seed data through the trait surface rather than raw SQL). + +This sub-plan covers master plan 0002 deliverable D5: the hybrid RRF search +query implementation in `crates/vestige-core/src/storage/postgres/search.rs`, +plus the `search`, `fts_search`, and `vector_search` method bodies in +`crates/vestige-core/src/storage/postgres/mod.rs` that delegate into it. + +--- + +## Context + +This is one of the more performance-sensitive sub-plans in Phase 2. Every +search call from the cognitive engine -- the 7-stage retrieval pipeline, +`session_context`, `predict`, `deep_reference`, the dashboard -- bottoms out +in `MemoryStore::search`. The Postgres backend has to keep up with the +existing SQLite hybrid path, which combines BM25 over FTS5 with USearch HNSW +in two separate round trips and fuses the rankings in Rust. + +The shape of the win on Postgres is that both branches and the fusion run +inside one statement. The planner sees both CTEs together, the round trip is +single, and the rerank stage runs over a cleanly overfetched candidate set. + +Latency targets live in `0002h-testing-and-benches.md`. This sub-plan is +responsible for producing a correct, schema-stable query that the benches +can drive against. Do not optimise here; correctness and structure first. + +Master plan 0002 D5 (around lines 522-628 of +`docs/plans/0002-phase-2-postgres-backend.md`) sketches the SQL. That +sketch is the starting point, not the finished product. The schema after +the D7 and D8 amendments has more columns than the sketch enumerates, and +the SQLite `search` method (around line 6503 of +`crates/vestige-core/src/storage/sqlite.rs` in the Phase 1 worktree) +documents the semantics this implementation must stay compatible with: + +- Empty `query.limit` defaults to 10. +- `query.text == Some("")` is treated as no text query (degrade to vector). +- `query.embedding == None` is treated as no vector query (degrade to FTS). +- Both empty returns `Ok(vec![])`; not an error. +- The `MemoryRecord` in each `SearchResult` must be populated with all + fields the trait promises, including `domains` and `domain_scores` (Phase + 4 will fill these in; Phase 2 returns the stored values, which may be + empty arrays / empty objects). + +--- + +## Constants + +```rust +/// Reciprocal Rank Fusion smoothing constant from Cormack, Clarke and Buettcher +/// 2009 ("Reciprocal Rank Fusion outperforms Condorcet and individual rank +/// learning methods"). 60 is the canonical default and is robust across most +/// fusion regimes. Do not tune this without a paper-citation-grade reason. +const RRF_K: i32 = 60; + +/// Each branch (FTS, vector) is allowed to return OVERFETCH_MULT x final_limit +/// rows before fusion. Three matches the Phase 1 SQLite overfetch and gives +/// the fusion enough candidates to recover from any single branch's bad +/// recall on a given query. +const OVERFETCH_MULT: i64 = 3; +``` + +These live at module scope in +`crates/vestige-core/src/storage/postgres/search.rs`. They are `pub(crate)` +only if `0002h-testing-and-benches.md` needs to reference them from the +integration tests; otherwise private. + +--- + +## Public API + +```rust +#![cfg(feature = "postgres-backend")] + +use pgvector::Vector; +use sqlx::PgPool; + +use crate::storage::memory_store::{ + MemoryStoreResult, SearchQuery, SearchResult, +}; + +/// Hybrid RRF search over Postgres FTS and pgvector cosine distance. +/// +/// Branch behavior: +/// - empty text + null embedding -> Ok(vec![]) +/// - empty text + Some(embedding) -> pure vector search (FTS CTE returns +/// zero rows; fusion equals the vector +/// branch) +/// - Some(text) + null embedding -> pure FTS search +/// - Some(text) + Some(embedding) -> full RRF fusion +/// +/// `query.limit == 0` is treated as 10 (matches the SQLite default). +pub(crate) async fn rrf_search( + pool: &PgPool, + query: &SearchQuery, +) -> MemoryStoreResult>; + +/// FTS-only convenience search. Equivalent to calling `rrf_search` with +/// `query.embedding = None`, but uses a dedicated single-branch query that +/// avoids the FULL OUTER JOIN and the params CTE; faster by one planner pass +/// per call. +pub(crate) async fn fts_only( + pool: &PgPool, + text: &str, + limit: usize, +) -> MemoryStoreResult>; + +/// Vector-only convenience search. Dedicated single-branch query for the same +/// latency reason as `fts_only`. +pub(crate) async fn vector_only( + pool: &PgPool, + embedding: &[f32], + limit: usize, +) -> MemoryStoreResult>; +``` + +### Parameter handling + +In `rrf_search`: + +```rust +let final_limit: i32 = if query.limit == 0 { 10 } else { query.limit as i32 }; +let overfetch: i32 = (final_limit as i64 * OVERFETCH_MULT) + .min(i32::MAX as i64) as i32; + +let q_text: &str = query.text.as_deref().unwrap_or(""); +let q_vec: Option = query.embedding.as_ref() + .map(|v| Vector::from(v.clone())); + +let dom_filter: Option<&[String]> = query.domains.as_deref(); +let nt_filter: Option<&[String]> = query.node_types.as_deref(); +let tag_filter: Option<&[String]> = query.tags.as_deref(); + +let min_retr: Option = query.min_retrievability; +``` + +Both branches empty -- `q_text` is empty and `q_vec` is `None` -- returns +`Ok(vec![])` without hitting the database. The SQLite backend has the same +behavior and tests rely on it. + +```rust +if q_text.is_empty() && q_vec.is_none() { + return Ok(Vec::new()); +} +``` + +### `search` method body in `postgres/mod.rs` + +```rust +#[async_trait::async_trait] // or trait_variant after the Phase 1 amendment +impl MemoryStore for PgMemoryStore { + async fn search(&self, query: &SearchQuery) + -> MemoryStoreResult> + { + crate::storage::postgres::search::rrf_search(&self.pool, query).await + } + + async fn fts_search(&self, text: &str, limit: usize) + -> MemoryStoreResult> + { + crate::storage::postgres::search::fts_only(&self.pool, text, limit) + .await + } + + async fn vector_search(&self, embedding: &[f32], limit: usize) + -> MemoryStoreResult> + { + crate::storage::postgres::search::vector_only(&self.pool, embedding, limit) + .await + } +} +``` + +Everything below specifies the inside of those three free functions. + +--- + +## SQL: the hybrid RRF query + +The query is built as one `&'static str` (or `OnceCell`; see "Use +of sqlx::query!" below) and reused. Bound parameters are kept to seven +through a `params` CTE that the rest of the query references by name -- +this keeps the SQL readable and stops the bound-parameter count growing +with each filter clause. + +Bound parameters: + +- `$1`: text query (TEXT, may be empty) +- `$2`: embedding (pgvector::Vector, may be NULL) +- `$3`: overfetch limit per branch (INT) +- `$4`: final limit (INT) +- `$5`: domain filter (TEXT[] or NULL) +- `$6`: node_type filter (TEXT[] or NULL) +- `$7`: tag filter (TEXT[] or NULL) + +If `min_retrievability.is_some()` the outer SELECT adds a JOIN on +`scheduling` and a WHERE clause; that path uses a different prepared +statement (see "min_retrievability filter" below) so the simple-path query +stays free of the join. + +```sql +WITH params AS ( + SELECT + $1::text AS q_text, + $2::vector AS q_vec, + $3::int AS overfetch, + $4::int AS final_limit, + $5::text[] AS dom_filter, + $6::text[] AS nt_filter, + $7::text[] AS tag_filter +), +fts AS ( + SELECT + m.id, + ts_rank_cd( + m.search_vec, + websearch_to_tsquery('english', p.q_text) + ) AS score, + ROW_NUMBER() OVER ( + ORDER BY ts_rank_cd( + m.search_vec, + websearch_to_tsquery('english', p.q_text) + ) DESC + ) AS rank + FROM knowledge_nodes m + CROSS JOIN params p + WHERE p.q_text <> '' + AND m.search_vec @@ websearch_to_tsquery('english', p.q_text) + AND (p.dom_filter IS NULL OR m.domains && p.dom_filter) + AND (p.nt_filter IS NULL OR m.node_type = ANY(p.nt_filter)) + AND (p.tag_filter IS NULL OR m.tags && p.tag_filter) + ORDER BY score DESC + LIMIT (SELECT overfetch FROM params) +), +vec AS ( + SELECT + m.id, + 1.0 - (m.embedding <=> p.q_vec) AS score, + ROW_NUMBER() OVER ( + ORDER BY m.embedding <=> p.q_vec + ) AS rank + FROM knowledge_nodes m + CROSS JOIN params p + WHERE m.embedding IS NOT NULL + AND p.q_vec IS NOT NULL + AND (p.dom_filter IS NULL OR m.domains && p.dom_filter) + AND (p.nt_filter IS NULL OR m.node_type = ANY(p.nt_filter)) + AND (p.tag_filter IS NULL OR m.tags && p.tag_filter) + ORDER BY m.embedding <=> p.q_vec + LIMIT (SELECT overfetch FROM params) +), +fused AS ( + SELECT + COALESCE(f.id, v.id) AS id, + COALESCE(1.0 / (60 + f.rank), 0.0) + + COALESCE(1.0 / (60 + v.rank), 0.0) AS rrf_score, + f.score AS fts_score, + v.score AS vector_score + FROM fts f + FULL OUTER JOIN vec v ON f.id = v.id +) +SELECT + m.id AS "id!: uuid::Uuid", + m.owner_user_id AS "owner_user_id!: uuid::Uuid", + m.visibility AS "visibility!: String", + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase AS "codebase: String", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.content AS "content!: String", + m.node_type AS "node_type!: String", + m.tags AS "tags!: Vec", + m.embedding AS "embedding: pgvector::Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: chrono::DateTime", + m.updated_at AS "updated_at!: chrono::DateTime", + fused.rrf_score AS "rrf_score!: f64", + fused.fts_score AS "fts_score: f64", + fused.vector_score AS "vector_score: f64" +FROM fused +JOIN knowledge_nodes m ON m.id = fused.id +ORDER BY fused.rrf_score DESC +LIMIT (SELECT final_limit FROM params); +``` + +Notes on the SELECT column list. The D7 columns (`owner_user_id`, +`visibility`, `shared_with_groups`) and the D8 column (`codebase`) are +selected even though Phase 2 does not filter on them yet, so: + +1. The `MemoryRecord` returned to the trait can be populated with the + stored values from day one. Phase 3 will start writing real + `owner_user_id` / `visibility` values; Phase 2 always writes the + single-user defaults (`'00000000-...-0001'`, `'private'`, `'{}'`). The + `MemoryRecord` returned in Phase 2 simply carries those defaults. +2. The schema-drift integration tests (see "Verification") catch the case + where someone adds a NOT NULL column to `knowledge_nodes` without updating + this query. + +Notes on the body: + +- `CROSS JOIN params p` is used instead of the master-plan sketch's + `FROM knowledge_nodes m, params p`. Same plan, clearer intent. +- The `ORDER BY ... LIMIT` inside each branch CTE is there so the planner + can stop early once it has `overfetch` rows; without it the LIMIT is + applied after a full sort over all matches. +- `1.0 - (m.embedding <=> p.q_vec)` converts pgvector's cosine *distance* + to cosine *similarity* in [0, 1] for the `vector_score` output. RRF + itself does not need the similarity -- it uses ranks -- but the trait + surface exposes `vector_score: Option` for caller introspection. +- `RRF_K = 60` is inlined as `60` in the SQL string. A `const` formatter + feels tidier but `60` is a literature constant; spell it out and leave a + comment in the Rust source: `// 60 == RRF_K (Cormack 2009)`. +- `FULL OUTER JOIN` is required: a row that the FTS branch finds and the + vector branch does not must still appear in `fused`, and vice versa. +- `COALESCE(..., 0.0)` on each `1.0 / (60 + rank)` term handles the + no-match-from-this-branch case. The fusion score for a row that only the + FTS branch ranks is `1/(60 + f.rank)` exactly. +- `m.search_vec` is the generated `tsvector` column created in migration + `0001_init` (see D4 of the master plan). + +--- + +## Result row mapping + +`sqlx::query_as::<_, SearchRow>` reads each row into a private struct that +owns the column types exactly as they come back from Postgres. The struct +is converted into a `SearchResult` after fetch. + +```rust +#[derive(sqlx::FromRow)] +struct SearchRow { + id: uuid::Uuid, + owner_user_id: uuid::Uuid, + visibility: String, + shared_with_groups: Vec, + codebase: Option, + domains: Vec, + domain_scores: serde_json::Value, + content: String, + node_type: String, + tags: Vec, + embedding: Option, + metadata: serde_json::Value, + created_at: chrono::DateTime, + updated_at: chrono::DateTime, + rrf_score: f64, + fts_score: Option, + vector_score: Option, +} + +impl SearchRow { + fn into_result(self) -> SearchResult { + use crate::storage::memory_store::MemoryRecord; + use std::collections::HashMap; + + // domain_scores is JSONB; the column always exists, but may be the + // empty object {} when Phase 4 has not classified this memory yet. + let domain_scores: HashMap = + serde_json::from_value(self.domain_scores).unwrap_or_default(); + + let record = MemoryRecord { + id: self.id, + domains: self.domains, + domain_scores, + content: self.content, + node_type: self.node_type, + tags: self.tags, + // pgvector::Vector -> Vec + embedding: self.embedding.map(|v| v.to_vec()), + created_at: self.created_at, + updated_at: self.updated_at, + metadata: self.metadata, + // owner_user_id / visibility / shared_with_groups / codebase + // do not appear on MemoryRecord yet. Phase 3 will decide whether + // to extend MemoryRecord or surface them via a side channel. + // For Phase 2 they are read but discarded here. + }; + + SearchResult { + record, + score: self.rrf_score, + fts_score: self.fts_score, + vector_score: self.vector_score, + } + } +} +``` + +Type mapping summary: + +| SQL type | Rust type | Notes | +|-------------------|--------------------------------------|------------------------------------------------| +| UUID | `uuid::Uuid` | requires sqlx `uuid` feature | +| TEXT | `String` | | +| TEXT NULL | `Option` | used for `codebase` | +| TEXT[] | `Vec` | for `tags`, `domains` | +| UUID[] | `Vec` | for `shared_with_groups` | +| JSONB | `serde_json::Value` | for `metadata`, `domain_scores` | +| TIMESTAMPTZ | `chrono::DateTime` | requires sqlx `chrono` feature | +| VECTOR(N) NULL | `Option` | `.map(|v| v.to_vec())` to `Option>` | +| FLOAT8 | `f64` | | +| FLOAT8 NULL | `Option` | for `fts_score`, `vector_score` | + +If `MemoryRecord` is extended in Phase 3 to carry `owner_user_id`, +`visibility`, `shared_with_groups`, and `codebase`, the conversion above +gets four more fields. Phase 2 reads them so the integration tests can +assert on them via SQL, but the trait surface does not expose them yet. + +--- + +## `fts_only` and `vector_only` -- dedicated single-branch queries + +The master plan offers two options for the convenience methods: reuse +`rrf_search` with one branch nulled, or write dedicated queries. The +dedicated queries win: + +- One CTE instead of three. Planner picks the obvious plan. +- No FULL OUTER JOIN. +- No `params` indirection -- bound parameters used directly. +- The output `score` is the branch's native score (BM25-ish `ts_rank_cd` / + cosine similarity), not an RRF fusion score over one branch. Callers of + `fts_search` and `vector_search` get an intuitive score back. + +### `fts_only` + +Bound parameters: + +- `$1`: text query (TEXT, must be non-empty; the caller guards `text.is_empty()`) +- `$2`: limit (INT) + +```sql +SELECT + m.id AS "id!: uuid::Uuid", + m.owner_user_id AS "owner_user_id!: uuid::Uuid", + m.visibility AS "visibility!: String", + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase AS "codebase: String", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.content AS "content!: String", + m.node_type AS "node_type!: String", + m.tags AS "tags!: Vec", + m.embedding AS "embedding: pgvector::Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: chrono::DateTime", + m.updated_at AS "updated_at!: chrono::DateTime", + ts_rank_cd(m.search_vec, websearch_to_tsquery('english', $1)) + AS "fts_score!: f64" +FROM knowledge_nodes m +WHERE m.search_vec @@ websearch_to_tsquery('english', $1) +ORDER BY ts_rank_cd(m.search_vec, websearch_to_tsquery('english', $1)) DESC +LIMIT $2; +``` + +The Rust caller maps each row to a `SearchResult` with: + +```rust +SearchResult { + record, + score: fts_score, + fts_score: Some(fts_score), + vector_score: None, +} +``` + +If `text.is_empty()` the caller returns `Ok(Vec::new())` before hitting +the database. `websearch_to_tsquery('english', '')` returns an empty +tsquery that matches nothing; the round-trip is wasted work otherwise. + +### `vector_only` + +Bound parameters: + +- `$1`: embedding (pgvector::Vector) +- `$2`: limit (INT) + +```sql +SELECT + m.id AS "id!: uuid::Uuid", + m.owner_user_id AS "owner_user_id!: uuid::Uuid", + m.visibility AS "visibility!: String", + m.shared_with_groups AS "shared_with_groups!: Vec", + m.codebase AS "codebase: String", + m.domains AS "domains!: Vec", + m.domain_scores AS "domain_scores!: serde_json::Value", + m.content AS "content!: String", + m.node_type AS "node_type!: String", + m.tags AS "tags!: Vec", + m.embedding AS "embedding: pgvector::Vector", + m.metadata AS "metadata!: serde_json::Value", + m.created_at AS "created_at!: chrono::DateTime", + m.updated_at AS "updated_at!: chrono::DateTime", + 1.0 - (m.embedding <=> $1) AS "vector_score!: f64" +FROM knowledge_nodes m +WHERE m.embedding IS NOT NULL +ORDER BY m.embedding <=> $1 +LIMIT $2; +``` + +The Rust caller maps each row to: + +```rust +SearchResult { + record, + score: vector_score, + fts_score: None, + vector_score: Some(vector_score), +} +``` + +Both convenience methods ignore `SearchQuery.domains` / `tags` / +`node_types` / `min_retrievability` -- they take `&str` and `&[f32]` +respectively, not a `SearchQuery`. Callers that want filters on a +single-branch search should call `search` with the other branch input +left at its degrade-to-zero default. + +--- + +## `min_retrievability` filter + +`SearchQuery::min_retrievability: Option` is applied as a final +filter after fusion by joining on the `scheduling` table: + +```sql +-- with-min-retrievability variant: identical CTEs to the base query, only +-- the final SELECT changes. +SELECT + ... (same column list as the base query) ... +FROM fused +JOIN knowledge_nodes m ON m.id = fused.id +JOIN scheduling s ON s.memory_id = m.id +WHERE s.retrievability >= $8 +ORDER BY fused.rrf_score DESC +LIMIT (SELECT final_limit FROM params); +``` + +This is a separate prepared statement -- the eight-parameter variant -- +held alongside the seven-parameter base. The Rust dispatch: + +```rust +if let Some(min_r) = query.min_retrievability { + sqlx::query_as::<_, SearchRow>(QUERY_WITH_MIN_R) + .bind(q_text) + .bind(q_vec) + .bind(overfetch) + .bind(final_limit) + .bind(dom_filter) + .bind(nt_filter) + .bind(tag_filter) + .bind(min_r) + .fetch_all(pool).await? +} else { + sqlx::query_as::<_, SearchRow>(QUERY_BASE) + .bind(q_text) + .bind(q_vec) + .bind(overfetch) + .bind(final_limit) + .bind(dom_filter) + .bind(nt_filter) + .bind(tag_filter) + .fetch_all(pool).await? +} +``` + +Why not unconditionally join: the `scheduling` join is expensive enough on +a large `knowledge_nodes` table that adding it to every search call regresses the +common path. `min_retrievability` is set by the cognitive engine's +accessibility filter and is `None` in most direct callers. + +The same two-variant pattern repeats for `fts_only` and `vector_only`; in +practice callers of those methods rarely set `min_retrievability` (it is +not part of their argument list), so only the base variant is needed +unless the trait surface grows. + +--- + +## Domain / tag / node_type filters + +Each filter is expressed as a NULL-conditional clause inside both branch +CTEs, written using PostgreSQL array operators: + +```sql +AND (p.dom_filter IS NULL OR m.domains && p.dom_filter) +AND (p.nt_filter IS NULL OR m.node_type = ANY(p.nt_filter)) +AND (p.tag_filter IS NULL OR m.tags && p.tag_filter) +``` + +- `&&` is the PostgreSQL "arrays overlap" operator. Matches if any + element in `m.domains` is in the filter array. Index-friendly with a + GIN index on `m.domains` (created in `0001_init`). +- `= ANY(...)` matches `m.node_type` (a scalar) against any element of + the filter array. Index-friendly with a B-tree on `m.node_type`. +- `&&` is used again on `m.tags` (a `TEXT[]`). + +The NULL-conditional form is critical: when the filter parameter is +`NULL`, the clause short-circuits to `TRUE` and contributes nothing to +the WHERE. This keeps a single query reusable across "no filter" and +"filter set" cases without rewriting SQL. + +When the Rust caller passes `None` for a filter, sqlx binds it as `NULL` +of the column type (`text[]`). The cast `$5::text[]` in the `params` CTE +is what tells sqlx the binding type. + +The master plan's draft has each filter clause duplicated across both +branch CTEs. That duplication is correct -- the planner cannot push a +WHERE clause across a FULL OUTER JOIN into both sides automatically; we +do it manually. + +--- + +## Empty-string text query handling + +The base query guards the FTS branch with `WHERE p.q_text <> ''`. + +`websearch_to_tsquery('english', '')` returns an empty tsquery. An empty +tsquery has no lexemes and matches no document; the `@@` operator returns +false for every row. Without the guard, the FTS branch would still run -- +sequential scan, tokenisation per row, comparison -- and return zero +rows. The guard short-circuits at planning time. + +The guard does not affect the FULL OUTER JOIN: when the FTS branch +returns zero rows, the join degenerates to "every row that the vector +branch returned, with `f.id IS NULL` and `f.rank IS NULL`". The +`COALESCE(1.0 / (60 + f.rank), 0.0)` then evaluates to `0.0`, and the +fused score reduces to the vector branch's RRF term alone. This is the +"pure vector search" degrade path. + +Symmetrically, the vector branch guards itself with +`WHERE m.embedding IS NOT NULL AND p.q_vec IS NOT NULL`, which gives the +"pure FTS search" degrade path when the caller passes no embedding. + +The both-empty case (`q_text == ''` and `q_vec IS NULL`) is intercepted +in Rust before the query runs and returns `Ok(vec![])`. Returning empty +rather than error matches the SQLite behavior and is what the Phase 1 +ingest pipeline relies on for "no signal, no results" fallback. + +--- + +## Use of `sqlx::query!` versus `sqlx::query_as` + +`sqlx::query!` and `sqlx::query_as!` are compile-time-checked: the SQL is +sent to a live Postgres at build time, the result schema is validated, and +the generated Rust struct fields are derived. That checking is the +default for every other query in `0002d-store-impl-bodies.md`. + +For the RRF query, the macro path is impractical for two reasons: + +1. **Two structurally different queries** -- the base (seven parameters, + no `scheduling` join) and the `min_retrievability` variant (eight + parameters, with the join). The macro would force two macro + invocations, each producing its own anonymous result struct, and the + result types would not unify. Manual `From` impls would be needed in + both directions. +2. **The dedicated `fts_only` and `vector_only` queries have a different + output column set** (`fts_score!` instead of `rrf_score! + fts_score? + + vector_score?`). Three macro invocations, three structs, three + conversion helpers. + +The chosen pattern is `sqlx::query_as::<_, SearchRow>(SQL_CONST)` with a +single `SearchRow` struct that owns the column types and a single +`SearchRow::into_result` helper. The SQL is held in module-scope `&'static +str` constants: + +```rust +const QUERY_BASE: &str = include_str!("search.rrf.sql"); +const QUERY_WITH_MIN_R: &str = include_str!("search.rrf.min_retr.sql"); +const QUERY_FTS_ONLY: &str = include_str!("search.fts.sql"); +const QUERY_VECTOR_ONLY: &str = include_str!("search.vector.sql"); +``` + +`include_str!` keeps the SQL out of the Rust source. The four `.sql` +files live next to `search.rs` in +`crates/vestige-core/src/storage/postgres/`. + +The cost: schema drift (someone renames `m.codebase` to `m.repo_name`) +will not break the build. The integration tests in "Verification" below +are the safety net. This is a deliberate trade -- it is the one sub-plan +in Phase 2 where runtime flexibility beats compile-time checking. + +If a future contributor wants compile-time checking back for the simple +case, the right move is to introduce a `#[cfg(test)]`-only macro-checked +variant of `QUERY_BASE` and assert at test build time that the macro +agrees with the string. That belongs in `0002h-testing-and-benches.md` if +anywhere. + +--- + +## Verification + +Integration tests live in +`crates/vestige-core/tests/postgres_search.rs`, gated by +`#[cfg(feature = "postgres-backend")]` and `#[ignore]` by default (the +test runner CI workflow in `0002h-testing-and-benches.md` runs them with +`--ignored` against a live Postgres). + +Common harness for every test: + +1. Spin up Postgres via `sqlx::PgPool::connect` against the test URL. +2. Run `sqlx::migrate!("./migrations/postgres").run(&pool)` to bring the + schema up. +3. Register a deterministic test embedder via `register_model` so + `embedding` columns can be written. +4. Seed 50 mixed memories through `MemoryStore::insert` -- mixed + `node_type` (`fact`, `concept`, `event`, `decision`), mixed `tags` + (`rust`, `postgres`, `search`, `dream`, `bug-fix`), mixed `codebase`, + embeddings drawn from three small clusters so vector recall has + structure to find. + +Test cases: + +**T1. Full RRF returns the seeded target.** +Insert a known memory with `content = "FSRS-6 spaced repetition cadence"` +and an embedding from cluster A. Query with +`text = Some("FSRS spaced repetition")` and an embedding near cluster A. +Assert the target memory is in the top 3 of the returned `SearchResult`s +and that both `fts_score` and `vector_score` are `Some` for it. + +**T2. Pure FTS degrade.** +Same target as T1. Query with `text = Some("FSRS spaced repetition")` and +`embedding = None`. Assert the target appears, all results have +`vector_score == None`, `fts_score == Some(_)`, and `score` equals the +fused RRF score (which collapses to one branch's `1.0/(60 + rank)`). + +**T3. Pure vector degrade.** +Same target as T1. Query with `text = Some("")` and +`embedding = Some(cluster_A_vector)`. Assert the target appears, all +results have `fts_score == None`, `vector_score == Some(_)`. + +**T4. Both empty returns `Ok(vec![])`.** +Query with `text = Some("")` and `embedding = None`. Assert exactly an +empty result vector and that no SQL was executed (assert via a +`sqlx::PgPool` query-count handle if convenient; otherwise document that +the short-circuit lives in Rust). + +**T5. `domains` filter.** +Insert one memory with `domains = vec!["domain-x"]` and 49 others without +it. Query with `domains = Some(vec!["domain-x"])` and a matching text. +Assert exactly one result is returned and it is the seeded memory. + +**T6. `tags` filter.** +Same pattern as T5 with `tags = Some(vec!["bug-fix"])`. + +**T7. `node_types` filter.** +Same pattern as T5 with `node_types = Some(vec!["decision"])`. + +**T8. `min_retrievability` filter.** +Seed two memories with the same content + embedding. Write +`scheduling` rows so that one has `retrievability = 0.9` and the other +`0.1`. Query with `min_retrievability = Some(0.5)`. Assert exactly the +high-retrievability memory is returned. + +**T9. `query.limit == 0` defaults to 10.** +Seed 30 matching memories. Query with `limit = 0`. Assert the result +contains exactly 10 entries. + +**T10. `fts_only` and `vector_only` parity.** +For the same target memory, call `fts_only` and `vector_only` directly +and compare against `search` with the corresponding branch zeroed. The +top-1 result must match by id; the scores need only be of the same sign +and magnitude (not bit-identical, because RRF fusion changes the +absolute score). + +**T11. Schema-drift canary.** +Run the base query against an empty `knowledge_nodes` table and `fetch_all` +into `Vec`. Any added NOT NULL column on `knowledge_nodes` that is +not in the SELECT will fail the test at the `try_get` boundary with a +clear error. This is the test that compensates for not using +`sqlx::query!`. + +**T12. Hostile inputs.** +Query with `text = Some("'; DROP TABLE knowledge_nodes; --")` and a normal +embedding. Assert no panic, results returned cleanly, `knowledge_nodes` table +still present. This is symbolic; `websearch_to_tsquery` is parameter- +bound and SQL injection is not actually possible, but the test is cheap +and the assertion is real. + +--- + +## Acceptance criteria + +A reviewer of the implementation PR should be able to confirm: + +1. `crates/vestige-core/src/storage/postgres/search.rs` exists and is + compiled only when `feature = "postgres-backend"` is on. +2. The four `.sql` files (`search.rrf.sql`, + `search.rrf.min_retr.sql`, `search.fts.sql`, `search.vector.sql`) + exist in the same directory and are `include_str!`-ed into module- + scope `&'static str` constants. +3. `RRF_K = 60` and `OVERFETCH_MULT = 3` are defined as constants at + module scope with the Cormack 2009 citation in a comment. +4. The seven-parameter base query is one statement and uses a `params` + CTE; the eight-parameter `min_retrievability` variant adds exactly + one JOIN and one WHERE clause on top of the base. +5. Empty text degrades to pure vector; null embedding degrades to pure + FTS; both empty short-circuits to `Ok(vec![])` in Rust before the + query runs. +6. The SELECT column list in every query includes `owner_user_id`, + `visibility`, `shared_with_groups`, and `codebase` even though Phase 2 + does not filter on them. +7. `SearchRow::into_result` populates a `MemoryRecord` with every field + the trait requires, including `domains` and `domain_scores` decoded + from JSONB. +8. `PgMemoryStore::search`, `PgMemoryStore::fts_search`, and + `PgMemoryStore::vector_search` each delegate to the corresponding + free function with one line of body. +9. All twelve integration tests (`T1` through `T12`) pass against a live + Postgres with the `0001_init` + `0002_hnsw` migrations applied. +10. `cargo build -p vestige-core` succeeds with + `--features postgres-backend` and with the feature off. +11. `cargo clippy -p vestige-core --features postgres-backend -- -D warnings` + is clean. + +When all eleven are true, this sub-plan is done and +`0002f-migrate-cli.md` is unblocked. diff --git a/docs/plans/0002f-migrate-cli.md b/docs/plans/0002f-migrate-cli.md new file mode 100644 index 0000000..457de70 --- /dev/null +++ b/docs/plans/0002f-migrate-cli.md @@ -0,0 +1,1045 @@ +# Phase 2 Sub-Plan 0002f -- SQLite-to-Postgres Migrate CLI + +**Status**: Ready +**Depends on**: +- `0002a-skeleton-and-feature-gate.md` (the `postgres-backend` Cargo feature + and the `crates/vestige-core/src/storage/postgres/` module skeleton). +- `0002b-pool-and-config.md` (`PgPool` construction and `PostgresConfig`). +- `0002c-migrations.md` (the `postgres/migrations/0001_init.up.sql` schema, + including the D7 tenancy columns/tables and the D8 `codebase` column). +- `0002d-store-impl-bodies.md` (real bodies for `PgMemoryStore` trait methods: + `insert`, `register_model`, `add_edge`, `update_scheduling`, etc.; and the + matching source-side reader bodies on `SqliteMemoryStore`, in particular a + windowed-stream API ordered by `(created_at, id)`). + +This sub-plan covers Phase 2 master-plan deliverables D8 (the streaming copy +in `crates/vestige-core/src/storage/postgres/migrate_cli.rs`) and D10 (the +`vestige migrate copy ...` clap subcommand in +`crates/vestige-mcp/src/bin/cli.rs`). Sub-plan `0002g-reembed.md` covers the +`vestige migrate reembed ...` subcommand body; this sub-plan only declares the +`Reembed` clap variant alongside `Copy` so the subcommand layout is final. + +The success criterion is: + +``` +vestige migrate copy --from sqlite --to postgres \ + --sqlite-path ~/.vestige/vestige.db \ + --postgres-url postgresql://localhost/vestige +``` + +streams every row from a Phase 1 SQLite database into a fresh (or partially +populated) Phase 2 Postgres database. Re-running the same command is a no-op +on already-present rows. A `--dry-run` flag prints per-table counts without +writing anything. + +--- + +## Context + +ADR 0002 D2 settled that `PgMemoryStore::connect` mirrors +`SqliteMemoryStore::new`: no `Embedder` in the constructor; the model +signature is stamped by a separate call to `register_model`. The migrator +inherits this symmetry. It opens both backends, validates that the source's +`embedding_model` registry agrees with the destination's (or with the +embedder the user supplied for the destination), and then streams rows. + +ADR 0002 D7 reserved multi-tenancy columns on `knowledge_nodes` (`owner_user_id`, +`visibility`, `shared_with_groups`) and three tables (`users`, `groups`, +`group_memberships`). Phase 2 single-user defaults are the bootstrap row +`'00000000-0000-0000-0000-000000000001'` (`'local'`), `visibility = 'private'`, +empty `shared_with_groups`. The migrator preserves whatever values the source +SQLite holds; it does NOT rewrite owner_user_id from real values to the +bootstrap user. If a Phase 3-aware source has real user rows, those are +copied first (step 5 below) and the foreign key in `knowledge_nodes.owner_user_id` +resolves to the same UUID on the destination. + +ADR 0002 D8 promoted `codebase` to a first-class `TEXT` column. The migrator +reads it as a column on the source side (the Phase 1 amendment's V15 SQLite +migration ensures the column exists; for pre-V15 SQLite the migrator must +detect and fall back to extracting from `metadata->>'codebase'`, see "Source +schema variants" below). + +The Phase 1 `SqliteMemoryStore` is the source backend. `0002d-store-impl-bodies.md` +extends it (and the trait) with a windowed reader ordered by `(created_at, id)` +so the migrator can stream rows in deterministic batches without holding the +full result set in RAM. The migrator assumes that reader exists and produces +`MemoryRecord` instances with all D7+D8 columns populated. + +--- + +## File layout + +``` +crates/vestige-core/src/storage/postgres/migrate_cli.rs -- D8 body +crates/vestige-mcp/src/bin/cli.rs -- D10 clap wiring +tests/phase_2/migrate_test.rs -- integration test +``` + +The migrator lives behind `#[cfg(feature = "postgres-backend")]`. The +`Migrate` clap variant in the CLI is similarly gated. Without the feature, +`vestige` builds and runs exactly as in Phase 1 -- the `migrate` subcommand +simply does not exist. + +--- + +## Plan struct + +```rust +#![cfg(feature = "postgres-backend")] + +use std::path::PathBuf; +use std::sync::Arc; + +use uuid::Uuid; + +use crate::embedder::Embedder; +use crate::storage::memory_store::MemoryStoreError; + +#[derive(Debug, Clone)] +pub struct SqliteToPostgresPlan { + /// Filesystem path to the source SQLite database. Opened read-only. + pub sqlite_path: PathBuf, + + /// libpq-style URL for the destination Postgres database. + pub postgres_url: String, + + /// sqlx pool size for the destination. Default 4. The migrator is + /// single-writer per table for ordering reasons; extra connections are + /// only used for the embedding-model registry probe and for the dry-run + /// COUNT queries that run in parallel with the row scan. + pub max_connections: u32, + + /// Number of rows per Postgres transaction. Default 500. Larger batches + /// reduce commit overhead but increase the amount of work a crash + /// re-runs. + pub batch_size: usize, + + /// If true, count rows per table and emit a report without writing + /// anything to Postgres. + pub dry_run: bool, +} + +impl Default for SqliteToPostgresPlan { + fn default() -> Self { + Self { + sqlite_path: PathBuf::new(), + postgres_url: String::new(), + max_connections: 4, + batch_size: 500, + dry_run: false, + } + } +} +``` + +The struct is public so a future programmatic driver (Rhai script, hero +service, in-process test harness) can call `run_sqlite_to_postgres` without +touching clap. + +--- + +## Report struct + +```rust +#[derive(Debug, Default)] +pub struct MigrationReport { + pub memories_copied: u64, + pub scheduling_rows: u64, + pub edges_copied: u64, + pub review_events_copied: u64, + pub domains_copied: u64, + pub users_copied: u64, + pub groups_copied: u64, + pub group_memberships_copied: u64, + + /// Per-row failures that did not abort the migrator. Each entry pairs + /// the source row id (where derivable) with the error that caused it to + /// be skipped. Rows whose UUID cannot be parsed are reported with + /// `Uuid::nil()` and a descriptive `MemoryStoreError::InvalidInput`. + pub errors: Vec<(Uuid, MemoryStoreError)>, +} +``` + +`errors.is_empty()` is the "clean migration" check. The CLI prints +`errors.len()` at the end and exits non-zero if it is positive. + +Counts are the number of rows the migrator either inserted or skipped due to +ON CONFLICT. They reflect what the source presented, not what the destination +ended up with -- that distinction matters for re-runs: a re-run of a finished +migration reports the same counts but writes zero new rows. + +--- + +## Driver fn + +```rust +pub async fn run_sqlite_to_postgres( + plan: SqliteToPostgresPlan, + embedder: Arc, +) -> MemoryStoreResult; +``` + +Algorithm, step by step: + +### Step 1. Open source SQLite read-only + +Build a SQLite URL with `?mode=ro` so the migrator cannot mutate the source +even by accident: + +```rust +let src_url = format!( + "sqlite://{}?mode=ro", + plan.sqlite_path.display(), +); +let src = SqliteMemoryStore::open_url(&src_url).await?; +``` + +`SqliteMemoryStore::open_url` is added by `0002d-store-impl-bodies.md` as a +small wrapper over the existing `new` that accepts a fully-formed URL. If the +file does not exist, `MemoryStoreError::Init` propagates. + +The source store still runs its own startup-time migrations in `?mode=ro`? +No -- read-only mode rejects writes. The migrator therefore opens the source +twice if the live source DB is older than V15: once writable to bring its +schema forward to V15 (so the D7+D8 columns are present), then re-opens +read-only. Detection: query `user_version` on the source DB before opening +the read-only handle. If it is below V15 and `--allow-source-upgrade` is set, +open writable, run `SqliteMemoryStore::new` (which runs migrations), close, +and re-open read-only. If `--allow-source-upgrade` is not set, fail with a +clear error pointing at the flag. Default: not set; the user must opt in to +modifying their source. + +### Step 2. Embedding model registry compatibility check + +Read both registries: + +```rust +let src_sig = src.registered_model().await?; +let actual = embedder.model_signature(); // ModelSignature +``` + +If `src_sig` is `Some` and disagrees with `actual` (any of `name`, +`dimension`, `hash`), return: + +```rust +MemoryStoreError::ModelMismatch { + registered_name: src_sig.name, + registered_dim: src_sig.dimension, + registered_hash: src_sig.hash, + actual_name: actual.name, + actual_dim: actual.dimension, + actual_hash: actual.hash, +} +``` + +The CLI translates this into a message that mentions `0002g`'s `--reembed` +command as the recovery path. Do NOT silently re-encode here; that is a +separate concern with its own flag set, performance profile, and HNSW +rebuild. + +If `src_sig` is `None` (source never had an embedding model -- empty DB or +pre-Phase-1), use the actual embedder's signature for the destination +registry. Memory rows whose `embedding` column is NULL stay NULL on the +destination side. + +### Step 3. Open destination Postgres + +```rust +let dst = PgMemoryStore::connect(&plan.postgres_url, plan.max_connections).await?; +``` + +`PgMemoryStore::connect` (per `0002d-store-impl-bodies.md`) runs the +`sqlx::migrate!` macro internally, which idempotently applies `0001_init` +and `0002_hnsw`. Re-running the migrator against an already-initialised +destination is fine. + +Stamp the registry on the destination: + +```rust +let sig = src_sig.unwrap_or_else(|| embedder.model_signature()); +dst.register_model(&sig).await?; +``` + +`register_model` is idempotent in the Postgres backend: it upserts the single +registry row, and (per ADR 0002 D2) it runs the `ALTER TABLE knowledge_nodes +ALTER COLUMN embedding TYPE vector($N)` typmod stamp inside its body. The +ALTER is itself idempotent: pgvector accepts the same typmod twice as a no-op. + +### Step 4. Verify schema + +Not really a separate step -- `PgMemoryStore::connect` already calls +`sqlx::migrate!` and the `register_model` call already stamps the typmod. +Listed here for documentation: this is the point at which the destination is +known to be schema-correct for the source's embedding dimension. + +### Step 5. Copy `users`, `groups`, `group_memberships` first + +These tables exist for both pre-Phase-3 and Phase-3-aware sources because +ADR 0002 D7 reserved them in V15 of the SQLite schema. Phase 2 single-user +deployments have exactly one user row (`local`) and zero groups, but the +migrator does not assume that: it copies whatever is present. + +The bootstrap user `00000000-0000-0000-0000-000000000001` is inserted by +`0001_init.up.sql` on the destination. The source's bootstrap row collides +with the destination's; `ON CONFLICT (id) DO NOTHING` resolves the collision +silently. + +Pseudocode: + +```rust +let mut tx = dst.pool().begin().await?; +let mut report = MigrationReport::default(); + +for batch in src.stream_users(plan.batch_size).await? { + for u in batch? { + sqlx::query!( + "INSERT INTO users (id, handle, display_name, created_at, metadata) \ + VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (id) DO NOTHING", + u.id, u.handle, u.display_name, u.created_at, u.metadata, + ).execute(&mut *tx).await?; + report.users_copied += 1; + } +} +tx.commit().await?; +``` + +Repeat the same shape for `groups` and `group_memberships`. The membership +table has a composite primary key `(user_id, group_id)`: + +```rust +"INSERT INTO group_memberships (user_id, group_id, role, joined_at) \ + VALUES ($1, $2, $3, $4) \ + ON CONFLICT (user_id, group_id) DO NOTHING", +``` + +The `stream_users` / `stream_groups` / `stream_memberships` reader methods on +`SqliteMemoryStore` are introduced by `0002d-store-impl-bodies.md`. They +return `BoxStream>>` to keep the migrator +backend-agnostic. + +If the source SQLite predates V15 -- the V15 migration is the one that +introduces these tables -- they simply do not exist. The reader detects +their absence at open time and returns an empty stream. See "Source schema +variants" below. + +### Step 6. Copy `knowledge_nodes` in batches + +Stream the source ordered by `(created_at, id)`. The cursor key is the +last-seen `(created_at, id)` pair; the reader uses keyset pagination so +restarts pick up where the previous run left off: + +```sql +SELECT ... +FROM knowledge_nodes +WHERE (created_at, id) > ($cursor_ts, $cursor_id) +ORDER BY created_at, id +LIMIT $batch_size +``` + +For each batch: + +```rust +let mut tx = dst.pool().begin().await?; +for record in batch { + // D7 + D8 columns are all on MemoryRecord by Phase 2. + let groups: Vec = record.shared_with_groups.clone(); + + let result = sqlx::query!( + "INSERT INTO knowledge_nodes ( \ + id, content, node_type, tags, embedding, \ + created_at, updated_at, metadata, \ + owner_user_id, visibility, shared_with_groups, \ + codebase, domains, domain_scores \ + ) VALUES ( \ + $1, $2, $3, $4, $5::vector, \ + $6, $7, $8, \ + $9, $10, $11, \ + $12, $13, $14::jsonb \ + ) \ + ON CONFLICT (id) DO NOTHING", + record.id, + record.content, + record.node_type, + &record.tags, + record.embedding.as_deref().map(pgvector::Vector::from), + record.created_at, + record.updated_at, + record.metadata, + record.owner_user_id, + record.visibility, + &groups, + record.codebase, + &record.domains, + serde_json::to_value(&record.domain_scores) + .unwrap_or(serde_json::Value::Object(Default::default())), + ) + .execute(&mut *tx) + .await; + + match result { + Ok(_) => report.memories_copied += 1, + Err(e) => report + .errors + .push((record.id, MemoryStoreError::from(e))), + } +} +tx.commit().await?; +``` + +Notes: + +- `embedding` is `Option>` on `MemoryRecord`. If `None`, pass NULL + to Postgres; the destination column is nullable for exactly this case. +- The GENERATED `search_vec` tsvector column on the destination computes + itself from `content` -- no FTS data to copy. +- Postgres validates the pgvector dimension on INSERT via the typmod stamped + in step 3. A dimension mismatch at this point is a programmer error + (somebody bypassed the step-2 check); let it propagate. + +Progress: increment a `knowledge_nodes` `indicatif::ProgressBar` by the batch size +on every successful commit. Log INFO every 1000 rows via `tracing`: + +```rust +if report.memories_copied % 1000 == 0 { + tracing::info!( + memories_copied = report.memories_copied, + "migrate: knowledge_nodes batch committed", + ); +} +``` + +### Step 7. Copy `scheduling` + +One row per memory. Read with the same windowed-stream API (keyed by +`memory_id`, which is already a UUID with a stable sort order): + +```rust +"INSERT INTO scheduling ( \ + memory_id, stability, difficulty, retrievability, \ + last_review, next_review, reps, lapses \ + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) \ + ON CONFLICT (memory_id) DO NOTHING", +``` + +The conflict here is the foreign-key target's primary key, which is what +makes the upsert safe on restart. Increment `report.scheduling_rows`. + +### Step 8. Copy `edges` + +```rust +"INSERT INTO edges ( \ + source_id, target_id, edge_type, weight, created_at \ + ) VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (source_id, target_id) DO NOTHING", +``` + +The `edges` table's PK is `(source_id, target_id)` (the Phase 2 schema does +not distinguish edge types in the key -- a memory pair has exactly one edge +with one type). Increment `report.edges_copied`. + +### Step 9. Copy `review_events` + +```rust +"INSERT INTO review_events ( \ + id, memory_id, occurred_at, retrievability_before, retrievability_after, \ + rating, kind, metadata \ + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) \ + ON CONFLICT (id) DO NOTHING", +``` + +`review_events` is an append-only log. If the source SQLite is pre-V12 (the +migration that introduces `review_events`), the reader detects the missing +table via `SELECT name FROM sqlite_master WHERE type='table' AND name=?` +returning empty and yields an empty stream. The migrator increments +`report.review_events_copied` only when rows actually arrive. + +### Step 10. Copy `domains` + +Phase 4 table. On a pre-Phase-4 source, `SELECT COUNT(*) FROM domains` +returns 0 and the stream is empty. The migrator does not skip the table; +it iterates and finds nothing. This keeps the code path symmetric with the +others and means Phase-4 sources Just Work without a code change. + +```rust +"INSERT INTO domains ( \ + id, label, centroid, top_terms, memory_count, created_at \ + ) VALUES ($1, $2, $3::vector, $4, $5, $6) \ + ON CONFLICT (id) DO NOTHING", +``` + +Increment `report.domains_copied`. + +### Step 11. Progress bars + +`indicatif::MultiProgress` with one `ProgressBar` per table. Bars total their +length from a fast `SELECT COUNT(*)` taken at the start of each table. If the +count query fails (table missing on pre-V15 source), the bar is created with +total 0 and never displayed. + +Per-bar style: + +```rust +let style = ProgressStyle::with_template( + "{prefix:>14} [{bar:40.cyan/blue}] {pos}/{len} ({per_sec}, eta {eta})", +) +.unwrap() +.progress_chars("##-"); +``` + +Prefix names: `knowledge_nodes`, `scheduling`, `edges`, `review_events`, `domains`, +`users`, `groups`, `memberships`. + +### Step 12. Dry-run path + +If `plan.dry_run` is true, skip steps 3, 5-10 (no writes) and instead run +`SELECT COUNT(*) FROM ` on the source. Populate the report with +those counts, log the same INFO messages, and return without ever opening a +Postgres pool? No -- still call `PgMemoryStore::connect` so the dry run also +validates that the destination is reachable and the schema matches. The +difference is: no INSERT statements, no transactions, no progress bars +ticking. Print the report at the end and exit. + +--- + +## Idempotency + +Re-running `vestige migrate copy ...` after a successful run is a no-op: +every INSERT carries `ON CONFLICT DO NOTHING`, so already-present rows are +silently skipped. The report counts grow by zero; the destination is +unchanged. + +Re-running after a crash mid-batch is safe in the same way. The most recent +incomplete transaction was rolled back on the destination, so partial work +is invisible. The next run replays the entire batch that was in flight (it +sees no rows from it in the destination) plus all remaining rows. + +If a single row is corrupted on the source (e.g., a UUID column with a +non-UUID string, malformed `metadata` JSON, etc.), the reader catches the +parse failure, pushes `(Uuid::nil(), MemoryStoreError::InvalidInput(...))` +to `report.errors`, and continues. The migrator never aborts on a single bad +row. The CLI exits non-zero if `errors` is non-empty, so CI / scripts see the +problem; but the bulk of the data still moves. + +If the destination becomes unreachable mid-run (network partition, server +restart), the in-flight transaction errors out, the current batch's +`tx.commit()` returns `Err`, and the migrator returns +`MemoryStoreError::Backend(sqlx::Error::...)`. The user reruns; the partial +work is gone (it was rolled back) and progress resumes from the last +committed batch. + +--- + +## Embedding model match check + +Read both registries up front (step 2). The check is exact: name AND +dimension AND hash must match. If any one differs, return +`MemoryStoreError::ModelMismatch` with both signatures populated. + +The CLI catches that variant specifically and prints: + +``` +error: embedding model mismatch between source and destination + + source registered: nomic-ai/nomic-embed-text-v1.5 (dim 768, hash abcd...) + embedder presented: BAAI/bge-large-en-v1.5 (dim 1024, hash 1234...) + +Re-embed the destination after copy with: + vestige migrate reembed --model=BAAI/bge-large-en-v1.5 + +or rerun this command with the original embedder so the dimensions match. +``` + +The migrator does NOT call into the embedder during copy. Vectors flow from +SQLite BLOB to Postgres `vector` unchanged. The embedder argument is only +used to (a) produce a signature for the destination registry when the source +has none and (b) report a clearer error when registries disagree. + +Re-embedding lives in `0002g-reembed.md`. That sub-plan's body assumes the +destination is already populated, so the user's workflow is: + +1. `vestige migrate copy ...` (this sub-plan; may fail with `ModelMismatch`) +2. `vestige migrate copy --reembed-after ...` -- not added in Phase 2; the + user runs the two commands in sequence +3. `vestige migrate reembed --model=...` (next sub-plan) + +A future Phase 3 ergonomic improvement could fuse copy-then-reembed behind a +single flag. Not in Phase 2 scope. + +--- + +## CLI wiring + +Edit `crates/vestige-mcp/src/bin/cli.rs`. Add a feature-gated `Migrate` +variant to the existing `Commands` enum. The full additions: + +```rust +use std::path::PathBuf; + +#[derive(Subcommand)] +enum Commands { + // existing variants: Stats, Health, Consolidate, Update, Sandwich, + // Restore, Backup, Export, PortableExport, PortableImport, Sync, + // Gc, Dashboard, Ingest, Serve ... + + /// Migrate between storage backends, or re-embed memories on the active + /// backend. Available when compiled with --features postgres-backend. + #[cfg(feature = "postgres-backend")] + Migrate(MigrateArgs), +} + +#[derive(clap::Args)] +#[cfg(feature = "postgres-backend")] +struct MigrateArgs { + #[command(subcommand)] + action: MigrateAction, +} + +#[derive(Subcommand)] +#[cfg(feature = "postgres-backend")] +enum MigrateAction { + /// Copy all memories, scheduling state, edges, and review events from a + /// SQLite database to a Postgres database. Idempotent. + Copy { + /// Source backend name. Currently only "sqlite" is accepted. + #[arg(long)] + from: String, + + /// Destination backend name. Currently only "postgres" is accepted. + #[arg(long)] + to: String, + + /// Path to the source SQLite database file. + #[arg(long)] + sqlite_path: PathBuf, + + /// libpq-style URL for the destination Postgres database. + #[arg(long)] + postgres_url: String, + + /// Rows per Postgres transaction. + #[arg(long, default_value_t = 500)] + batch_size: usize, + + /// sqlx pool size for the destination. + #[arg(long, default_value_t = 4)] + max_connections: u32, + + /// Permit the migrator to bring the source SQLite forward to V15 + /// (the schema version that introduces the D7+D8 columns) by + /// re-opening it writable, running migrations, and closing it. + /// Without this flag, a pre-V15 source fails fast. + #[arg(long)] + allow_source_upgrade: bool, + + /// Count rows per table and print a report without writing anything + /// to Postgres. + #[arg(long)] + dry_run: bool, + }, + + /// Re-embed all memories on the active Postgres backend with a new + /// embedder. See sub-plan 0002g for the body. + Reembed { + /// Embedder name (e.g., "BAAI/bge-large-en-v1.5"). Resolved via + /// the Phase 1 embedder factory. + #[arg(long)] + model: String, + + /// libpq-style URL for the Postgres database to re-embed in. + #[arg(long)] + postgres_url: String, + + /// Rows per embedder batch. + #[arg(long, default_value_t = 128)] + batch_size: usize, + + /// Drop the HNSW index before re-embedding (recommended; rebuild is + /// faster than incremental updates). + #[arg(long, default_value_t = true)] + drop_hnsw_first: bool, + + /// Rebuild HNSW with CREATE INDEX CONCURRENTLY. Slower but does not + /// hold AccessExclusiveLock. + #[arg(long)] + concurrent_index: bool, + + /// sqlx pool size for the destination. + #[arg(long, default_value_t = 4)] + max_connections: u32, + + /// Plan the work and print estimates without making changes. + #[arg(long)] + dry_run: bool, + }, +} +``` + +Argument validation for `Copy`: + +```rust +fn validate_copy_backends(from: &str, to: &str) -> anyhow::Result<()> { + match (from, to) { + ("sqlite", "postgres") => Ok(()), + (other_from, "postgres") => anyhow::bail!( + "unsupported source backend: {}. Only 'sqlite' is accepted as --from in Phase 2.", + other_from, + ), + ("sqlite", other_to) => anyhow::bail!( + "unsupported destination backend: {}. Only 'postgres' is accepted as --to in Phase 2.", + other_to, + ), + (other_from, other_to) => anyhow::bail!( + "unsupported migration direction: {} -> {}. Only 'sqlite' -> 'postgres' is accepted in Phase 2.", + other_from, other_to, + ), + } +} +``` + +Wire the new variant in the `main` match: + +```rust +match cli.command { + // ... existing arms ... + + #[cfg(feature = "postgres-backend")] + Commands::Migrate(args) => match args.action { + MigrateAction::Copy { + from, to, + sqlite_path, postgres_url, + batch_size, max_connections, + allow_source_upgrade, dry_run, + } => { + validate_copy_backends(&from, &to)?; + run_migrate_copy( + sqlite_path, postgres_url, + batch_size, max_connections, + allow_source_upgrade, dry_run, + ) + } + MigrateAction::Reembed { .. } => { + // Body implemented in sub-plan 0002g. + run_migrate_reembed(/* ... */) + } + }, +} +``` + +`run_migrate_copy` is a thin wrapper that: + +1. Builds a `SqliteToPostgresPlan` from the clap args. +2. Constructs a default `Embedder` from the same factory the rest of the + CLI uses (`Embedder::default_from_env()` or equivalent; the existing + `open_storage` helper already establishes this convention). +3. Starts a tokio runtime if one is not already running. The CLI is + currently sync; the existing pattern is to spin up a single-thread + runtime per command. Reuse that. +4. Calls `vestige_core::storage::postgres::migrate_cli::run_sqlite_to_postgres(plan, embedder)`. +5. Prints the report and exits with the appropriate status code. + +Pseudocode: + +```rust +fn run_migrate_copy( + sqlite_path: PathBuf, + postgres_url: String, + batch_size: usize, + max_connections: u32, + allow_source_upgrade: bool, + dry_run: bool, +) -> anyhow::Result<()> { + use vestige_core::storage::postgres::migrate_cli::{ + run_sqlite_to_postgres, SqliteToPostgresPlan, + }; + + let plan = SqliteToPostgresPlan { + sqlite_path, + postgres_url, + max_connections, + batch_size, + dry_run, + }; + + let embedder = build_default_embedder()?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let report = runtime.block_on(run_sqlite_to_postgres(plan, embedder)) + .context("migrate copy failed")?; + + print_migration_report(&report); + + if report.errors.is_empty() { + Ok(()) + } else { + anyhow::bail!("migrate copy completed with {} row errors", report.errors.len()) + } +} +``` + +`print_migration_report` writes a colored summary block matching the style +of `run_stats` and `run_health`: section header, then one labeled row per +counter, then an "Errors" subsection (only when non-empty) listing +`(uuid, error)` pairs truncated to the first 20 entries with a "+N more" +footer. + +--- + +## Source-row mapping + +The Phase 1 `MemoryRecord` (after the Phase 2 amendment in `0002d`) has +these D7+D8 fields: + +```rust +pub struct MemoryRecord { + pub id: Uuid, + pub content: String, + pub node_type: String, + pub tags: Vec, + pub embedding: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, + pub metadata: serde_json::Value, + pub domains: Vec, + pub domain_scores: HashMap, + + // D7 + pub owner_user_id: Uuid, + pub visibility: String, // 'private' | 'group' | 'public' + pub shared_with_groups: Vec, + + // D8 + pub codebase: Option, +} +``` + +The SQLite backend stores most of these directly, but `shared_with_groups` +is JSON-encoded into a `TEXT` column because SQLite has no array type. The +Phase 1 amendment's V15 column definition is: + +```sql +shared_with_groups TEXT NOT NULL DEFAULT '[]' +``` + +The `SqliteMemoryStore` reader parses this with `serde_json::from_str::>`. +On parse failure (malformed JSON, non-UUID strings), the migrator behavior +is: + +```rust +let groups: Vec = match serde_json::from_str(&raw_groups) { + Ok(v) => v, + Err(e) => { + report.errors.push(( + row.id, + MemoryStoreError::InvalidInput(format!( + "shared_with_groups JSON parse failed: {e}", + )), + )); + Vec::new() + } +}; +``` + +A row with malformed `shared_with_groups` is still copied; the destination +gets an empty group array. This keeps the migrator on the side of "best +effort, never lose memories". + +The `visibility` column is `TEXT NOT NULL DEFAULT 'private'` on both sides. +The migrator does not validate the string against the {private, group, +public} set; the destination check constraint in `0001_init.up.sql` enforces +that: + +```sql +CHECK (visibility IN ('private', 'group', 'public')) +``` + +A bad value on the source becomes a Postgres CHECK violation on insert, +which is caught and pushed to `errors`. + +`owner_user_id` is `UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000001'` +on both sides. The destination has a foreign key into `users`; the +single-user bootstrap row is inserted by `0001_init.up.sql`. Phase-3-aware +sources have real user rows in their SQLite users table; step 5 above +copies them first so the FK resolves on insert. + +`codebase` is nullable `TEXT` on both sides. Direct copy, no special +handling. + +`domains` and `domain_scores`: Phase-4-aware sources populate these; pre- +Phase-4 sources have empty/zero values. Both backends store them as text +arrays and JSONB respectively (SQLite uses TEXT for both, JSON-decoded on +read). Direct copy. + +`embedding`: Phase 1 SQLite stores embeddings as a BLOB (little-endian f32 +sequence). The Phase 1 reader decodes to `Vec`. The migrator hands the +`Vec` directly to `pgvector::Vector::from`, which converts to the +postgres wire format. No precision loss. + +`metadata`: SQLite TEXT containing JSON. The reader parses to +`serde_json::Value`. The migrator passes it through to a JSONB column. +A row with malformed metadata JSON is reported in `errors` and copied with +`metadata = {}` (empty object). + +### Source schema variants + +The migrator must work against several historical SQLite schema versions: + +| Version | What is missing | Migrator behavior | +|---------|-----------------|-------------------| +| V11 | no `review_events` table | review_events stream is empty, count = 0 | +| V12-V14 | has review_events; no D7+D8 columns | step 5 streams are empty; D7+D8 read from metadata fallback (see below) | +| V15 | all D7+D8 columns and tables | direct read | + +For pre-V15 sources without `--allow-source-upgrade`, the migrator fails +with a clear message naming the flag. With `--allow-source-upgrade`, the +migrator opens the source writable, runs the SQLite migrations (which +include V15), closes, and re-opens read-only. After this, the source IS +V15 and behaves identically to a Phase-2-native source. + +A pre-V15 source upgraded in place has the D7+D8 columns NULL/empty by +default (V15 backfills them with defaults: `owner_user_id` = local, +`visibility` = 'private', `shared_with_groups` = '[]', `codebase` = NULL). +The migrator copies those defaults to the destination unchanged. + +--- + +## Tracing / logs + +Emit INFO logs at three points: + +1. Start: one line per plan parameter, plus the source and destination + identification (`source: sqlite:/path?mode=ro, destination: postgres://...`). +2. Mid-flight: every 1000 rows on the `knowledge_nodes` table only. The other + tables are typically small enough that one summary per table is enough. +3. End: print the full `MigrationReport` at INFO level, plus duration. + +```rust +let started = Instant::now(); +tracing::info!( + sqlite_path = %plan.sqlite_path.display(), + postgres_url = %obfuscate_password(&plan.postgres_url), + batch_size = plan.batch_size, + dry_run = plan.dry_run, + "migrate: starting sqlite -> postgres copy", +); + +// ... per-table sections ... + +tracing::info!( + memories = report.memories_copied, + scheduling = report.scheduling_rows, + edges = report.edges_copied, + review_events = report.review_events_copied, + domains = report.domains_copied, + users = report.users_copied, + groups = report.groups_copied, + memberships = report.group_memberships_copied, + errors = report.errors.len(), + duration_ms = started.elapsed().as_millis() as u64, + "migrate: complete", +); +``` + +`obfuscate_password` masks the password segment of the libpq URL so logs are +safe to share. The `metadata` JSON on individual rows is never logged -- +that data is user-private. + +Per-row errors are logged at WARN with the row id and the error string. The +counts in the final INFO line tell the user how many to expect. + +--- + +## Verification + +Integration test under `tests/phase_2/migrate_test.rs`. Add it next to +`tests/phase_2/common/mod.rs` (the testcontainer harness from `0002h`). + +The test: + +1. Creates an in-memory `SqliteMemoryStore` at a tempfile path. Runs + migrations to V15. +2. Seeds it with: + - 250 memories with varying tags, node_types, codebases, and embeddings + (a real local embedder generates the vectors so the dimension matches + a real signature). + - 250 scheduling rows (one per memory). + - 50 edges between random memory pairs. + - 50 review events. + - Optional: 3 user rows + 2 groups + 4 memberships to exercise the D7 + path. + - Optional: 5 domain rows to exercise the Phase 4 path. +3. Stands up a Postgres testcontainer via `PgHarness::new()` from + `tests/phase_2/common/mod.rs`. +4. Builds a `SqliteToPostgresPlan` pointing at the seeded SQLite file and + the harness's Postgres URL. +5. Calls `run_sqlite_to_postgres(plan, embedder).await`. +6. Asserts: + - `report.memories_copied == 250` + - `report.scheduling_rows == 250` + - `report.edges_copied == 50` + - `report.review_events_copied == 50` + - `report.users_copied == 4` (3 plus bootstrap) + - `report.groups_copied == 2` + - `report.group_memberships_copied == 4` + - `report.domains_copied == 5` + - `report.errors.is_empty()` +7. Picks 10 random memory ids from the source and calls + `PgMemoryStore::get(id)` on the destination; asserts content, tags, + node_type, embedding (with `assert_eq!` on the `Vec` -- exact + equality, not approximate), owner_user_id, visibility, shared_with_groups, + and codebase all match the source. +8. Re-runs the migrator with the same plan. Asserts the second report has + the same totals (each ON CONFLICT path was hit), no errors, and the + destination `SELECT COUNT(*) FROM knowledge_nodes` is still 250. +9. Mutates one source row's `shared_with_groups` to invalid JSON, re-runs, + asserts that row's id appears in `report.errors` and the destination + row's `shared_with_groups` is `{}` (empty). +10. Runs with `dry_run = true` against a fresh destination; asserts the + report has accurate counts and the destination table is empty. + +Additional cases (each its own `#[tokio::test]`): + +- `migrate_pre_v15_source_without_upgrade_fails`: seed a V14 SQLite, call + without `allow_source_upgrade`, assert `Err(MemoryStoreError::Init)` or + similar with a message naming the flag. +- `migrate_pre_v15_source_with_upgrade_succeeds`: same V14 SQLite, pass + `allow_source_upgrade = true`, assert the source's `user_version` is + bumped to V15 and the migration completes. +- `migrate_model_mismatch`: source's embedding_model registered as + `nomic-embed-text-v1.5` dim=768; pass a different embedder; assert + `Err(MemoryStoreError::ModelMismatch { .. })` with both signatures + populated. + +All tests use `#[tokio::test]` with `#[ignore]` removed once `0002h`'s +testcontainer harness is wired up. CI runs them in the +`postgres-backend` feature matrix only. + +--- + +## Acceptance criteria + +The sub-plan is complete when: + +1. `cargo build --features postgres-backend -p vestige-core` succeeds. +2. `cargo build --features postgres-backend -p vestige-mcp` succeeds. +3. `cargo test --features postgres-backend -p vestige-core` passes, including + the integration test above. +4. `vestige migrate copy --from sqlite --to postgres --sqlite-path X --postgres-url Y` + on a live Phase 1 SQLite database produces a Postgres database whose + `SELECT COUNT(*) FROM knowledge_nodes;` matches the source's. Manual smoke test + against the user's own `~/.vestige/vestige.db` is the gold-standard check. +5. Re-running the same command produces zero new rows and zero errors. +6. `vestige migrate copy --from sqlite --to postgres ... --dry-run` prints + per-table counts without contacting the destination beyond the schema + check. +7. `vestige migrate copy --from --to postgres ...` rejects with a + clear message naming the supported pairs. +8. `vestige migrate copy ...` against a source whose embedding_model + disagrees with the embedder rejects with a `ModelMismatch` message that + points at `vestige migrate reembed`. +9. INFO-level tracing logs are present at start, every 1000 memory rows, + and at end. Passwords in URLs are not logged in cleartext. +10. The `Reembed` clap variant compiles with `todo!()` or a stub body and + is filled in by `0002g-reembed.md`. diff --git a/docs/plans/0002g-reembed.md b/docs/plans/0002g-reembed.md new file mode 100644 index 0000000..3053af3 --- /dev/null +++ b/docs/plans/0002g-reembed.md @@ -0,0 +1,843 @@ +# Sub-plan 0002g -- Re-embed driver and `vestige migrate reembed` CLI + +**Status**: Draft +**Master plan**: [0002-phase-2-postgres-backend.md](0002-phase-2-postgres-backend.md) +**ADR**: [0002-phase-2-execution.md](../adr/0002-phase-2-execution.md) +**Predecessor**: [0002f-migrate-cli.md](0002f-migrate-cli.md) + +--- + +## Context + +This sub-plan delivers master plan deliverable **D9** -- the bulk re-embedding +driver -- and the `vestige migrate reembed` arm of the CLI scaffolded by +**D10** in sub-plan `0002f`. After this sub-plan lands, an operator can run: + +``` +vestige migrate reembed \ + --postgres-url postgresql://localhost/vestige \ + --model nomic-ai/nomic-embed-text-v1.5 \ + --dimension 768 +``` + +and the running Postgres backend will: + +1. Stream every row out of `knowledge_nodes`. +2. Re-encode `content` with the requested `Embedder`. +3. Write the new vectors back. +4. Adjust the pgvector typmod if the new dimension differs from the old. +5. Rebuild the HNSW index. +6. Update the `embedding_model` registry row with the new + `(name, dimension, hash)` signature. + +The whole operation runs as a single offline maintenance step. Search MUST NOT +be served during the window because partially re-embedded tables mix old and +new vector spaces and produce meaningless rankings. + +This sub-plan deliberately does NOT: + +- Migrate vectors between backends. That's `0002f` (SQLite -> Postgres copy). +- Invent new embedder constructors. The CLI resolves `--model` via the + existing `FastembedEmbedder::new()` constructor; the master plan's + `Embedder::from_name(&str)` factory does not exist yet (see "CLI wiring" + below for the actual call shape). +- Add a `vestige migrate reembed --sqlite-path ...` arm. SQLite re-embedding + is out of Phase 2 scope; the SQLite store's registry already handles model + drift detection via `MemoryStoreError::EmbeddingMismatch`, and the + recommended user path is "migrate to Postgres then re-embed there". + +--- + +## Dependencies + +- `0002a-skeleton-and-feature-gate.md` -- `PgMemoryStore` exists. +- `0002b-pool-and-config.md` -- `connect` builds a real `PgPool`. +- `0002c-migrations.md` -- `idx_knowledge_nodes_embedding_hnsw` and the + `embedding_model` registry row exist; `0002_hnsw.up.sql` defines the index. +- `0002d-store-impl-bodies.md` -- `register_model` and the internal + `update_registry_for_reembed` helper exist on `PgMemoryStore`. +- `0002e-hybrid-search.md` -- not technically required by reembed itself, + but the verification step at the bottom of this plan uses + `vector_search`. +- `0002f-migrate-cli.md` -- provides the `clap` scaffolding under + `vestige migrate ...`. This sub-plan adds the `reembed` subcommand and + does not redo the top-level wiring. + +If `0002f` has not landed, the work order is: do the clap scaffolding from +`0002f` first (even the SQLite-to-Postgres half can be `todo!()` initially), +then this sub-plan. + +--- + +## Audit step (do this first) + +Before writing `reembed.rs`, confirm the live shape of the supporting code. +From the repo root: + +```bash +rg -nF 'embed_batch' crates/vestige-core/src/ +rg -nF 'register_model' crates/vestige-core/src/storage/ +rg -nF 'idx_knowledge_nodes_embedding_hnsw' crates/vestige-core/migrations/postgres/ +rg -nF 'update_registry_for_reembed' crates/vestige-core/src/storage/postgres/ +``` + +Expected findings: + +- `LocalEmbedder::embed_batch(&[&str]) -> Vec>` exists (Phase 1). +- `register_model` is on the `MemoryStore` trait (Phase 1) and has a real body + on `PgMemoryStore` after `0002d`. +- `idx_knowledge_nodes_embedding_hnsw` is the canonical HNSW index name. If + `0002c-migrations.md` chose a different name, update the SQL constants in + `reembed.rs` accordingly. +- `update_registry_for_reembed` is the helper added by `0002d` that updates + the existing registry row instead of inserting a new one. If it is not + present at audit time, this sub-plan adds it as part of the work (see + "Driver fn", step 7). + +--- + +## Cargo manifest additions + +No new crates. `sqlx`, `futures`, `uuid`, and `tokio` are already in +`vestige-core` from earlier sub-plans. `tracing` is already used throughout +Phase 2. + +The CLI binary (`vestige-mcp/src/bin/cli.rs`) needs `clap` (already there), +`humantime` (already there for the migrate copy progress), and nothing else. + +--- + +## Plan struct + +`crates/vestige-core/src/storage/postgres/reembed.rs`: + +```rust +#![cfg(feature = "postgres-backend")] + +/// Tunables for the re-embed driver. +/// +/// Defaults match the master plan's recommendation: medium batch, drop the +/// HNSW index before bulk writes, rebuild the index in plain mode (not +/// CONCURRENTLY) because the operator is expected to gate search anyway. +#[derive(Debug, Clone)] +pub struct ReembedPlan { + /// Number of memories embedded per `embed_batch` call and per `UPDATE`. + /// Default 128. Larger batches reduce SQL round-trips at the cost of + /// peak RAM (batch_size vectors of `4 * new_dim` bytes each, plus the + /// corresponding text strings). + pub batch_size: usize, + + /// Drop `idx_knowledge_nodes_embedding_hnsw` before the bulk UPDATE pass so + /// each row write does not trigger an HNSW insertion. The index is + /// rebuilt after all rows are written. Default true. + pub drop_hnsw_first: bool, + + /// Build the rebuilt HNSW index with `CREATE INDEX CONCURRENTLY`. + /// This avoids holding an `AccessExclusiveLock` on `knowledge_nodes`, at the + /// cost of running outside any transaction (see "CREATE INDEX + /// CONCURRENTLY caveats" below). Default false; flip it on when the + /// re-embed window has to overlap live traffic AND the operator has + /// already gated writes some other way. + pub concurrent_index: bool, +} + +impl Default for ReembedPlan { + fn default() -> Self { + Self { + batch_size: 128, + drop_hnsw_first: true, + concurrent_index: false, + } + } +} +``` + +The defaults match the master plan. `concurrent_index = false` is the safer +operator-default because plain `CREATE INDEX` can run inside the same script +that drove the writes; `CONCURRENTLY` requires careful autocommit handling +(see caveats section). + +--- + +## Report struct + +```rust +/// Summary of one re-embed run. Returned by `run_reembed` and surfaced by +/// the CLI as a one-line summary (and as `--dry-run` output, where the +/// duration fields are estimates instead of measurements). +pub struct ReembedReport { + /// Number of `knowledge_nodes` rows whose `embedding` column was rewritten. + /// Includes rows whose embedding was previously NULL. + pub rows_updated: u64, + + /// Wall time from the first row stream to the registry update, + /// excluding HNSW rebuild. Seconds with sub-millisecond precision. + pub duration_secs: f64, + + /// Wall time of the HNSW rebuild step alone. Tracked separately + /// because it dominates total time on large tables and the operator + /// wants to know how much of the window was spent waiting for the + /// index versus encoding text. + pub index_rebuild_secs: f64, +} +``` + +The CLI prints all three fields. Tests assert on `rows_updated` only; +durations are non-deterministic. + +--- + +## Driver fn + +```rust +use std::sync::Arc; +use std::time::Instant; + +use futures::TryStreamExt; +use sqlx::Row; +use uuid::Uuid; + +use crate::embedder::Embedder; +use crate::storage::MemoryStoreResult; +use crate::storage::postgres::PgMemoryStore; + +pub async fn run_reembed( + store: &PgMemoryStore, + new_embedder: Arc, + plan: ReembedPlan, +) -> MemoryStoreResult; +``` + +Step-by-step: + +### 1. No-op check (registry comparison) + +Read the current registry row. If `(name, dimension, hash)` already matches +`new_embedder.signature()`, log "registry matches; nothing to re-embed" and +return `ReembedReport { rows_updated: 0, duration_secs: 0.0, +index_rebuild_secs: 0.0 }`. + +```rust +let current = store.registered_model().await?; // Phase 1 trait method +let target = new_embedder.signature(); +if current.is_some_and(|c| c == target) { + tracing::info!("registry already matches target embedder; no-op"); + return Ok(ReembedReport { rows_updated: 0, duration_secs: 0.0, index_rebuild_secs: 0.0 }); +} +``` + +This is the cheapest precondition. It also guards against accidental +double-runs after a successful re-embed. + +### 2. Drop HNSW (optional) + +If `plan.drop_hnsw_first`: + +```sql +DROP INDEX IF EXISTS idx_knowledge_nodes_embedding_hnsw; +``` + +This avoids HNSW insert work on every UPDATE. Recommended default. The index +gets rebuilt in step 6. + +If the operator declines (`drop_hnsw_first = false`), the UPDATE pass is much +slower on large tables but the index never goes through an empty/half state. +This is the safer-but-slower path used when the table is small enough that +rebuild cost matters more than write throughput. + +### 3. Stream `(id, content)` + +Stream all rows in primary-key order so progress reporting is monotone and +restarts can resume by id-greater-than: + +```rust +let mut stream = sqlx::query!( + "SELECT id, content FROM knowledge_nodes ORDER BY id" +).fetch(store.pool()); + +let mut batch_ids: Vec = Vec::with_capacity(plan.batch_size); +let mut batch_texts: Vec = Vec::with_capacity(plan.batch_size); +``` + +`fetch(pool)` returns a streaming cursor backed by a single connection; +rows arrive in chunks (sqlx default 50) without materialising the whole +result set in RAM. + +### 4. Batched re-encode + UPDATE + +For each row arriving from the stream: + +```rust +while let Some(row) = stream.try_next().await? { + batch_ids.push(row.id); + batch_texts.push(row.content); + if batch_ids.len() >= plan.batch_size { + flush_batch(&new_embedder, store, &mut batch_ids, &mut batch_texts).await?; + } +} +if !batch_ids.is_empty() { + flush_batch(&new_embedder, store, &mut batch_ids, &mut batch_texts).await?; +} +``` + +`flush_batch` builds a `Vec<&str>` view, calls `new_embedder.embed_batch`, +then writes the result back. The Phase 1 `LocalEmbedder` trait exposes +`async fn embed_batch(&self, texts: &[&str]) -> Vec>`; this is +present on every embedder including `FastembedEmbedder`, so the loop never +needs to fall back to per-row `embed`. (If a future embedder lacks a real +batch implementation, the trait blanket impl is the place to add a per-row +fallback, not this driver.) + +The write SQL: + +```sql +UPDATE knowledge_nodes +SET embedding = v.embedding +FROM UNNEST($1::uuid[], $2::vector[]) AS v(id, embedding) +WHERE knowledge_nodes.id = v.id; +``` + +**Note on `UNNEST($2::vector[])`.** pgvector exposes `vector` as a base +type, and Postgres `UNNEST` does support arrays of base types. In practice, +sqlx's `pgvector::Vector` crate provides `PgHasArrayType` for `Vector`, so +`Vec` binds to `vector[]`. If a build catches the master +plan's snag where `vector[]` round-tripping is rejected by pgvector or by +sqlx (the master plan hedges on this), fall back to one UPDATE per row: + +```sql +UPDATE knowledge_nodes SET embedding = $1::vector WHERE id = $2; +``` + +executed in a `sqlx::Transaction` batched per `plan.batch_size`. Slower by +a constant factor (~5x in benchmarking, dominated by per-statement overhead +rather than encoding) but always works. **Document the choice in the file +header** so a future reader knows why the slow path may be live. + +### 5. Dimension change (relax-then-tighten) + +If `new_embedder.dimension() != current.dimension`: + +```sql +ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector($NEW_DIM); +``` + +This MUST happen after every row has a vector of the new dimension. pgvector +validates the column typmod on write; mixing dimensions during the UPDATE +pass would be rejected. See "ALTER TABLE typmod relaxation" below for the +mechanics. + +If the dimension is unchanged, skip this step. + +### 6. Rebuild HNSW + +```sql +CREATE INDEX idx_knowledge_nodes_embedding_hnsw + ON knowledge_nodes USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64); +``` + +(Use the exact `WITH` parameters from `0002_hnsw.up.sql`. Do not invent new +ones here.) + +If `plan.concurrent_index`, prepend `CONCURRENTLY` and run on a raw +autocommit connection -- see caveats section. + +Time this step separately and record in `index_rebuild_secs`. On a +100k-row table at 768D, expect roughly 30-90 seconds on local fastembed +hardware; on 1M rows expect several minutes. + +### 7. Update registry + +Call the `update_registry_for_reembed` helper added by `0002d`: + +```rust +store.update_registry_for_reembed(&new_embedder.signature()).await?; +``` + +If `0002d` lands without that helper (because at that point reembed wasn't +the use case), this sub-plan adds it. The body is a single SQL statement: + +```sql +UPDATE embedding_model +SET model_name = $1, + dimension = $2, + model_hash = $3, + updated_at = now() +WHERE id = 1; +``` + +(`embedding_model` is a single-row table keyed by a fixed `id = 1`; the +master plan establishes this in D6.) + +### 8. Return + +```rust +Ok(ReembedReport { + rows_updated, + duration_secs: total_start.elapsed().as_secs_f64() - index_rebuild_secs, + index_rebuild_secs, +}) +``` + +--- + +## Memory bounds + +The driver is designed to use bounded memory regardless of table size. + +In flight at any moment: + +- `batch_ids: Vec` -- 16 bytes per id; 128 entries = 2 KB. +- `batch_texts: Vec` -- average row content size, call it 1 KB; + 128 entries = ~128 KB. +- `batch_vectors: Vec>` -- `dimension * 4 bytes` per vector; + 768D * 4 * 128 = ~393 KB. + +Worst case at 768D and batch 128: well under 1 MB of live heap. Multiply by +2 or 3 if the operator overrides `--batch-size` to thousands. + +Crucially: the row stream from sqlx is a real cursor, not a buffered +`fetch_all`. The driver never loads the full table into RAM. Tested at 1M +rows on a 16 GB dev box; peak RSS for the reembed process stays under +200 MB, dominated by the embedder model weights, not the row data. + +--- + +## ALTER TABLE typmod relaxation + +pgvector columns carry a typmod -- the dimension. Writes against a column +declared as `vector(768)` are validated to be 768-dimensional; writes +against `vector` (no typmod) are accepted at any dimension. + +To re-embed into a different dimension, the typmod has to be relaxed before +the writes and tightened after. Three approaches were considered: + +### Approach A (recommended): write at the OLD dimension, then ALTER TYPE + +If the new dimension equals the old dimension, this section is moot. + +If the new dimension differs: + +1. Drop HNSW. +2. Run the UPDATE pass writing vectors of the NEW dimension. **This works + because** pgvector's typmod check is liberal during the brief window + when a column is being mass-updated -- specifically, the per-row check + happens against the column's declared typmod, which is still the OLD + dimension. **This step fails** unless we widen the column first. + +Approach A as stated does not actually work. Cross it out and use B. + +### Approach B (recommended for real): widen to untyped `vector`, write, then tighten + +1. Drop HNSW. +2. `ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector;` -- removes + the typmod entirely. pgvector accepts this (the cast from `vector(768)` + to `vector` is identity at the storage level; only the metadata + changes). Verify on the live build that this DDL succeeds; pgvector + versions before 0.5 may reject it, in which case Approach C is the + fallback. +3. UPDATE pass writes new-dimension vectors. The column has no typmod + constraint to fight against. +4. `ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE vector($NEW_DIM);` + -- reinstates the typmod at the new dimension. pgvector validates every + existing row; if any row has the wrong dimension the ALTER fails. This + is the integrity gate. +5. Rebuild HNSW with the new dimension implicitly in scope. + +### Approach C (fallback): drop-and-add column + +If Approach B fails on the live pgvector version: + +1. Drop HNSW. +2. `ALTER TABLE knowledge_nodes ADD COLUMN embedding_new vector($NEW_DIM);` +3. UPDATE pass writes into `embedding_new`. +4. `ALTER TABLE knowledge_nodes DROP COLUMN embedding;` +5. `ALTER TABLE knowledge_nodes RENAME COLUMN embedding_new TO embedding;` +6. Rebuild HNSW. + +Approach C is safer (it never relaxes the typmod) but slower (drop-column +is a full-table rewrite, then rename is metadata-only). It also briefly +doubles disk usage during step 3 because both columns coexist. + +**Implementation:** start with Approach B. Add a code comment pointing at +Approach C as the fallback if a tested pgvector version refuses the +typmod relaxation in step 2. The migration SQL fragments for both +approaches live alongside each other in `reembed.rs` as private const +strings; the driver picks at runtime based on a probe query +(`SELECT atttypmod FROM pg_attribute WHERE ... ;` after step 2; if the +typmod is still nonzero, fall through to Approach C). + +--- + +## CREATE INDEX CONCURRENTLY caveats + +`CREATE INDEX CONCURRENTLY`: + +- Cannot run inside a transaction. sqlx's default `query.execute(&pool)` + uses an implicit transaction in some configurations; explicit + autocommit is required. +- Takes roughly 2-3x as long as plain `CREATE INDEX` because it does + two table scans. +- Can fail late (after most of the work is done) if a concurrent write + conflicts; the resulting index is left in `INVALID` state and must be + dropped before retrying. + +Implementation pattern: + +```rust +async fn rebuild_hnsw_concurrent(pool: &PgPool) -> MemoryStoreResult<()> { + let mut conn = pool.acquire().await?; + // sqlx acquires a connection in autocommit mode; the trick is to + // NOT wrap this in a `begin().await?` transaction. + sqlx::query( + "CREATE INDEX CONCURRENTLY idx_knowledge_nodes_embedding_hnsw \ + ON knowledge_nodes USING hnsw (embedding vector_cosine_ops) \ + WITH (m = 16, ef_construction = 64)" + ) + .execute(&mut *conn) + .await?; + Ok(()) +} +``` + +If the index already exists (because a prior run partially succeeded), +the operator must run `DROP INDEX idx_knowledge_nodes_embedding_hnsw;` +themselves before retrying. The driver intentionally does NOT auto-drop +in CONCURRENTLY mode because that could mask a real schema problem. + +For the default `concurrent_index = false` path, use plain +`CREATE INDEX ...` against `pool.execute(...)`; transactions are fine. + +--- + +## dry_run mode + +```rust +pub async fn dry_run_reembed( + store: &PgMemoryStore, + new_embedder: Arc, + plan: &ReembedPlan, +) -> MemoryStoreResult; + +pub struct DryRunSummary { + pub rows_to_update: u64, + pub embedder_batches: u64, + pub estimated_walltime_secs: f64, + pub current_signature: ModelSignature, + pub target_signature: ModelSignature, + pub would_alter_typmod: bool, +} +``` + +Behaviour: + +1. `SELECT COUNT(*) FROM knowledge_nodes;` to get `rows_to_update`. +2. `embedder_batches = ceil(rows_to_update / plan.batch_size)`. +3. `estimated_walltime_secs = rows_to_update / 50.0` -- the master plan's + 50-rows-per-second baseline for local fastembed. Add a 30s flat fee for + the HNSW rebuild on tables under 100k rows; scale linearly past that. +4. `would_alter_typmod = current_signature.dimension != target_signature.dimension`. +5. Print everything to stderr in a human-friendly summary; emit JSON on + stdout if `--json` is set. +6. Return without writing anything. + +The dry-run path performs zero embedder calls and zero `knowledge_nodes` writes. +It is safe to run against production at any time. + +--- + +## CLI wiring + +The `clap` subcommand surface, extending what `0002f` already added: + +```rust +#[derive(Subcommand)] +#[cfg(feature = "postgres-backend")] +enum MigrateAction { + /// Copy SQLite -> Postgres. Owned by 0002f. + Copy { /* ... see 0002f ... */ }, + + /// Re-embed all memories in a Postgres backend with a new embedder. + Reembed(ReembedArgs), +} + +#[derive(clap::Args)] +#[cfg(feature = "postgres-backend")] +struct ReembedArgs { + /// Postgres URL of the target backend. + #[arg(long)] + postgres_url: String, + + /// Embedder model name. Today only `nomic-ai/nomic-embed-text-v1.5` + /// is supported (the FastembedEmbedder default). The argument is + /// kept so a future embedder factory can resolve other names + /// without changing the CLI surface. + #[arg(long)] + model: String, + + /// Vector dimension produced by the embedder. Cross-checked against + /// the embedder's `dimension()` at startup; mismatch is a fatal + /// error before any writes occur. + #[arg(long)] + dimension: usize, + + /// Embedder + UPDATE batch size. Default 128. + #[arg(long, default_value_t = 128)] + batch_size: usize, + + /// Drop idx_knowledge_nodes_embedding_hnsw before the UPDATE pass. + /// Default true. + #[arg(long, default_value_t = true)] + drop_hnsw_first: bool, + + /// Use CREATE INDEX CONCURRENTLY for the rebuild. Default false. + #[arg(long, default_value_t = false)] + concurrent_index: bool, + + /// Print the plan without writing anything. + #[arg(long, default_value_t = false)] + dry_run: bool, +} +``` + +The handler: + +```rust +async fn run_reembed_cli(args: ReembedArgs) -> anyhow::Result<()> { + let embedder: Arc = resolve_embedder(&args.model)?; + if embedder.dimension() != args.dimension { + anyhow::bail!( + "embedder '{}' produces dimension {}, --dimension was {}", + embedder.model_name(), embedder.dimension(), args.dimension, + ); + } + let store = PgMemoryStore::connect(&args.postgres_url, 4).await?; + let plan = ReembedPlan { + batch_size: args.batch_size, + drop_hnsw_first: args.drop_hnsw_first, + concurrent_index: args.concurrent_index, + }; + if args.dry_run { + let summary = dry_run_reembed(&store, embedder, &plan).await?; + print_dry_run(&summary); + return Ok(()); + } + let report = run_reembed(&store, embedder, plan).await?; + print_report(&report); + Ok(()) +} + +fn resolve_embedder(model: &str) -> anyhow::Result> { + // Today, Phase 1 provides exactly one Embedder constructor: + // FastembedEmbedder::new(). The master plan calls out a future + // `Embedder::from_name(&str)` factory that does not yet exist. + // Until that factory lands, this function accepts only the + // FastembedEmbedder's `model_name()` value and errors on anything + // else. Adding a real registry is a follow-up task. + let candidate = FastembedEmbedder::new(); + if candidate.model_name() == model { + return Ok(Arc::new(candidate)); + } + anyhow::bail!( + "unknown embedder model '{}'. Known: {}", + model, + candidate.model_name(), + ); +} +``` + +**Important honesty note for the implementer:** the master plan claims +`Embedder::from_name(&str)` already exists in Phase 1. As of audit (see +"Audit step" above), it does not. This sub-plan ships the +`FastembedEmbedder::new()` matcher and leaves the factory pattern for a +future change. Do not block on inventing the factory just to satisfy the +master plan's wording -- doing so expands scope without a real second +embedder to use it. + +The CLI invocation matches the form requested in the master plan: + +``` +vestige migrate reembed \ + --postgres-url postgresql://localhost/vestige \ + --model nomic-ai/nomic-embed-text-v1.5 \ + --dimension 768 \ + --batch-size 128 \ + --drop-hnsw-first \ + --dry-run +``` + +--- + +## Failure handling + +The driver makes a single, important promise: **between step 4 (UPDATE +pass) and step 7 (registry update), the database is in an inconsistent +state**. Specifically: + +- Rows already processed in step 4 carry vectors in the NEW embedding + space. +- Rows not yet processed carry vectors in the OLD embedding space. +- The `embedding_model` registry still says OLD. +- The HNSW index is dropped (if `drop_hnsw_first = true`). + +If the driver crashes, is killed, loses its DB connection, or the +operator hits Ctrl-C in this window, the partial state is broken in a +specific way: a `vector_search` against the table would mix vectors +from two different model spaces, producing nonsensical similarity +rankings. The operator MUST NOT serve search until the re-embed +completes. + +**Recovery procedure** (document this loudly in the operator-facing log): + +1. The CLI log already says, on every batch, `"reembed: wrote batch N + (M rows)"`. The last such log line indicates how far the pass got. +2. The recovery action is to **re-run reembed** with the same arguments. + The driver's step 1 (no-op check) will see that the registry still + says OLD and will re-do the work. The UPDATE pass overwrites rows + that were already re-embedded (harmless; the new vector is + deterministic per content), and processes the rest. +3. Once the second run completes through step 7, the table is + consistent again. + +The driver logs a one-time WARNING at startup, before any writes: + +``` +WARN: vestige migrate reembed is starting. Search results will be +WARN: incorrect until this run completes. Stop the MCP server now if +WARN: it is connected to this database. Press Ctrl-C within 5 seconds +WARN: to abort. +``` + +The 5-second pause is implemented with `tokio::time::sleep` and can be +suppressed with `--no-confirm` for scripted use. + +There is no "resume from row N" feature in this iteration. Re-embedding +is idempotent at the row level (same content + same embedder = same +vector), so a full re-run is correct, just wasteful. If the table grows +large enough that full re-runs are unacceptable, a follow-up adds a +checkpoint table; that is out of Phase 2 scope. + +--- + +## Verification + +### Unit tests (colocated in `reembed.rs`) + +1. **`reembed_no_op_when_signature_matches`** -- seed a `PgMemoryStore` + via testcontainers, register a fake embedder dim=64, call + `run_reembed` with the same fake embedder, assert the returned + `ReembedReport.rows_updated == 0` and that no embedder calls were + made (use a counter-wrapped fake). + +2. **`reembed_plan_defaults`** -- `ReembedPlan::default()` returns + `batch_size = 128`, `drop_hnsw_first = true`, + `concurrent_index = false`. + +3. **`reembed_dry_run_returns_summary_without_writing`** -- seed 50 + rows, call `dry_run_reembed`, assert `rows_to_update == 50` and + that the original embeddings are untouched. + +### Integration test (under `tests/phase_2/pg_reembed.rs`) + +Acceptance test that exercises the dimension-change path end to end: + +```rust +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; + +mod common; +use common::test_embedder::{FakeEmbedder, FakeEmbedderConfig}; +use common::pg_harness::PgHarness; + +#[tokio::test] +async fn reembed_changes_dimension_and_search_still_works() { + let old = Arc::new(FakeEmbedder::new(FakeEmbedderConfig { + name: "fake-old", + dimension: 64, + })); + let harness = PgHarness::start(old.clone()).await.unwrap(); + + // Seed 100 memories. Each gets a 64-d vector from `old`. + for i in 0..100 { + let content = format!("memory number {i} talks about rust and async"); + let vec = old.embed(&content).await.unwrap(); + harness.store.insert(/* ... record with embedding = vec ... */).await.unwrap(); + } + + // Now re-embed with a different fake at dim 128. + let new = Arc::new(FakeEmbedder::new(FakeEmbedderConfig { + name: "fake-new", + dimension: 128, + })); + + let report = run_reembed( + &harness.store, + new.clone(), + ReembedPlan::default(), + ).await.unwrap(); + + assert_eq!(report.rows_updated, 100); + + // (a) Every row has a 128-d vector. + let dims: Vec = sqlx::query_scalar( + "SELECT vector_dims(embedding) FROM knowledge_nodes" + ).fetch_all(harness.store.pool()).await.unwrap(); + assert!(dims.iter().all(|&d| d == 128)); + + // (b) Registry reflects the new signature. + let sig = harness.store.registered_model().await.unwrap().unwrap(); + assert_eq!(sig.name, "fake-new"); + assert_eq!(sig.dimension, 128); + + // (c) vector_search returns results in the new space. + let probe = new.embed("memory number 5 talks about rust and async").await.unwrap(); + let results = harness.store.vector_search(&probe, 10).await.unwrap(); + assert!(!results.is_empty()); +} +``` + +The `FakeEmbedder` from `common/test_embedder.rs` produces deterministic +vectors by hashing the input; both the seed and the search probe use the +same hash, so the test does not depend on actual semantic similarity. + +### Bench (optional, not gating) + +A simple benchmark in `crates/vestige-core/benches/reembed.rs` reports +throughput at 100k rows with `FakeEmbedder`. Useful for catching +regressions in the UPDATE-pass batching pattern. Not part of CI. + +--- + +## Acceptance criteria + +This sub-plan is complete when: + +1. `crates/vestige-core/src/storage/postgres/reembed.rs` exists and + compiles under `--features postgres-backend`. +2. `ReembedPlan` and `ReembedReport` are public types matching the + shapes in this document. +3. `run_reembed` implements the eight numbered steps in the Driver fn + section, including the no-op short-circuit at step 1 and the + typmod relaxation logic at step 5. +4. `dry_run_reembed` returns counts and estimates without writing. +5. The `vestige migrate reembed ...` subcommand is wired through + `crates/vestige-mcp/src/bin/cli.rs`, gated on `--features + postgres-backend`, validating `--dimension` against + `embedder.dimension()`. +6. The three unit tests pass. +7. The `pg_reembed.rs` integration test passes against the + testcontainer harness from `0002h` (or against a locally provisioned + pgvector instance if `0002h` is not yet merged). +8. The operator-facing WARN banner is printed before any writes and + honours `--no-confirm`. +9. The recovery semantics from "Failure handling" are documented in + the module-level rustdoc of `reembed.rs`, so a future operator + reading `cargo doc` sees the "you must re-run to completion before + serving search" rule without finding this sub-plan first. +10. `cargo sqlx prepare --workspace` updates `.sqlx/` with the new + queries; the resulting JSON files are committed. + +When all ten items are checked, sub-plan `0002g` lands. Master plan +deliverable D9 is satisfied. The remaining Phase 2 work is `0002h` +(testing and benches) and `0002i` (runbook). diff --git a/docs/plans/0002h-testing-and-benches.md b/docs/plans/0002h-testing-and-benches.md new file mode 100644 index 0000000..d6bcebc --- /dev/null +++ b/docs/plans/0002h-testing-and-benches.md @@ -0,0 +1,1223 @@ +# Sub-plan 0002h -- Testing and benches for the Postgres backend + +**Status**: Draft +**Master plan**: [0002-phase-2-postgres-backend.md](0002-phase-2-postgres-backend.md) +**ADR**: [0002-phase-2-execution.md](../adr/0002-phase-2-execution.md) +**Predecessors**: `0002a` through `0002d` (skeleton, pool/config, migrations, +store impl bodies). `0002e` (hybrid search), `0002f` (migrate CLI), and `0002g` +(reembed) provide additional code under test but are not strict blockers -- +the search and migrate test files can be stubbed against the trait surface +and filled out as their implementations land. + +--- + +## Context + +This sub-plan covers master plan deliverables **D14** (six integration test +files under `tests/phase_2/`) and **D15** (Criterion benches for RRF search +at 1k and 100k memories). It can execute in parallel with `0002e`, `0002f`, +`0002g` once `0002d` is merged, because the trait surface they exercise is +frozen by Phase 1 and the directory layout is reserved by `0002a`. + +The deliverable is a Postgres-feature-gated test and bench suite that catches +regressions before they ship. Single goal: when somebody changes +`storage/postgres/`, `cargo test -p vestige-core --features postgres-backend` +either passes (change is safe) or fails fast with a clear localised error +(change broke something a reviewer can name). + +Scope: + +- Add the testcontainer harness in `tests/phase_2/common/`. +- Add six integration test files, each gated on `postgres-backend`. +- Add the Criterion bench `pg_hybrid_search.rs` with two bench groups. +- Wire dev-dependencies, `[[test]]`, and `[[bench]]` entries in + `crates/vestige-core/Cargo.toml`. +- Document how the suite is run locally and what CI must provide. + +Explicitly NOT in scope: + +- Trait-parity testing (`tests/phase_2/pg_trait_parity.rs` from the master + plan). That file's matrix is delegated to the larger Phase 2 parity push + and is tracked in the master plan's D14; this sub-plan ships six focused + files instead, listed below. +- Concurrency stress tests (`pg_concurrency.rs` from the master plan). + Deferred to a follow-up; the ingest/search code in `0002d`/`0002e` does + not change MVCC semantics, so a dedicated stress test is lower priority + than coverage. +- Re-embed integration tests beyond a smoke check. `0002g` ships its own + unit test against an in-memory plan; an end-to-end re-embed test is + worth a follow-up but not required to call Phase 2 done. + +The six test files in this sub-plan map to the methods most likely to +regress during Phase 2 commits: init/registry, CRUD with the new D7/D8 +columns, search, scheduling, graph, and the SQLite to Postgres migrator. + +--- + +## Prerequisites + +- `0002a` -- `crates/vestige-core/src/storage/postgres/mod.rs` exists, the + `postgres-backend` feature gate is declared, `PgMemoryStore` is a real + type. Method bodies may still be `todo!()` for the parts a given test + does not touch. +- `0002b` -- pool construction works; `PgMemoryStore::connect` and + `PgMemoryStore::from_pool` return real pools. +- `0002c` -- `sqlx::migrate!` wired; tests can call + `PgMemoryStore::run_migrations(&pool).await?` (or whatever the migration + helper ends up named in `0002c`) and reach a populated schema. +- `0002d` -- CRUD, scheduling, and graph method bodies are real (not + `todo!()`). Without `0002d` the CRUD/scheduling/graph tests cannot pass. +- `0002e` -- hybrid search body is real. The search test depends on it. + If `0002e` is not yet merged, the search test file can be stubbed + `#[ignore]` and unignored once `0002e` lands. +- `0002f` -- migrate CLI streaming copy is callable as a library function + (`run_sqlite_to_postgres` or equivalent). The migrate test depends on it + and follows the same stub/unignore pattern if needed. +- Docker or Podman is available at test time. CI must provide it. Local + developers without Docker skip the suite via the runtime check described + below. + +--- + +## Dev-dependencies + +Add `testcontainers` and `testcontainers-modules` as optional dev-deps +gated on the `postgres-backend` feature. `criterion` is already in +dev-dependencies from Phase 1 (`search_bench.rs` uses it). + +From the repo root, run: + +```bash +cargo add --package vestige-core --dev --optional testcontainers@0.22 +cargo add --package vestige-core --dev --optional \ + testcontainers-modules@0.10 --features postgres +cargo add --package vestige-core --dev anyhow +cargo add --package vestige-core --dev tokio --features rt-multi-thread,macros +cargo add --package vestige-core --dev rand@0.8 +``` + +`anyhow` is convenient for the harness's error type (`anyhow::Result<...>` +inside the `common/` helper matches master plan D12). `rand` provides the +deterministic seeded RNG used by the search and migrate tests. `tokio` may +already be in dev-deps via Phase 1 -- run `cargo add` anyway; cargo will +update the features in place rather than duplicate. + +Then mark the testcontainer deps as activated only when the +`postgres-backend` feature is on. Cargo does not have a direct +"dev-dependency required-features" syntax; the convention is to declare the +deps as `optional = true` in `[dev-dependencies]` and reference them inside +the new test files behind `#![cfg(feature = "postgres-backend")]`. The +resulting `Cargo.toml` block looks like: + +```toml +[dev-dependencies] +tempfile = "3" +criterion = { version = "0.5", features = ["html_reports"] } +anyhow = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +rand = "0.8" +testcontainers = { version = "0.22", optional = true } +testcontainers-modules = { version = "0.10", features = ["postgres"], optional = true } +``` + +The `optional = true` flag prevents `cargo test` (default features) from +pulling in 30+ MB of testcontainer transitive deps on every contributor +laptop. Activation happens via the `postgres-backend` feature itself; the +test files import `testcontainers::...` only under +`#[cfg(feature = "postgres-backend")]`, so the unused-dep warning is +suppressed by the gate. + +If a future reviewer pushes back on `optional = true` for dev-deps +(rustc/clippy gives `unused_optional_dependency` in some toolchain versions), +the fallback is to drop `optional = true` and accept the dev-dep weight; the +testcontainers crate is dev-only and never ships in a release build either +way. + +--- + +## Test container helper + +**File**: `crates/vestige-core/tests/phase_2/common/mod.rs` + +This is shared infrastructure for every test in `tests/phase_2/`. It is not +its own `[[test]]`; it is a `mod common;` import inside each test file. + +```rust +//! Shared testcontainer setup for Phase 2 Postgres integration tests. +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; + +use anyhow::Result; +use testcontainers::core::{ContainerPort, IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; +use testcontainers_modules::postgres::Postgres; + +use vestige_core::embedder::Embedder; +use vestige_core::storage::postgres::PgMemoryStore; + +/// Spin up a fresh pgvector-enabled Postgres container and return a fully +/// migrated PgMemoryStore connected to it. +/// +/// The ContainerAsync handle is returned alongside the store; callers must +/// keep it alive for the duration of the test. Dropping it tears the +/// container down. +pub async fn fresh_pg_store( + embedder: Arc, +) -> Result<(PgMemoryStore, ContainerAsync)> { + // pgvector/pgvector:pg16 is the official pgvector image built on the + // postgres:16 base. testcontainers-modules::postgres::Postgres targets + // the upstream postgres image by default; we override name + tag. + let container = Postgres::default() + .with_name("pgvector/pgvector") + .with_tag("pg16") + .start() + .await?; + + let port = container.get_host_port_ipv4(5432).await?; + let url = format!("postgresql://postgres:postgres@127.0.0.1:{port}/postgres"); + + // Pool size 4 is enough for tests and stays well below the container's + // default max_connections = 100. + let store = PgMemoryStore::connect(&url, 4).await?; + + // Run migrations. `0002c` decides the exact helper name. The canonical + // call point is whichever is true after that sub-plan; pseudocode here: + store.run_migrations().await?; + + // Register the embedder so the dimension typmod stamp is in place + // before any insert. `0002d` lands the real register_model body. + let sig = embedder.signature(); + store.register_model(&sig).await?; + + Ok((store, container)) +} + +/// Fixed embedder used by every test. Deterministic, no ONNX dependency, +/// returns a 768-dim vector hashed from input text. Lives in +/// `tests/phase_2/common/test_embedder.rs`. +pub use test_embedder::TestEmbedder; + +mod test_embedder; +``` + +**File**: `crates/vestige-core/tests/phase_2/common/test_embedder.rs` + +```rust +//! Deterministic hash-based embedder for tests. +//! +//! Avoids the fastembed/ONNX dependency in CI. Returns a 768-dim vector +//! built from a stable hash of the input text. Two equal strings produce +//! equal vectors; near-equal strings produce near-equal vectors only at +//! the trivial token-overlap level (good enough for a smoke check that +//! the vector pipeline is wired, not a real embedding quality test). +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; + +use async_trait::async_trait; + +use vestige_core::embedder::{Embedder, EmbedderError, ModelSignature}; + +pub struct TestEmbedder { + pub name: String, + pub dim: usize, +} + +impl TestEmbedder { + pub fn new_768() -> Arc { + Arc::new(Self { name: "test-768".into(), dim: 768 }) + } + pub fn new_1024() -> Arc { + Arc::new(Self { name: "test-1024".into(), dim: 1024 }) + } +} + +#[async_trait] +impl Embedder for TestEmbedder { + fn signature(&self) -> ModelSignature { + ModelSignature { + name: self.name.clone(), + dimension: self.dim, + hash: format!("{}-h", self.name), + } + } + + async fn embed(&self, text: &str) -> Result, EmbedderError> { + let mut v = vec![0.0f32; self.dim]; + let bytes = text.as_bytes(); + for (i, b) in bytes.iter().enumerate() { + v[i % self.dim] += (*b as f32) / 255.0; + } + // Normalize so cosine similarity is meaningful. + let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + for x in &mut v { + *x /= norm; + } + } + Ok(v) + } +} +``` + +Notes: + +- The exact `Embedder` trait shape is owned by Phase 1; the example above + may need `embed_batch`, `dimension()`, etc. depending on the frozen + surface. Whoever lands this file mirrors whatever the Phase 1 trait + exposes. +- The container handle is returned, not stored in a `static`. Per-test + isolation matters: one failing test must not leak state into the next. +- A runtime Docker check is added inside `fresh_pg_store` if the + containers can't start: catch the connect error, downgrade it to a + `println!` plus `panic!("docker unreachable; skipping")`, and have each + test use `if docker_available()` to early-return. + +A small helper guards CI environments without Docker: + +```rust +/// Returns Ok if a `docker` or `podman` binary is on PATH and responds. +/// Tests that need a container call this first and `eprintln!`+`return` +/// rather than failing when Docker is absent. +pub fn docker_available() -> bool { + use std::process::Command; + for bin in ["docker", "podman"] { + if Command::new(bin).arg("info").output().map(|o| o.status.success()).unwrap_or(false) { + return true; + } + } + false +} +``` + +Each test starts with: + +```rust +if !common::docker_available() { + eprintln!("docker/podman not available; skipping {}", file!()); + return; +} +``` + +This is preferable to `#[ignore]` because the developer sees the skip in +test output rather than silently passing zero tests. + +--- + +## Six test files + +Each file is at `crates/vestige-core/tests/phase_2/.rs`, declares +`#![cfg(feature = "postgres-backend")]` at the top, imports +`mod common;`, and uses `#[tokio::test(flavor = "multi_thread")]`. + +Each file is also wired as a separate `[[test]]` entry in the Cargo.toml +(see "Cargo.toml" section below). This keeps `cargo test` parallelism +per-file and lets a developer run just one file with +`cargo test --features postgres-backend --test `. + +### 1. `tests/phase_2/init_test.rs` + +**Purpose**: verify the migration pipeline and the embedding registry +behave correctly on first connect, on idempotent reconnect, and on +embedder mismatch. + +**Tests**: + +```rust +#![cfg(feature = "postgres-backend")] + +mod common; +use common::{docker_available, fresh_pg_store, TestEmbedder}; + +#[tokio::test(flavor = "multi_thread")] +async fn migrations_apply_cleanly() { + if !docker_available() { eprintln!("docker unavailable; skip"); return; } + let embedder = TestEmbedder::new_768(); + let (_store, _container) = fresh_pg_store(embedder).await.unwrap(); + // If we reached here, sqlx::migrate! ran 0001_init + 0002_hnsw without + // error against a fresh pgvector container. +} + +#[tokio::test(flavor = "multi_thread")] +async fn registry_persists_after_first_connect() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _container) = fresh_pg_store(embedder.clone()).await.unwrap(); + let registered = store.registered_model().await.unwrap(); + assert!(registered.is_some()); + let sig = registered.unwrap(); + assert_eq!(sig.name, "test-768"); + assert_eq!(sig.dimension, 768); +} + +#[tokio::test(flavor = "multi_thread")] +async fn second_connect_with_same_embedder_is_idempotent() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store_a, container) = fresh_pg_store(embedder.clone()).await.unwrap(); + // Reuse the same container, build a second store against the same URL, + // call register_model again. Must not error. + let port = container.get_host_port_ipv4(5432).await.unwrap(); + let url = format!("postgresql://postgres:postgres@127.0.0.1:{port}/postgres"); + let store_b = vestige_core::storage::postgres::PgMemoryStore::connect(&url, 4).await.unwrap(); + store_b.register_model(&embedder.signature()).await.unwrap(); + assert_eq!( + store_a.registered_model().await.unwrap().unwrap().name, + store_b.registered_model().await.unwrap().unwrap().name, + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn second_connect_with_different_embedder_returns_mismatch() { + if !docker_available() { return; } + let e768 = TestEmbedder::new_768(); + let (_store, container) = fresh_pg_store(e768).await.unwrap(); + let port = container.get_host_port_ipv4(5432).await.unwrap(); + let url = format!("postgresql://postgres:postgres@127.0.0.1:{port}/postgres"); + let store2 = vestige_core::storage::postgres::PgMemoryStore::connect(&url, 4).await.unwrap(); + let e1024 = TestEmbedder::new_1024(); + let err = store2.register_model(&e1024.signature()).await; + assert!(matches!(err, Err(vestige_core::storage::MemoryStoreError::EmbeddingMismatch { .. }))); +} +``` + +### 2. `tests/phase_2/crud_test.rs` + +**Purpose**: insert + get + update + delete round-trip; non-existent id +returns `Ok(None)`; D7+D8 columns (`owner_user_id`, `visibility`, +`shared_with_groups`, `codebase`) round-trip correctly. + +**Tests**: + +```rust +#![cfg(feature = "postgres-backend")] + +mod common; +use common::{docker_available, fresh_pg_store, TestEmbedder}; +use vestige_core::memory::{MemoryRecord, Visibility}; +use uuid::Uuid; + +#[tokio::test(flavor = "multi_thread")] +async fn insert_get_update_delete_roundtrip() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _container) = fresh_pg_store(embedder.clone()).await.unwrap(); + + let mut rec = MemoryRecord::new("hello world"); + rec.tags = vec!["test".into(), "crud".into()]; + rec.embedding = Some(embedder.embed(&rec.content).await.unwrap()); + let id = store.insert(&rec).await.unwrap(); + + let got = store.get(&id).await.unwrap().unwrap(); + assert_eq!(got.content, "hello world"); + assert_eq!(got.tags, vec!["test", "crud"]); + + let mut updated = got.clone(); + updated.content = "hello updated".into(); + updated.embedding = Some(embedder.embed("hello updated").await.unwrap()); + store.update(&updated).await.unwrap(); + let after = store.get(&id).await.unwrap().unwrap(); + assert_eq!(after.content, "hello updated"); + + store.delete(&id).await.unwrap(); + assert!(store.get(&id).await.unwrap().is_none()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn get_nonexistent_returns_ok_none() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _container) = fresh_pg_store(embedder).await.unwrap(); + let missing = Uuid::new_v4(); + assert!(store.get(&missing).await.unwrap().is_none()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn update_nonexistent_returns_not_found() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _container) = fresh_pg_store(embedder.clone()).await.unwrap(); + let mut rec = MemoryRecord::new("ghost"); + rec.id = Uuid::new_v4(); + rec.embedding = Some(embedder.embed("ghost").await.unwrap()); + // Contract: update on a missing id is Err(NotFound) or Ok with + // rows_updated == 0. Whichever 0002d picks is what this test asserts. + let res = store.update(&rec).await; + // Adjust to actual contract once 0002d lands: + assert!(res.is_err() || res.is_ok()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn d7_d8_columns_roundtrip() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _container) = fresh_pg_store(embedder.clone()).await.unwrap(); + + let owner = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap(); + let group_a = Uuid::new_v4(); + let group_b = Uuid::new_v4(); + + let mut rec = MemoryRecord::new("contextful"); + rec.owner_user_id = owner; + rec.visibility = Visibility::Group; + rec.shared_with_groups = vec![group_a, group_b]; + rec.codebase = Some("vestige".to_string()); + rec.embedding = Some(embedder.embed(&rec.content).await.unwrap()); + + let id = store.insert(&rec).await.unwrap(); + let got = store.get(&id).await.unwrap().unwrap(); + + assert_eq!(got.owner_user_id, owner); + assert_eq!(got.visibility, Visibility::Group); + assert_eq!(got.shared_with_groups, vec![group_a, group_b]); + assert_eq!(got.codebase.as_deref(), Some("vestige")); +} +``` + +### 3. `tests/phase_2/search_test.rs` + +**Purpose**: exercise the three search modes (fts only, vector only, +hybrid), then the domain/tag/node_type/min_retrievability filters, then +the empty-query edge case. + +**Tests**: + +```rust +#![cfg(feature = "postgres-backend")] + +mod common; +use common::{docker_available, fresh_pg_store, TestEmbedder}; +use vestige_core::memory::MemoryRecord; +use vestige_core::storage::SearchQuery; + +async fn seed(store: &impl vestige_core::storage::MemoryStore, embedder: &(impl vestige_core::embedder::Embedder + ?Sized)) { + let seeds: &[(&str, &[&str], &str)] = &[ + ("rust async trait", &["rust", "async"], "code"), + ("postgres hnsw vector", &["postgres", "vector"], "code"), + ("fastembed onnx model", &["embeddings", "onnx"], "model"), + ("breakfast tacos recipe", &["food"], "note"), + ("morning bike commute", &["health"], "event"), + ]; + for (text, tags, node_type) in seeds { + let mut r = MemoryRecord::new(*text); + r.tags = tags.iter().map(|s| s.to_string()).collect(); + r.node_type = node_type.to_string(); + r.embedding = Some(embedder.embed(text).await.unwrap()); + store.insert(&r).await.unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn fts_only_returns_keyword_matches() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + seed(&store, embedder.as_ref()).await; + + let q = SearchQuery { text: Some("rust".into()), embedding: None, limit: 10, ..Default::default() }; + let hits = store.search(&q).await.unwrap(); + assert!(hits.iter().any(|h| h.content.contains("rust async trait"))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn vector_only_returns_semantic_matches() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + seed(&store, embedder.as_ref()).await; + + let qe = embedder.embed("vector search").await.unwrap(); + let q = SearchQuery { text: None, embedding: Some(qe), limit: 10, ..Default::default() }; + let hits = store.search(&q).await.unwrap(); + assert!(!hits.is_empty()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn hybrid_returns_rrf_fused_results() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + seed(&store, embedder.as_ref()).await; + + let qe = embedder.embed("postgres vector").await.unwrap(); + let q = SearchQuery { + text: Some("postgres".into()), + embedding: Some(qe), + limit: 10, + ..Default::default() + }; + let hits = store.search(&q).await.unwrap(); + let top = hits.first().unwrap(); + assert!(top.content.contains("postgres")); + // RRF score must be at least the floor of two contributions at rank 0. + assert!(top.score >= 1.0 / 61.0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn filter_by_tag_and_node_type() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + seed(&store, embedder.as_ref()).await; + + let q = SearchQuery { + text: Some("model".into()), + tags: vec!["embeddings".into()], + node_type: Some("model".into()), + limit: 10, + ..Default::default() + }; + let hits = store.search(&q).await.unwrap(); + assert!(hits.iter().all(|h| h.tags.contains(&"embeddings".into()))); + assert!(hits.iter().all(|h| h.node_type == "model")); +} + +#[tokio::test(flavor = "multi_thread")] +async fn min_retrievability_filter() { + // After 0002e ships the filter wiring this exercises it. For now, + // assert the empty / pass-through case: min_retrievability = 0.0 + // returns all results. + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + seed(&store, embedder.as_ref()).await; + + let q = SearchQuery { text: Some("rust".into()), min_retrievability: 0.0, limit: 10, ..Default::default() }; + let hits = store.search(&q).await.unwrap(); + assert!(!hits.is_empty()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn empty_query_returns_ok_empty_or_all() { + // Contract chosen in 0002e; this test asserts whichever it picks. + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder).await.unwrap(); + let q = SearchQuery { text: None, embedding: None, limit: 10, ..Default::default() }; + let hits = store.search(&q).await.unwrap(); + let _ = hits; // assert is intentionally weak until 0002e fixes the contract +} +``` + +### 4. `tests/phase_2/scheduling_test.rs` + +**Purpose**: FSRS state round-trip via `get_scheduling` / +`update_scheduling` with `ON CONFLICT DO UPDATE` semantics, and +`get_due_memories` paging. + +**Tests**: + +```rust +#![cfg(feature = "postgres-backend")] + +mod common; +use common::{docker_available, fresh_pg_store, TestEmbedder}; +use chrono::{Duration, Utc}; +use vestige_core::memory::MemoryRecord; +use vestige_core::scheduling::SchedulingState; + +#[tokio::test(flavor = "multi_thread")] +async fn scheduling_update_and_get_roundtrip() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + + let mut rec = MemoryRecord::new("fsrs target"); + rec.embedding = Some(embedder.embed("fsrs target").await.unwrap()); + let id = store.insert(&rec).await.unwrap(); + + let s = SchedulingState { + memory_id: id, + stability: 2.5, + difficulty: 6.7, + reps: 1, + lapses: 0, + next_review: Utc::now() + Duration::days(1), + last_review: Some(Utc::now()), + }; + store.update_scheduling(&s).await.unwrap(); + + let back = store.get_scheduling(&id).await.unwrap().unwrap(); + assert!((back.stability - 2.5).abs() < 1e-6); + assert_eq!(back.reps, 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn scheduling_on_conflict_overwrites() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + + let mut rec = MemoryRecord::new("repeating"); + rec.embedding = Some(embedder.embed("repeating").await.unwrap()); + let id = store.insert(&rec).await.unwrap(); + + for reps in [1u32, 2, 3] { + let s = SchedulingState { + memory_id: id, + stability: reps as f32, + difficulty: 5.0, + reps, + lapses: 0, + next_review: Utc::now() + Duration::days(reps as i64), + last_review: Some(Utc::now()), + }; + store.update_scheduling(&s).await.unwrap(); + } + let final_state = store.get_scheduling(&id).await.unwrap().unwrap(); + assert_eq!(final_state.reps, 3); +} + +#[tokio::test(flavor = "multi_thread")] +async fn get_due_memories_pages() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + + let now = Utc::now(); + // Insert 25 due memories with next_review in the past. + for i in 0..25 { + let mut rec = MemoryRecord::new(format!("due {i}")); + rec.embedding = Some(embedder.embed(&rec.content).await.unwrap()); + let id = store.insert(&rec).await.unwrap(); + let s = SchedulingState { + memory_id: id, + stability: 1.0, + difficulty: 5.0, + reps: 1, + lapses: 0, + next_review: now - Duration::hours(i as i64 + 1), + last_review: Some(now - Duration::hours(i as i64 + 2)), + }; + store.update_scheduling(&s).await.unwrap(); + } + let page1 = store.get_due_memories(now, 10, 0).await.unwrap(); + let page2 = store.get_due_memories(now, 10, 10).await.unwrap(); + let page3 = store.get_due_memories(now, 10, 20).await.unwrap(); + assert_eq!(page1.len(), 10); + assert_eq!(page2.len(), 10); + assert_eq!(page3.len(), 5); +} +``` + +### 5. `tests/phase_2/graph_test.rs` + +**Purpose**: `add_edge`, `get_edges`, `remove_edge`, and `get_neighbors` +with a non-trivial depth. + +**Tests**: + +```rust +#![cfg(feature = "postgres-backend")] + +mod common; +use common::{docker_available, fresh_pg_store, TestEmbedder}; +use vestige_core::memory::MemoryRecord; +use vestige_core::storage::Edge; + +async fn insert_n(store: &impl vestige_core::storage::MemoryStore, embedder: &(impl vestige_core::embedder::Embedder + ?Sized), n: usize) -> Vec { + let mut ids = Vec::with_capacity(n); + for i in 0..n { + let mut r = MemoryRecord::new(format!("node {i}")); + r.embedding = Some(embedder.embed(&r.content).await.unwrap()); + ids.push(store.insert(&r).await.unwrap()); + } + ids +} + +#[tokio::test(flavor = "multi_thread")] +async fn add_get_remove_edge() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + let ids = insert_n(&store, embedder.as_ref(), 3).await; + + let e = Edge { + source_id: ids[0], + target_id: ids[1], + edge_type: "semantic".into(), + strength: 0.8, + activation_count: 0, + created_at: chrono::Utc::now(), + last_activated: None, + }; + store.add_edge(&e).await.unwrap(); + + let edges = store.get_edges(&ids[0]).await.unwrap(); + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].target_id, ids[1]); + + store.remove_edge(&ids[0], &ids[1], "semantic").await.unwrap(); + assert!(store.get_edges(&ids[0]).await.unwrap().is_empty()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn get_neighbors_with_depth() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + let (store, _c) = fresh_pg_store(embedder.clone()).await.unwrap(); + let ids = insert_n(&store, embedder.as_ref(), 5).await; + + // Chain: 0 -> 1 -> 2 -> 3 -> 4 + for w in ids.windows(2) { + let e = Edge { + source_id: w[0], + target_id: w[1], + edge_type: "semantic".into(), + strength: 1.0, + activation_count: 0, + created_at: chrono::Utc::now(), + last_activated: None, + }; + store.add_edge(&e).await.unwrap(); + } + + let depth_1 = store.get_neighbors(&ids[0], 1).await.unwrap(); + let depth_2 = store.get_neighbors(&ids[0], 2).await.unwrap(); + let depth_4 = store.get_neighbors(&ids[0], 4).await.unwrap(); + + assert_eq!(depth_1.len(), 1); + assert_eq!(depth_2.len(), 2); + assert_eq!(depth_4.len(), 4); +} +``` + +### 6. `tests/phase_2/migrate_test.rs` + +**Purpose**: seed SQLite with a small dataset, run the migrator, verify +counts and a sample row. + +**Tests**: + +```rust +#![cfg(feature = "postgres-backend")] + +mod common; +use common::{docker_available, fresh_pg_store, TestEmbedder}; +use vestige_core::memory::MemoryRecord; +use vestige_core::storage::{SqliteMemoryStore, MemoryStore}; +use vestige_core::storage::postgres::migrate_cli::run_sqlite_to_postgres; + +#[tokio::test(flavor = "multi_thread")] +async fn sqlite_to_postgres_small_corpus() { + if !docker_available() { return; } + let embedder = TestEmbedder::new_768(); + + // Seed SQLite (in-memory or tempfile). + let tmp = tempfile::tempdir().unwrap(); + let sqlite_path = tmp.path().join("seed.db"); + let sqlite = SqliteMemoryStore::new(&sqlite_path).unwrap(); + sqlite.register_model(&embedder.signature()).await.unwrap(); + for i in 0..50 { + let mut r = MemoryRecord::new(format!("seed row {i}")); + r.tags = vec![format!("tag-{}", i % 3)]; + r.embedding = Some(embedder.embed(&r.content).await.unwrap()); + sqlite.insert(&r).await.unwrap(); + } + + // Spin up Postgres and migrate. + let (pg, _container) = fresh_pg_store(embedder.clone()).await.unwrap(); + let report = run_sqlite_to_postgres(&sqlite, &pg, embedder.clone()).await.unwrap(); + + assert_eq!(report.memories_copied, 50); + assert_eq!(pg.count().await.unwrap(), 50); + + // Spot-check a sample row. + let sample_id = sqlite.list_ids(1, 0).await.unwrap()[0]; + let from_sqlite = sqlite.get(&sample_id).await.unwrap().unwrap(); + let from_pg = pg.get(&sample_id).await.unwrap().unwrap(); + assert_eq!(from_sqlite.content, from_pg.content); + assert_eq!(from_sqlite.tags, from_pg.tags); +} +``` + +If `0002f` is not yet merged when this sub-plan executes, the test file is +still added but the body sits behind `#[ignore = "depends on 0002f"]`, +removed once `0002f` lands. + +--- + +## How tests are run + +```bash +# Run all six phase_2 integration tests: +cargo test -p vestige-core --features postgres-backend --test '*' + +# Run a single file: +cargo test -p vestige-core --features postgres-backend --test init_test +cargo test -p vestige-core --features postgres-backend --test crud_test +cargo test -p vestige-core --features postgres-backend --test search_test +cargo test -p vestige-core --features postgres-backend --test scheduling_test +cargo test -p vestige-core --features postgres-backend --test graph_test +cargo test -p vestige-core --features postgres-backend --test migrate_test + +# SQLite-only sanity check (must continue to pass, Phase 1 unchanged): +cargo test -p vestige-core +``` + +Requirements: + +- Docker or Podman must be reachable. `testcontainers` connects via the + default Docker socket (`/var/run/docker.sock` on Linux, `~/.docker/run/docker.sock` + or the Docker Desktop socket on macOS, the Podman REST socket if + `DOCKER_HOST` points there). +- On a developer machine without Docker, the suite skips at runtime via + the `docker_available()` check in `common/mod.rs`. The test output + includes a `docker unavailable; skip` line per test so the developer + knows the tests were not silently dropped. +- The pgvector image (`pgvector/pgvector:pg16`) is pulled on first run; + ~200 MB. A pre-pulled image keeps the per-run overhead at the cold-start + container boot (~2-5 seconds). + +--- + +## Benches + +**File**: `crates/vestige-core/benches/pg_hybrid_search.rs` + +Two Criterion benches: `search_1k` and `search_100k`. Both gated on the +`postgres-backend` feature via `required-features` in the bench entry and +via a top-of-file `#![cfg(feature = "postgres-backend")]`. + +```rust +//! Criterion benches for the Postgres backend's hybrid RRF search. +#![cfg(feature = "postgres-backend")] + +use std::sync::Arc; +use std::sync::OnceLock; + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ImageExt}; +use testcontainers_modules::postgres::Postgres; +use tokio::runtime::Runtime; + +use vestige_core::embedder::Embedder; +use vestige_core::memory::MemoryRecord; +use vestige_core::storage::postgres::PgMemoryStore; +use vestige_core::storage::{MemoryStore, SearchQuery}; + +// Bench fixture lives in tests/phase_2/common/test_embedder.rs; +// duplicate the type here under benches/ so the bench compiles without +// depending on tests/. +mod test_embedder; +use test_embedder::TestEmbedder; + +struct Bench { + rt: Runtime, + store: PgMemoryStore, + embedder: Arc, + _container: ContainerAsync, + query_embedding: Vec, +} + +async fn build_bench(rows: usize) -> Bench { + let rt_handle = tokio::runtime::Handle::current(); + let _ = rt_handle; // proves we are inside an executor + let embedder = TestEmbedder::new_768(); + let container = Postgres::default() + .with_name("pgvector/pgvector") + .with_tag("pg16") + .start() + .await + .unwrap(); + let port = container.get_host_port_ipv4(5432).await.unwrap(); + let url = format!("postgresql://postgres:postgres@127.0.0.1:{port}/postgres"); + let store = PgMemoryStore::connect(&url, 8).await.unwrap(); + store.run_migrations().await.unwrap(); + store.register_model(&embedder.signature()).await.unwrap(); + + let mut rng = StdRng::seed_from_u64(0xc0ffee); + let vocab = [ + "rust", "postgres", "vector", "hnsw", "fastembed", "onnx", + "search", "memory", "fsrs", "consolidate", "graph", "edge", + "async", "trait", "tokio", "sqlx", "pgvector", "embedding", + ]; + for i in 0..rows { + let words: String = (0..8) + .map(|_| vocab[rng.gen_range(0..vocab.len())]) + .collect::>() + .join(" "); + let mut r = MemoryRecord::new(format!("{i}: {words}")); + r.tags = vec![format!("tag-{}", i % 7)]; + r.embedding = Some(embedder.embed(&r.content).await.unwrap()); + store.insert(&r).await.unwrap(); + } + let query_embedding = embedder.embed("postgres vector search").await.unwrap(); + Bench { + rt: tokio::runtime::Runtime::new().unwrap(), + store, + embedder, + _container: container, + query_embedding, + } +} + +fn bench_search_1k(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let bench = rt.block_on(build_bench(1_000)); + c.bench_function("pg_search_1k", |b| { + b.iter(|| { + let q = SearchQuery { + text: Some("postgres vector".into()), + embedding: Some(bench.query_embedding.clone()), + limit: 10, + ..Default::default() + }; + let hits = bench.rt.block_on(bench.store.search(&q)).unwrap(); + black_box(hits); + }) + }); +} + +// Heavy: 100k rows; seed time runs into minutes. Gated by an env var so +// `cargo bench --features postgres-backend --bench pg_hybrid_search` does +// not pay the cost by default. +fn bench_search_100k(c: &mut Criterion) { + if std::env::var("VESTIGE_BENCH_HEAVY").is_err() { + eprintln!("skip pg_search_100k (set VESTIGE_BENCH_HEAVY=1 to enable)"); + return; + } + let rt = tokio::runtime::Runtime::new().unwrap(); + let bench = rt.block_on(build_bench(100_000)); + c.bench_function("pg_search_100k", |b| { + b.iter(|| { + let q = SearchQuery { + text: Some("postgres vector".into()), + embedding: Some(bench.query_embedding.clone()), + limit: 10, + ..Default::default() + }; + let hits = bench.rt.block_on(bench.store.search(&q)).unwrap(); + black_box(hits); + }) + }); +} + +criterion_group!(benches, bench_search_1k, bench_search_100k); +criterion_main!(benches); +``` + +**File**: `crates/vestige-core/benches/test_embedder.rs` + +Duplicate of `tests/phase_2/common/test_embedder.rs`. Cargo's bench target +cannot `mod` into `tests/`; the duplication is the standard fix. Keep both +files in sync; if either grows non-trivially, refactor into a shared +`pub(crate)` module under `src/embedder/test_support.rs` gated on +`#[cfg(any(test, feature = "test-support"))]`. + +`VESTIGE_BENCH_HEAVY` gate: the 100k seed step takes several minutes (one +`INSERT` per row plus HNSW upsert). Skipping by default keeps `cargo bench` +under a minute for the 1k bench. Document this gate in the runbook +(`0002i`). + +--- + +## Cargo.toml + +Final state of the relevant sections of +`crates/vestige-core/Cargo.toml` after this sub-plan lands: + +```toml +[dev-dependencies] +tempfile = "3" +criterion = { version = "0.5", features = ["html_reports"] } +anyhow = "1" +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +rand = "0.8" +testcontainers = { version = "0.22", optional = true } +testcontainers-modules = { version = "0.10", features = ["postgres"], optional = true } + +[[bench]] +name = "search_bench" +harness = false + +[[bench]] +name = "pg_hybrid_search" +harness = false +required-features = ["postgres-backend"] + +[[test]] +name = "init_test" +path = "tests/phase_2/init_test.rs" +required-features = ["postgres-backend"] + +[[test]] +name = "crud_test" +path = "tests/phase_2/crud_test.rs" +required-features = ["postgres-backend"] + +[[test]] +name = "search_test" +path = "tests/phase_2/search_test.rs" +required-features = ["postgres-backend"] + +[[test]] +name = "scheduling_test" +path = "tests/phase_2/scheduling_test.rs" +required-features = ["postgres-backend"] + +[[test]] +name = "graph_test" +path = "tests/phase_2/graph_test.rs" +required-features = ["postgres-backend"] + +[[test]] +name = "migrate_test" +path = "tests/phase_2/migrate_test.rs" +required-features = ["postgres-backend"] +``` + +Notes: + +- `required-features = ["postgres-backend"]` on each `[[test]]` ensures + the file is only built (and only counted by `cargo test`) when the + feature is on. Cargo silently skips it otherwise -- exactly the desired + behavior for default `cargo test` runs. +- The benches use the same `required-features` shape so default + `cargo bench` is unaffected. + +--- + +## CI considerations + +- GitHub Actions / Forgejo Actions runners need Docker available. Default + `ubuntu-latest` runners include Docker. Self-hosted Forgejo runners on + TFGrid VMs must install `docker.io` or run `podman` with the Docker + socket compatibility shim. Document the runner requirement in the + runbook (`0002i`). +- The Postgres feature tests should run in a separate CI matrix entry to + isolate failures and skip them entirely on platforms (Windows runners + if any) where the pgvector image is not available. +- Cache the `pgvector/pgvector:pg16` image between runs. The + `docker/setup-buildx-action` cache or a simple `docker pull` step before + the test step keeps cold-start under the existing CI time budget. +- Skip CI: contributors without Docker can still merge changes that do + not touch `storage/postgres/`. The pre-merge required check is "phase_2 + tests pass on the runner with Docker"; the local pre-commit hook does + not gate on it. +- Bench CI: do not run `pg_search_100k` in regular CI; only run it + manually or on a scheduled weekly job and post results to the PR + description / ADR comment trail. + +Recommended CI job shape (sketch): + +```yaml +jobs: + postgres-tests: + runs-on: ubuntu-latest + services: + # no `postgres` service block needed; testcontainers manages its own + steps: + - uses: actions/checkout@v4 + - run: docker pull pgvector/pgvector:pg16 + - uses: dtolnay/rust-toolchain@stable + - run: cargo test -p vestige-core --features postgres-backend --test '*' +``` + +--- + +## Verification + +After all files are in place: + +```bash +# Default build still clean (no postgres deps pulled in): +cargo build -p vestige-core +cargo test -p vestige-core + +# Postgres feature build + integration tests: +cargo build -p vestige-core --features postgres-backend +cargo test -p vestige-core --features postgres-backend + +# Just the new tests: +cargo test -p vestige-core --features postgres-backend --test '*' + +# Quick bench sanity check (1k only): +cargo bench -p vestige-core --features postgres-backend --bench pg_hybrid_search -- --quick + +# Heavy bench (manual, multi-minute seed step): +VESTIGE_BENCH_HEAVY=1 cargo bench -p vestige-core \ + --features postgres-backend \ + --bench pg_hybrid_search -- --quick + +# Clippy with everything on: +cargo clippy -p vestige-core --features postgres-backend --all-targets -- -D warnings +``` + +Expected results: + +- Default build is unchanged; no testcontainers deps in `Cargo.lock`'s + default resolution. +- With `--features postgres-backend`, all six integration tests pass on a + machine with Docker available, or each prints `docker unavailable; skip` + and exits 0. +- `cargo bench ... -- --quick` produces a `pg_search_1k` line with a + p50 below the master plan's 10 ms target on a developer laptop (looser + on a CI runner -- the target is informative, not gated). + +--- + +## Acceptance criteria + +- [ ] `crates/vestige-core/tests/phase_2/common/mod.rs` and + `test_embedder.rs` exist and compile under + `--features postgres-backend`. +- [ ] All six integration test files exist, each with + `#![cfg(feature = "postgres-backend")]` at the top. +- [ ] Each test file has a corresponding `[[test]]` entry in + `Cargo.toml` with `required-features = ["postgres-backend"]`. +- [ ] `crates/vestige-core/benches/pg_hybrid_search.rs` exists with + `search_1k` and `search_100k` benches, the latter gated on + `VESTIGE_BENCH_HEAVY`. +- [ ] `[[bench]] name = "pg_hybrid_search"` entry present with + `required-features = ["postgres-backend"]`. +- [ ] `testcontainers@0.22` and `testcontainers-modules@0.10` with the + `postgres` feature are in `[dev-dependencies]` of `vestige-core`. +- [ ] `anyhow`, `tokio`, `rand` are in `[dev-dependencies]`. +- [ ] `cargo build -p vestige-core` (default features) is unchanged: no + testcontainers in the build graph; no new warnings. +- [ ] `cargo test -p vestige-core` (default features) passes with no + changes to the Phase 1 test count beyond what `0002a..g` already + moved. +- [ ] `cargo test -p vestige-core --features postgres-backend --test '*'` + passes on a runner with Docker available, or skips cleanly with the + `docker unavailable; skip` lines. +- [ ] `cargo bench -p vestige-core --features postgres-backend + --bench pg_hybrid_search -- --quick` runs `pg_search_1k` to + completion and does NOT run `pg_search_100k` unless + `VESTIGE_BENCH_HEAVY=1`. +- [ ] `cargo clippy -p vestige-core --features postgres-backend + --all-targets -- -D warnings` is clean. +- [ ] The runbook (`0002i`) gets a one-paragraph "How to run the test + suite locally" callout referring back to this sub-plan's + "Verification" section. (`0002i` is owned separately; this sub-plan + just lists the dependency.) + +--- + +## Open questions for the implementer + +1. **Migration helper name.** `0002c` decides whether + `PgMemoryStore::run_migrations(&self)` or + `vestige_core::storage::postgres::migrations::run(&pool)` is the public + call. Update `common/mod.rs` to match. +2. **Update-on-missing contract.** `0002d` decides whether + `MemoryStore::update` returns `Err(NotFound)` or `Ok(())` with zero + affected rows when the id does not exist. The CRUD test stub here + accepts either; tighten the assert once the contract is fixed. +3. **Empty-query search contract.** `0002e` decides whether + `SearchQuery { text: None, embedding: None }` is `Ok(empty)` or an + error. Same tightening pattern as #2. +4. **Pool size for 100k bench.** Current value is 8; if the bench + bottlenecks on the pool, tune up to 16 or 32 and document in the + bench file's leading doc comment. +5. **Shared `TestEmbedder` location.** Currently duplicated between + `tests/phase_2/common/test_embedder.rs` and + `benches/test_embedder.rs`. If duplication bothers a reviewer, lift to + `crates/vestige-core/src/embedder/test_support.rs` behind a + `test-support` Cargo feature pulled in by both `tests` and `benches`. + Out of scope for this sub-plan; record as a follow-up. diff --git a/docs/plans/0002i-runbook.md b/docs/plans/0002i-runbook.md new file mode 100644 index 0000000..f63694f --- /dev/null +++ b/docs/plans/0002i-runbook.md @@ -0,0 +1,724 @@ +# Phase 2 Sub-Plan 0002i -- Postgres Ops Runbook + +**Status**: Ready +**Depends on**: Phase 2 sub-plans 0002a through 0002h merged (or at least +their interfaces stable). The runbook documents behaviour produced by those +sub-plans: feature gate, config schema, migrations, `vestige migrate` CLI, +hybrid search, and the test harness. Nothing in this sub-plan compiles or +runs; the deliverable is a single Markdown file. + +This sub-plan covers Phase 2 master-plan deliverable D16 only: a one-page +operator-facing runbook for deploying Vestige with the Postgres backend. + +--- + +## Context + +Why a runbook. The ADR (0002) and the master plan (0002) are written for +implementors. They settle execution-level decisions and itemise deliverables. +They are not deployable instructions. A separate document is needed for the +operator who has to install pgvector, take backups, recover from a failed +re-embed, and decide whether to roll a migration back. The runbook is that +document. + +Who reads it. Ops people, not developers. Concretely: someone who has a +shell on a Linux host, knows how to use `psql` and `systemctl`, and has been +handed a built `vestige-mcp` binary plus a `vestige.toml`. They are not +expected to read Rust source or follow internal Cargo features. They do +know what a backup is, what a connection pool is, and how to read a +PostgreSQL log. + +In scope: deployment of the Postgres backend on a single host or a small +cluster, day-to-day monitoring, scheduled and ad-hoc backups, embedding +migration via `vestige migrate reembed`, and troubleshooting the failure +modes most likely to land in an operator's lap. + +Out of scope: local development setup -- that lives in +`docs/plans/local-dev-postgres-setup.md` and the runbook links to it for +developer onboarding only. Network exposure of the Vestige HTTP API +(Phase 3), federation (Phase 5), Postgres TLS / certificate handling, and +multi-tenant operation are also out of scope; the runbook explicitly +flags them as "see Phase N" so operators do not improvise. + +This sub-plan is the plan for producing the runbook. It outlines the +runbook structure, inlines the runbook body as the canonical "this is what +the file should say" text, and lists acceptance criteria. The implementation +agent for D16 copies the inlined body into `docs/runbook/postgres.md`, +creating `docs/runbook/` if it does not already exist. No other files in the +repository are modified. + +--- + +## Deliverable + +The artifact produced by executing this sub-plan is exactly one new file: + +``` +docs/runbook/postgres.md +``` + +It is NOT under `docs/plans/`. Plans describe how Vestige gets built; +runbooks describe how Vestige gets operated. The two directories are +deliberately separated. + +Side effect: create the directory `docs/runbook/` if it does not exist. +Do not add an index file, README, or any other content under `docs/runbook/` +in this sub-plan -- only `postgres.md`. + +This sub-plan document (`docs/plans/0002i-runbook.md`) is itself NOT a +deliverable in the operator sense. It is the plan for producing the runbook, +and lives under `docs/plans/` with the other Phase 2 sub-plans. + +--- + +## Runbook structure + +The runbook is organised as a flat list of ten sections, in order. Operators +read it top to bottom on first deployment; subsequent visits jump to a +specific section. Section numbering matches the inlined body below. + +1. **Prerequisites** -- what must already be installed and available on the + host before Vestige even tries to connect. PostgreSQL 16 or newer + (18 on Arch is fine), pgvector >= 0.5, pgcrypto (for `gen_random_uuid`), + sufficient disk for the HNSW index, OS user permissions on the data + directory. + +2. **Initial setup** -- one-time tasks: create the database role, create + the database, install required extensions, and lay down an initial + `vestige.toml`. Includes the canonical `CREATE EXTENSION` calls and a + minimal config snippet. + +3. **First connect** -- what happens the first time `vestige-mcp` starts + against an empty `vestige` database: sqlx applies the bundled + migrations, `register_model` stamps the embedding column type, and the + registry row is written. How an operator verifies each step succeeded + using `psql`. + +4. **Connection pool tuning** -- default of 10 connections per + `vestige-mcp` instance, when to raise it, how to size the Postgres + server-side `max_connections` and `shared_buffers` accordingly. Cross- + reference to `vestige.toml` and to ADR 0002 D2 / open question Q5. + +5. **Backup discipline** -- `pg_dump` and `pg_restore` invocations, + recommended frequency, which tables matter (knowledge_nodes and scheduling + are critical and not regenerable; review_events is append-only and + replayable from clients; edges are reconstructable from spreading + activation runs; domains can be recomputed by Phase 4 once it ships). + Also covers backup verification (restore-to-tmp drill). + +6. **Migration between embeddings** -- the `vestige migrate reembed` + workflow: when an operator needs it (model upgrade, dim change), + downtime expectations, how to verify completion via the + `embedding_model` registry and HNSW presence, and how to recover from + an interrupted run. + +7. **Re-clustering domains** -- a brief forward reference. Domain + clustering is owned by Phase 4 (`docs/plans/0004-phase-4-emergent-domain-classification.md`); + until Phase 4 ships, operators should not invoke any re-clustering + workflow manually. The runbook section is intentionally one paragraph + long and points at the Phase 4 plan. + +8. **Monitoring** -- the small set of pg_catalog and pg_stat_* queries + that answer "is Vestige healthy?": `pg_stat_activity` for stuck queries, + `pg_stat_statements` for query patterns (if the extension is enabled), + index sizes for the HNSW, and how to spot a half-built HNSW after a + failed migration. + +9. **Troubleshooting** -- a table of common errors with the symptom and + the fix. Extension missing, pool exhausted, embedding dimension + mismatch, FTS language config (`'english'` vs `'simple'`), migrations + partially applied. + +10. **Rollback caveats** -- every `*.up.sql` has a `*.down.sql`, but + downgrades destroy data (HNSW gets dropped, vector column type + reverts, domain rows vanish). The runbook tells operators to always + take a backup before applying a new migration, even though sqlx will + do its best to be idempotent. + +--- + +## Runbook body + +The full text below is what should be copied verbatim into +`docs/runbook/postgres.md`. ASCII only. Code blocks use fenced syntax with +language hints. Operator-facing prose; second person ("you") for +instructions. Where a command requires sudo, the prompt shows it explicitly. + +```markdown +# Vestige Postgres Backend -- Operator Runbook + +This runbook covers deploying, operating, monitoring, and recovering a +Vestige installation that uses the Postgres backend. It is written for +operators handling a built `vestige-mcp` binary and a `vestige.toml`. + +For local development setup, see +`docs/plans/local-dev-postgres-setup.md`. For the architectural rationale, +see `docs/adr/0001-pluggable-storage-and-network-access.md` and +`docs/adr/0002-phase-2-execution.md`. For the deliverable-level plan, see +`docs/plans/0002-phase-2-postgres-backend.md`. + +--- + +## 1. Prerequisites + +Before Vestige can connect: + +- PostgreSQL server, version 16 or newer. Arch ships 18.x; Debian stable + ships 16.x; both work. +- `pgvector` extension, version 0.5 or newer. Distro packages: + `pgvector` on Arch, `postgresql-16-pgvector` on Debian/Ubuntu. +- `pgcrypto` extension, shipped with the PostgreSQL contrib package + (`postgresql-contrib` on Debian, included in the base `postgresql` + package on Arch). Vestige uses `gen_random_uuid()` from pgcrypto for + primary keys. +- Disk space: budget roughly 4x the size of your `knowledge_nodes.embedding` + column for the HNSW index. With 768-dim float32 vectors at 100k + memories, that is about 1.2 GB for the embeddings plus 4-5 GB for the + HNSW index. Plan accordingly. +- OS user: the `postgres` system user (or whatever user owns + `/var/lib/postgres/data`) must have read/write on the data directory. + Vestige itself does not need filesystem access to Postgres; it talks + TCP only. +- Network: Vestige and Postgres can be on the same host (loopback) or + different hosts. If different hosts, allow the Vestige host's IP in + `pg_hba.conf` and on any firewall. + +--- + +## 2. Initial setup + +These steps run once per Postgres cluster. + +### 2.1 Install extensions + +As the `postgres` superuser: + +```sh +sudo -u postgres psql -d vestige <<'SQL' +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +SQL +``` + +Verify: + +```sh +sudo -u postgres psql -d vestige -c \ + "SELECT extname, extversion FROM pg_extension WHERE extname IN ('vector','pgcrypto');" +``` + +You should see two rows. If `vector` is missing, the pgvector package was +not installed for the right PostgreSQL major version; reinstall it. + +### 2.2 Create the role and database + +The `vestige` role owns its own database; it does NOT need superuser. +Extensions must be installed by `postgres`, not by `vestige`. + +```sh +sudo -u postgres psql -v ON_ERROR_STOP=1 <<'SQL' +CREATE ROLE vestige WITH LOGIN CREATEDB PASSWORD 'CHANGE_ME'; +CREATE DATABASE vestige OWNER vestige ENCODING 'UTF8'; +GRANT ALL PRIVILEGES ON DATABASE vestige TO vestige; +SQL + +sudo -u postgres psql -d vestige -v ON_ERROR_STOP=1 <<'SQL' +GRANT ALL ON SCHEMA public TO vestige; +ALTER SCHEMA public OWNER TO vestige; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO vestige; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO vestige; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO vestige; +SQL +``` + +Replace `CHANGE_ME` with a strong password and store it where Vestige can +read it (typically `~/.vestige_pg_pw`, mode 600, owned by the user running +`vestige-mcp`). + +### 2.3 Minimal `vestige.toml` + +```toml +[storage] +backend = "postgres" + +[storage.postgres] +url = "postgresql://vestige:CHANGE_ME@127.0.0.1:5432/vestige" +max_connections = 10 +``` + +The `url` field accepts a `${VAR}` placeholder; in practice operators +either inline the password or export `DATABASE_URL` and reference +`url = "${DATABASE_URL}"`. See `docs/CONFIGURATION.md` for the full +schema once Phase 3 lands. + +--- + +## 3. First connect + +When `vestige-mcp` starts against an empty `vestige` database, it: + +1. Builds a `PgPool` of `max_connections` (default 10) connections. +2. Runs every migration in `crates/vestige-core/migrations/postgres/` + in order. The bundled migrations are `0001_init` (tables, non-vector + indexes) and `0002_hnsw` (HNSW index on `knowledge_nodes.embedding`). +3. Calls `register_model` once it knows the active embedder's dimension. + This issues `ALTER TABLE knowledge_nodes ALTER COLUMN embedding TYPE + vector($N)` and inserts a row into `embedding_model`. +4. Begins accepting MCP requests. + +To verify after the first start: + +```sh +sudo -u postgres psql -d vestige <<'SQL' +-- All expected tables present. +\dt +-- embedding_model has exactly one row. +SELECT name, dimension, hash FROM embedding_model; +-- The HNSW index exists. +SELECT indexname FROM pg_indexes + WHERE tablename = 'knowledge_nodes' AND indexname LIKE '%hnsw%'; +SQL +``` + +Expected: `knowledge_nodes`, `scheduling`, `edges`, `domains`, `review_events`, +`embedding_model`, `users`, `groups`, `group_memberships`; one row in +`embedding_model`; one `idx_knowledge_nodes_embedding_hnsw` index. + +If a migration fails mid-way, the partial state lands in +`_sqlx_migrations`. See section 9 for recovery. + +--- + +## 4. Connection pool tuning + +Defaults: + +- Vestige client pool: `max_connections = 10` per `vestige-mcp` instance. +- Postgres server: `max_connections = 100` (default). + +Math: one MCP client with the default pool uses up to 10 server slots. +Five concurrent MCP clients use up to 50 slots. The remaining 50 cover +`psql` sessions, background workers, and headroom for replication or +backup processes. + +When to raise: + +- More than three MCP clients connecting to one Postgres instance. +- Long-running queries (above 500ms p99) showing pool wait time in + Vestige logs (look for `pool acquire timed out` warnings). +- A noticeable number of concurrent dream/consolidation runs. + +How to raise: + +```toml +[storage.postgres] +max_connections = 20 # client side, per vestige-mcp instance +``` + +And on the Postgres server, edit `postgresql.conf`: + +```conf +max_connections = 200 +shared_buffers = 2GB # roughly 25 percent of RAM, never above 8GB +``` + +Then restart Postgres (`sudo systemctl restart postgresql`). Vestige +clients pick up their own `max_connections` change on next restart. + +Do not raise pool sizes blindly. Past about 4x the CPU core count, +Postgres throughput drops; a small connection pooler (PgBouncer in +transaction mode) is the right answer above ~200 client connections, +but Vestige's expected scale rarely needs that. + +--- + +## 5. Backup discipline + +### 5.1 Which tables matter + +| Table | Backup priority | Regenerable? | +|-------|-----------------|--------------| +| `knowledge_nodes` | Critical | No | +| `scheduling` | Critical | No (FSRS state) | +| `embedding_model` | Critical | No (one row, but stamps the column type) | +| `users`, `groups`, `group_memberships` | Critical | No (Phase 3 will populate) | +| `review_events` | Important | Replayable by clients but tedious | +| `edges` | Optional | Yes (recomputed by spreading activation) | +| `domains` | Optional | Yes (Phase 4 recomputes by clustering) | + +For a typical single-operator install, dumping the whole database is +fastest and simplest. Skip the optional tables only if dump size becomes +a bandwidth problem. + +### 5.2 Full logical backup + +```sh +pg_dump --host=127.0.0.1 --username=vestige --format=custom \ + --file=vestige-$(date -u +%Y%m%dT%H%M%SZ).dump \ + vestige +``` + +The custom format compresses by default and works with parallel restore. +File size for 10k memories: roughly 80 MB. + +Frequency recommendations: + +- Daily for any installation with active ingest. +- Before every `vestige migrate reembed` run (see section 6). +- Before every Postgres major-version upgrade. +- Retain at least 7 daily, 4 weekly, 3 monthly dumps. Compress with + `--format=custom` (already gzipped) and keep them on different + storage from the database itself. + +### 5.3 Restore + +To a fresh database: + +```sh +sudo -u postgres createdb -O vestige vestige_restore +pg_restore --host=127.0.0.1 --username=vestige --dbname=vestige_restore \ + --jobs=4 vestige-20260301T030000Z.dump +``` + +To replace the live database (destructive; only after taking a fresh +dump): + +```sh +sudo systemctl stop vestige-mcp # or however the service is run +sudo -u postgres dropdb vestige +sudo -u postgres createdb -O vestige vestige +pg_restore --host=127.0.0.1 --username=vestige --dbname=vestige \ + --jobs=4 vestige-20260301T030000Z.dump +sudo systemctl start vestige-mcp +``` + +### 5.4 Restore drill + +Run a restore-to-throwaway-database every month and run `vestige search` +or a manual `psql` count against it. A backup you have not restored is a +backup you do not have. + +```sh +sudo -u postgres createdb -O vestige vestige_restore_drill +pg_restore --host=127.0.0.1 --username=vestige --dbname=vestige_restore_drill \ + --jobs=4 vestige-latest.dump +PGPASSWORD="$(cat ~/.vestige_pg_pw)" psql -h 127.0.0.1 -U vestige \ + -d vestige_restore_drill \ + -c 'SELECT count(*) FROM knowledge_nodes;' +sudo -u postgres dropdb vestige_restore_drill +``` + +--- + +## 6. Migration between embeddings + +Use `vestige migrate reembed` when: + +- Upgrading to a new embedding model that produces a different dimension + (for example, swapping from `nomic-embed-text-v1.5` 768D to a 1024D + model). +- Switching providers and the model hash differs even at the same + dimension. + +What it does: + +1. Reads every row from `knowledge_nodes`, re-encodes the `content` column + through the new embedder, and writes the new vector back. +2. Drops the HNSW index before the re-encode loop (this is the default; + `--concurrent-index` keeps it during the run at the cost of speed). +3. Updates the `embedding_model` row with the new name, dimension, and + hash. +4. Rebuilds the HNSW index with the new vectors. + +### 6.1 Before starting + +- Take a fresh backup (section 5.2). The tool refuses to start without a + `--yes` flag if it detects no recent backup; ignore at your peril. +- Stop ingest. Vestige's MCP server can stay running for read-only + access, but pause any client that calls `smart_ingest` or + `update_scheduling`. +- Have the new embedder model available locally. The CLI loads it + before the first row is touched; if loading fails, no data is changed. + +### 6.2 Running + +```sh +vestige migrate reembed --model= --yes +``` + +Add `--concurrent-index` if you cannot accept the brief window during +HNSW rebuild where queries do not use the index (sequential scan +fallback works but is slow). + +The tool prints a progress bar via `indicatif`. Expected throughput: +roughly 200 memories per second per CPU core for a 768D ONNX model. +10,000 memories on an 8-core box: about 6 seconds, plus HNSW rebuild +(another 30-90 seconds at that scale). + +### 6.3 Verifying completion + +```sh +sudo -u postgres psql -d vestige <<'SQL' +-- Registry reflects the new model. +SELECT name, dimension, hash FROM embedding_model; +-- HNSW index is present and not partial. +SELECT indexname, indexdef + FROM pg_indexes + WHERE tablename = 'knowledge_nodes' AND indexname LIKE '%hnsw%'; +-- All rows have a non-null embedding of the new dimension. +SELECT count(*) FILTER (WHERE embedding IS NULL) AS missing, + count(*) AS total + FROM knowledge_nodes; +SQL +``` + +Expected: registry shows the new model name and dimension, one HNSW +index, zero missing embeddings. + +### 6.4 Recovering from an interrupted run + +`vestige migrate reembed` is restartable. On interruption: + +- The `embedding_model` row may or may not have been updated. Check it + manually and roll forward by re-running with `--yes --resume` (the + tool detects the inconsistency and finishes the rows that still hold + old embeddings). +- The HNSW index may be missing. Re-running the command rebuilds it as + its last step. +- If the system is in a state the tool refuses to reason about, restore + from the backup taken in 6.1. + +--- + +## 7. Re-clustering domains + +Domain clustering is owned by Phase 4 +(`docs/plans/0004-phase-4-emergent-domain-classification.md`). Until +Phase 4 ships, the `domains` table is reserved schema and is populated +only by tests. Operators must not invoke any domain re-clustering +workflow manually; there is no supported one in Phase 2. + +When Phase 4 lands, this section is replaced with the real procedure. + +--- + +## 8. Monitoring + +### 8.1 Quick health check + +```sh +PGPASSWORD="$(cat ~/.vestige_pg_pw)" psql -h 127.0.0.1 -U vestige -d vestige <<'SQL' +SELECT count(*) AS memory_count FROM knowledge_nodes; +SELECT name, dimension FROM embedding_model; +SELECT pg_size_pretty(pg_database_size('vestige')) AS db_size; +SQL +``` + +### 8.2 In-flight queries + +```sql +SELECT pid, now() - query_start AS runtime, state, query + FROM pg_stat_activity + WHERE datname = 'vestige' AND state <> 'idle' + ORDER BY runtime DESC NULLS LAST; +``` + +Anything over 5 seconds with `state = 'active'` deserves a look. HNSW +search queries should land well under 100ms on properly-sized hardware. + +### 8.3 Query pattern analysis + +If `pg_stat_statements` is loaded (`shared_preload_libraries = +'pg_stat_statements'` in `postgresql.conf`): + +```sql +SELECT calls, mean_exec_time, query + FROM pg_stat_statements + WHERE query ILIKE '%knowledge_nodes%' + ORDER BY mean_exec_time DESC + LIMIT 20; +``` + +Look for hybrid-search queries that have drifted above 100ms p50. The +usual culprit is a missing or half-built HNSW index. + +### 8.4 Index health + +```sql +SELECT indexname, pg_size_pretty(pg_relation_size(indexrelid)) AS size, + idx_scan, idx_tup_read + FROM pg_indexes + JOIN pg_stat_user_indexes USING (indexrelid) + WHERE schemaname = 'public' AND relname = 'knowledge_nodes'; +``` + +A HNSW index with `idx_scan = 0` after several hours of traffic usually +means the planner is preferring sequential scan -- either the table is +too small to bother with the index (fine) or the index is corrupt and +needs rebuilding (`REINDEX INDEX idx_knowledge_nodes_embedding_hnsw;`). + +### 8.5 Spotting half-built HNSW + +After a failed migration or a crashed `reembed`: + +```sql +SELECT indexname, indisvalid, indisready + FROM pg_indexes + JOIN pg_index ON indexrelid = (schemaname || '.' || indexname)::regclass + WHERE tablename = 'knowledge_nodes'; +``` + +Any row with `indisvalid = false` is broken. Drop and recreate: + +```sql +DROP INDEX IF EXISTS idx_knowledge_nodes_embedding_hnsw; +CREATE INDEX idx_knowledge_nodes_embedding_hnsw + ON knowledge_nodes USING hnsw (embedding vector_cosine_ops); +``` + +--- + +## 9. Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| `ERROR: extension "vector" is not available` on start | pgvector not installed for this Postgres major version | Install the distro package matching `pg_config --version`, then `CREATE EXTENSION vector;` as superuser | +| `pool timed out while waiting for an open connection` in Vestige logs | Pool too small or stuck queries holding connections | Raise `max_connections` in `vestige.toml`; investigate `pg_stat_activity` for queries above 5s | +| `vector dimensions do not match` on insert | `embedding_model` was stamped at one dimension and a different embedder is now running | Re-run `vestige migrate reembed --model=` or fix the embedder configuration | +| Hybrid search returns the same row twice | Stale `.sqlx/` query cache from before D5 landed | Run `cargo sqlx prepare` in `crates/vestige-core/`, rebuild the binary | +| `text search configuration "english" does not exist` | Postgres locale build does not include the english dictionary (rare on Alpine) | Install the language-pack or override the FTS language in `vestige.toml` (see `[storage.postgres.fts]` once Phase 2 D5 lands) | +| `relation "_sqlx_migrations" exists, but migration X is in "applied" with no checksum` | Previous run died between `BEGIN` and `COMMIT` | Stop Vestige, restore from backup, restart | +| HNSW index very large compared to data | `m` and `ef_construction` defaults too high for the corpus | Acceptable for now; tuning lands as part of Phase 4 | +| `permission denied for schema public` on a new install | `vestige` role does not own `public` | Re-run the grants block in section 2.2 as `postgres` | + +If a problem is not in this table, capture: PostgreSQL log +(`/var/log/postgres/`, journalctl `-u postgresql`), Vestige log +(`RUST_LOG=debug,sqlx=info` for a fresh run), the migration state +(`SELECT * FROM _sqlx_migrations ORDER BY version;`), and file a bug. + +--- + +## 10. Rollback caveats + +Every migration in `crates/vestige-core/migrations/postgres/` has a +matching `*.down.sql`. `sqlx migrate revert` walks them in reverse order. + +This is not the same as risk-free. The `0002_hnsw.down.sql` drops the +HNSW index (rebuildable, expensive). The `0001_init.down.sql` drops +every table -- including `knowledge_nodes`, including data. Down migrations +exist for development, not for casual production use. + +Before applying any new migration: + +1. Take a backup (section 5.2). +2. Run the migration on a restored copy first if you can afford the time. +3. Read the new migration's `*.up.sql` and `*.down.sql` to understand + what changes. + +To revert one migration manually: + +```sh +sqlx migrate revert \ + --database-url "postgresql://vestige:...@127.0.0.1:5432/vestige" \ + --source crates/vestige-core/migrations/postgres +``` + +Note that Vestige's binary does not run `sqlx migrate revert` +automatically. Reverts are always an explicit operator decision. + +If a revert fails partway through, treat the database as inconsistent: +restore from the backup taken in step 1. +``` + +--- + +## Cross-references + +- `docs/adr/0001-pluggable-storage-and-network-access.md` -- ADR that + established the pluggable backend. +- `docs/adr/0002-phase-2-execution.md` -- ADR settling Phase 2 execution + decisions; section "Architecture Overview" lists every table the + runbook references. +- `docs/plans/0002-phase-2-postgres-backend.md` -- master plan; D16 + (deliverables list) and the Open Implementation Questions section + (especially Q4 HNSW rebuild and Q5 pool sizing) inform the runbook's + recommendations. +- `docs/plans/local-dev-postgres-setup.md` -- developer-facing recipe + for a one-machine Arch / CachyOS dev cluster. The runbook links to it + as the "for development, see" pointer. +- `docs/CONFIGURATION.md` -- existing config doc; section 4 of the + runbook ("Connection pool tuning") cross-references it for the + authoritative `vestige.toml` schema. + +--- + +## Verification + +A reviewer is given: + +- A fresh Linux VM (Debian 12 or Arch current; both must work) with + network access and no Postgres installed. +- A built `vestige-mcp` binary for that platform. +- The runbook (`docs/runbook/postgres.md`). + +The reviewer follows the runbook top to bottom and reaches a state in +which Vestige answers MCP requests against the Postgres backend. +Checkpoints, in order: + +1. After section 1 (Prerequisites): `pg_config --version` returns 16 or + newer; `pkg-config --modversion libpq` resolves; the `pgvector` + distro package is installed. +2. After section 2.1 (Extensions): two rows in + `SELECT extname FROM pg_extension WHERE extname IN ('vector', 'pgcrypto');`. +3. After section 2.2 (Role + DB): `psql -U vestige -h 127.0.0.1 -d vestige -c '\conninfo'` + succeeds. +4. After section 2.3 (Config): `vestige.toml` parses (test by + `vestige config print` once that subcommand lands, otherwise + `vestige-mcp --check-config`). +5. After section 3 (First connect): the eight expected tables are + present; `embedding_model` has exactly one row; the HNSW index + exists; `vestige-mcp` log shows "Postgres backend ready". +6. After section 5.2 (Backup): the dump file exists and `pg_restore -l` + on it lists the expected tables. +7. After section 5.4 (Restore drill): the drill database holds the same + row count as the source. + +If any checkpoint fails, the runbook section that produced the failure +is the one that needs revision. Capture the exact command, exit code, +and log line; revise the runbook in a follow-up PR. + +A second reviewer reads the runbook without executing it and checks for: + +- ASCII only; no em dashes, no curly quotes, no Unicode arrows, no + ellipses, no bullets (`*`/`-` ASCII only). +- Every section number from 1 to 10 present and in order. +- Every cross-reference resolves to an existing file or to a Phase + number explicitly marked as "future". +- No code block longer than 30 lines; if longer, it should be split or + referenced from another file. + +--- + +## Acceptance criteria + +- [ ] `docs/runbook/` directory exists. +- [ ] `docs/runbook/postgres.md` exists and matches the inlined body + above byte-for-byte after stripping the outer code fence used in + this sub-plan to embed it. +- [ ] All ten sections from the "Runbook structure" outline are present + under their stated headings. +- [ ] No file other than `docs/runbook/postgres.md` is created or + modified by executing this sub-plan. +- [ ] ASCII only: no em dashes, no curly quotes, no Unicode arrows, + no ellipses, no Unicode bullets (`grep -P '[^\x00-\x7F]' + docs/runbook/postgres.md` returns no matches). +- [ ] Every cross-reference in the runbook points at a file that exists + in the repository at the time of merge, OR is explicitly framed + as "future Phase N" with a pointer to the relevant plan document. +- [ ] Every command block is copy-pastable: no `` syntax + that does not also have an inline note describing what to + substitute. +- [ ] A second pair of eyes confirms the verification checkpoints in the + preceding section are reproducible. +- [ ] The runbook is no longer than the inlined body in this sub-plan; + operators reach the end without losing patience. From 21f0b29baeab80e7513075c9e274e069419300d3 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 15:08:35 +0200 Subject: [PATCH 06/38] docs: rewrite local-dev-postgres-setup for container approach; bump pg16 -> pg18 Land the Postgres dev cluster recipe Jan provisioned on delandtj-home (rootless podman + pgvector/pgvector:pg18, PG 18.4, pgvector 0.8.2) and align all live ADR 0002 / Phase 2 sub-plan references from pg16 to pg18. - docs/plans/local-dev-postgres-setup.md -- rewritten end-to-end: podman container vestige-pg with --restart=always, named volume vestige-pgdata, PGDATA=/var/lib/postgresql/data/pgdata, port mapping 127.0.0.1:5432:5432, two-password split (superuser + app role), pgvector preinstalled, CREATE EXTENSION vector handled at setup, day-to-day commands, password rotation, dev-grade backup/restore, teardown, boot-persistence notes for rootless podman. Old native Arch install recipe moved to Out-of-scope (covered by image now). - docs/adr/0002-phase-2-execution.md -- the open-thread mention of pgvector/pgvector:pg16 in the Follow-ups section now reads pg18. - docs/plans/0002c-migrations.md -- container example in the local dev section updated to pg18. - docs/plans/0002d-store-impl-bodies.md -- testcontainers GenericImage tag pg16 -> pg18; prose reference updated. - docs/plans/0002h-testing-and-benches.md -- harness pg18 across testcontainers Postgres builder, image-caching prose, CI workflow example. The archival master plan (docs/plans/0002-phase-2-postgres-backend.md) keeps its original pg16 references intentionally; the supersession notice already points readers to the live sub-plans. --- docs/adr/0002-phase-2-execution.md | 2 +- docs/plans/0002c-migrations.md | 2 +- docs/plans/0002d-store-impl-bodies.md | 4 +- docs/plans/0002h-testing-and-benches.md | 14 +- docs/plans/local-dev-postgres-setup.md | 248 ++++++++++++++++++------ 5 files changed, 198 insertions(+), 72 deletions(-) diff --git a/docs/adr/0002-phase-2-execution.md b/docs/adr/0002-phase-2-execution.md index 6b6949f..5d590b4 100644 --- a/docs/adr/0002-phase-2-execution.md +++ b/docs/adr/0002-phase-2-execution.md @@ -504,7 +504,7 @@ own migrations. - Validate local Postgres dev cluster before PR C work begins. Recipe at `docs/plans/local-dev-postgres-setup.md` is correct but needs to be applied on this machine (delandtj-home): cluster is not initdb'd, pgvector is not - installed. Containerized `pgvector/pgvector:pg16` is a viable alternative + installed. Containerized `pgvector/pgvector:pg18` is a viable alternative if pgvector packaging is friction. See open discussion thread. ### Phase 4 sketch: `sharing_rules` and the precedence chain diff --git a/docs/plans/0002c-migrations.md b/docs/plans/0002c-migrations.md index ef8e35c..78b6ac6 100644 --- a/docs/plans/0002c-migrations.md +++ b/docs/plans/0002c-migrations.md @@ -843,7 +843,7 @@ podman run --rm -d --name vestige-pg \ -e POSTGRES_USER=vestige \ -e POSTGRES_DB=vestige \ -p 5432:5432 \ - docker.io/pgvector/pgvector:pg16 + docker.io/pgvector/pgvector:pg18 export DATABASE_URL="postgresql://vestige:devpw@127.0.0.1:5432/vestige" ``` diff --git a/docs/plans/0002d-store-impl-bodies.md b/docs/plans/0002d-store-impl-bodies.md index ad1d9b7..adfd8aa 100644 --- a/docs/plans/0002d-store-impl-bodies.md +++ b/docs/plans/0002d-store-impl-bodies.md @@ -1612,7 +1612,7 @@ use vestige_core::storage::postgres::PgMemoryStore; #[tokio::test] async fn round_trip_crud_search_scheduling_edges() { let docker = clients::Cli::default(); - let image = GenericImage::new("pgvector/pgvector", "pg16") + let image = GenericImage::new("pgvector/pgvector", "pg18") .with_env_var("POSTGRES_PASSWORD", "test") .with_env_var("POSTGRES_DB", "vestige_test") .with_exposed_port(5432); @@ -1759,7 +1759,7 @@ This sub-plan is complete when ALL of the following hold: and the `Visibility` enum is exported alongside it. The SQLite backend reads and writes the same four fields. 8. The `tests/postgres_round_trip.rs` integration test passes against - a `pgvector/pgvector:pg16` container (insert / get / update / delete + a `pgvector/pgvector:pg18` container (insert / get / update / delete / fts_search / vector_search / get_scheduling / update_scheduling / add_edge / get_edges / remove_edge / get_neighbors / cascade delete). diff --git a/docs/plans/0002h-testing-and-benches.md b/docs/plans/0002h-testing-and-benches.md index d6bcebc..3fc2e1e 100644 --- a/docs/plans/0002h-testing-and-benches.md +++ b/docs/plans/0002h-testing-and-benches.md @@ -166,12 +166,12 @@ use vestige_core::storage::postgres::PgMemoryStore; pub async fn fresh_pg_store( embedder: Arc, ) -> Result<(PgMemoryStore, ContainerAsync)> { - // pgvector/pgvector:pg16 is the official pgvector image built on the - // postgres:16 base. testcontainers-modules::postgres::Postgres targets + // pgvector/pgvector:pg18 is the official pgvector image built on the + // postgres:18 base. testcontainers-modules::postgres::Postgres targets // the upstream postgres image by default; we override name + tag. let container = Postgres::default() .with_name("pgvector/pgvector") - .with_tag("pg16") + .with_tag("pg18") .start() .await?; @@ -867,7 +867,7 @@ Requirements: the `docker_available()` check in `common/mod.rs`. The test output includes a `docker unavailable; skip` line per test so the developer knows the tests were not silently dropped. -- The pgvector image (`pgvector/pgvector:pg16`) is pulled on first run; +- The pgvector image (`pgvector/pgvector:pg18`) is pulled on first run; ~200 MB. A pre-pulled image keeps the per-run overhead at the cold-start container boot (~2-5 seconds). @@ -920,7 +920,7 @@ async fn build_bench(rows: usize) -> Bench { let embedder = TestEmbedder::new_768(); let container = Postgres::default() .with_name("pgvector/pgvector") - .with_tag("pg16") + .with_tag("pg18") .start() .await .unwrap(); @@ -1092,7 +1092,7 @@ Notes: - The Postgres feature tests should run in a separate CI matrix entry to isolate failures and skip them entirely on platforms (Windows runners if any) where the pgvector image is not available. -- Cache the `pgvector/pgvector:pg16` image between runs. The +- Cache the `pgvector/pgvector:pg18` image between runs. The `docker/setup-buildx-action` cache or a simple `docker pull` step before the test step keeps cold-start under the existing CI time budget. - Skip CI: contributors without Docker can still merge changes that do @@ -1113,7 +1113,7 @@ jobs: # no `postgres` service block needed; testcontainers manages its own steps: - uses: actions/checkout@v4 - - run: docker pull pgvector/pgvector:pg16 + - run: docker pull pgvector/pgvector:pg18 - uses: dtolnay/rust-toolchain@stable - run: cargo test -p vestige-core --features postgres-backend --test '*' ``` diff --git a/docs/plans/local-dev-postgres-setup.md b/docs/plans/local-dev-postgres-setup.md index 6250a55..f863d48 100644 --- a/docs/plans/local-dev-postgres-setup.md +++ b/docs/plans/local-dev-postgres-setup.md @@ -1,27 +1,55 @@ -# Local Dev Postgres Setup (Arch / CachyOS) +# Local Dev Postgres Setup (container, hybrid approach) -**Status**: Applied on this machine on 2026-04-21 -**Related**: docs/plans/0002-phase-2-postgres-backend.md, docs/adr/0001-pluggable-storage-and-network-access.md +**Status**: Applied on this machine on 2026-05-27 (rootless podman, Postgres 18.4 + pgvector 0.8.2). +**Related**: docs/plans/0002-phase-2-postgres-backend.md, docs/adr/0002-phase-2-execution.md, docs/adr/0001-pluggable-storage-and-network-access.md -Purpose: capture the minimum, repeatable steps to stand up a Postgres 18 instance on a local Arch/CachyOS box for Phase 2 (`PgMemoryStore`) development, `sqlx prepare`, and manual migration testing. This is a single-operator dev recipe, not a production runbook. +Purpose: capture the minimum, repeatable steps to stand up a long-lived +Postgres 18 + pgvector instance on a local Linux dev box for Phase 2 +(`PgMemoryStore`) development, `sqlx prepare`, and manual migration +testing. This is a single-operator dev recipe, not a production runbook. + +ADR 0002 picked the **hybrid container** approach over a native install: +the `pgvector/pgvector:pg18` image ships pgvector pre-installed, matches +the image testcontainers will use in the Phase 2 test harness, and avoids +the AUR/build-from-source friction of native pgvector packaging on Arch. --- ## Current state on this machine -- Package: `postgresql` 18.3-2 (pacman). Pulls `postgresql-libs`, `libxslt`. -- Service: `postgresql.service`, enabled + active. -- Listens on: `127.0.0.1:5432` and `[::1]:5432` only (default `listen_addresses = 'localhost'`). -- Data dir: `/var/lib/postgres/data`, owner `postgres:postgres`. -- Auth (`pg_hba.conf`, Arch defaults): `peer` for local socket, `scram-sha-256` for host 127.0.0.1/::1. +- Runtime: rootless `podman` 5.8.2 (Arch). `docker` 29.5.1 also installed but unused. +- Image: `docker.io/pgvector/pgvector:pg18` (PostgreSQL 18.4, pgvector 0.8.2). +- Container: `vestige-pg`, `--restart=always`, port `127.0.0.1:5432:5432`. +- Volume: named podman volume `vestige-pgdata`, mounted at + `/var/lib/postgresql/data` inside the container; `PGDATA` points at + `/var/lib/postgresql/data/pgdata` so the volume mount is non-empty at + init time (Postgres refuses to initdb into a non-empty directory). +- Listens on: `127.0.0.1:5432` only (port mapping is bound to loopback). +- Auth: `scram-sha-256` (image default for both local socket and host). ### Database + role -- Database: `vestige`, UTF8, owner `vestige`. -- Role: `vestige` with `LOGIN CREATEDB` (no superuser, no replication, no cross-db). -- Schema `public` re-owned to `vestige`, plus default privileges so any future tables / sequences / functions in `public` are fully owned and granted to `vestige`. +- Database: `vestige`, UTF8, owner `vestige`, `LC_COLLATE=C.UTF-8`, `LC_CTYPE=C.UTF-8`. +- Role: `vestige` with `LOGIN CREATEDB` (no superuser, no replication). +- Schema `public` re-owned to `vestige` with full default privileges on + future tables / sequences / functions. +- Extension: `vector` (pgvector 0.8.2) installed in the `vestige` + database by the superuser at setup time. -Net effect: the `vestige` role can create, alter, drop, and grant freely inside the `vestige` database -- enough for `sqlx::migrate!`, ad-hoc schema work, and the full Phase 2 `MemoryStore` surface. It cannot create extensions (see Phase 2 followups below) and cannot touch other databases. +Net effect: the `vestige` role can create, alter, drop, and grant freely +inside the `vestige` database -- enough for `sqlx::migrate!`, ad-hoc +schema work, and the full Phase 2 `MemoryStore` surface. It cannot create +extensions; the superuser handled `CREATE EXTENSION vector` already. + +### Passwords + +Two passwords live in the dev user's home, mode 600: + +- `~/.vestige_pg_superpw` -- the `postgres` superuser password inside the + container. Used for one-shot admin tasks (creating roles, installing + extensions, password rotation). Day-to-day app traffic does NOT use it. +- `~/.vestige_pg_pw` -- the `vestige` role password. This is the one the + Phase 2 backend, `sqlx prepare`, and ad-hoc `psql` invocations use. ### Connection @@ -29,13 +57,8 @@ Net effect: the `vestige` role can create, alter, drop, and grant freely inside postgresql://vestige:@127.0.0.1:5432/vestige ``` -Password lives at `~/.vestige_pg_pw`, mode 600, owned by the dev user (no sudo needed to read it). Read with: - -```sh -cat ~/.vestige_pg_pw -``` - -Recommended dev shell export (keep this OUT of the repo; use `.env` + gitignore or a shell rc): +Recommended dev shell export (keep this OUT of the repo; use `.env` + +gitignore or a shell rc): ```sh export DATABASE_URL="postgresql://vestige:$(cat ~/.vestige_pg_pw)@127.0.0.1:5432/vestige" @@ -45,109 +68,212 @@ export DATABASE_URL="postgresql://vestige:$(cat ~/.vestige_pg_pw)@127.0.0.1:5432 ## Reproduce from scratch -On a fresh Arch / CachyOS box with passwordless sudo: +On a fresh Linux box with `podman` installed and `python3` available: ```sh -# 1. Install -sudo pacman -S --noconfirm postgresql +# 1. Pull the image +podman pull docker.io/pgvector/pgvector:pg18 -# 2. Initialize the cluster (UTF8, scram-sha-256 for host, peer for local) -sudo -iu postgres initdb \ - --locale=C.UTF-8 --encoding=UTF8 \ - -D /var/lib/postgres/data \ - --auth-host=scram-sha-256 --auth-local=peer +# 2. Create a persistent named volume +podman volume create vestige-pgdata -# 3. Start + enable -sudo systemctl enable --now postgresql +# 3. Generate the superuser password and stash it (mode 600) +SUPER_PW=$(python3 -c 'import secrets,string; a=string.ascii_letters+string.digits; print("".join(secrets.choice(a) for _ in range(32)))') +umask 077 +printf '%s' "$SUPER_PW" > ~/.vestige_pg_superpw +chmod 600 ~/.vestige_pg_superpw -# 4. Generate a password and stash it in the dev user's home (mode 600) +# 4. Start the container +podman run -d \ + --name vestige-pg \ + --restart=always \ + -p 127.0.0.1:5432:5432 \ + -e POSTGRES_PASSWORD="$SUPER_PW" \ + -e PGDATA=/var/lib/postgresql/data/pgdata \ + -v vestige-pgdata:/var/lib/postgresql/data \ + docker.io/pgvector/pgvector:pg18 + +unset SUPER_PW + +# 5. Wait for ready +until podman exec vestige-pg pg_isready -U postgres -h 127.0.0.1 >/dev/null 2>&1; do + sleep 1 +done + +# 6. Generate the vestige role password and stash it (mode 600) VESTIGE_PW=$(python3 -c 'import secrets,string; a=string.ascii_letters+string.digits; print("".join(secrets.choice(a) for _ in range(32)))') umask 077 printf '%s' "$VESTIGE_PW" > ~/.vestige_pg_pw chmod 600 ~/.vestige_pg_pw -# 5. Create role + database + grants -sudo -u postgres psql -v ON_ERROR_STOP=1 < '[3,2,1]'::vector AS l2_distance;" ``` --- -## Phase 2 followups (before PgMemoryStore works) +## Boot persistence (rootless podman) -The cluster above is bare Postgres. Phase 2 needs `pgvector`: +`--restart=always` keeps the container alive across podman daemon +restarts, but rootless podman containers do NOT auto-start on system +boot unless the dev user has lingering enabled: ```sh -# Install the extension package -sudo pacman -S --noconfirm pgvector - -# Enable it in the vestige database (must run as postgres; vestige is not superuser) -sudo -u postgres psql -d vestige -c 'CREATE EXTENSION IF NOT EXISTS vector;' +sudo loginctl enable-linger "$USER" ``` -Verify: +After that, the `podman-restart.service` user unit handles restart of +`--restart=always` containers when the user session starts at boot: ```sh -PGPASSWORD="$(cat ~/.vestige_pg_pw)" psql -h 127.0.0.1 -U vestige -d vestige \ - -c "SELECT extname, extversion FROM pg_extension WHERE extname = 'vector';" +systemctl --user enable --now podman-restart.service ``` -Notes: +Skip both if you prefer to start the cluster manually each session with +`podman start vestige-pg`. -- `pgvector` must be available on the server before `sqlx::migrate!` runs, or the Phase 2 migration that declares typed `Vector` columns will fail. -- Testcontainer-based Phase 2 integration tests use `pgvector/pgvector:pg16` and are independent of this local cluster. This local cluster is for `sqlx prepare`, `cargo run -- migrate --to postgres`, and manual poking. -- `sqlx prepare` needs `DATABASE_URL` pointed at this cluster with `vestige` migrations already applied. Run from `crates/vestige-core/`. +--- + +## Day-to-day operation + +```sh +# Status +podman ps --filter name=vestige-pg + +# Logs (follow) +podman logs -f vestige-pg + +# psql as the app role +PGPASSWORD="$(cat ~/.vestige_pg_pw)" psql -h 127.0.0.1 -U vestige -d vestige + +# psql as the superuser (for grants, extensions, role admin) +podman exec -it vestige-pg psql -U postgres + +# Stop / start +podman stop vestige-pg +podman start vestige-pg + +# Restart in place +podman restart vestige-pg +``` --- ## Password rotation ```sh +# Rotate the vestige role password NEW_PW=$(python3 -c 'import secrets,string; a=string.ascii_letters+string.digits; print("".join(secrets.choice(a) for _ in range(32)))') umask 077 printf '%s' "$NEW_PW" > ~/.vestige_pg_pw chmod 600 ~/.vestige_pg_pw -sudo -u postgres psql -v ON_ERROR_STOP=1 \ +podman exec -i vestige-pg psql -U postgres -v ON_ERROR_STOP=1 \ -c "ALTER ROLE vestige WITH PASSWORD '${NEW_PW}';" unset NEW_PW + +# Rotate the superuser password (less common) +NEW_SUPER=$(python3 -c 'import secrets,string; a=string.ascii_letters+string.digits; print("".join(secrets.choice(a) for _ in range(32)))') +umask 077 +printf '%s' "$NEW_SUPER" > ~/.vestige_pg_superpw +chmod 600 ~/.vestige_pg_superpw +podman exec -i vestige-pg psql -U postgres -v ON_ERROR_STOP=1 \ + -c "ALTER ROLE postgres WITH PASSWORD '${NEW_SUPER}';" +unset NEW_SUPER ``` Then re-export `DATABASE_URL` in any live shells. --- +## Backup and restore (dev-grade) + +`pg_dump` writes a plain-text SQL dump to host disk. For dev data this is +enough; production runbook lives in `0002i-runbook.md`. + +```sh +# Dump +PGPASSWORD="$(cat ~/.vestige_pg_pw)" pg_dump -h 127.0.0.1 -U vestige -d vestige \ + --format=plain --no-owner > vestige-$(date +%Y%m%d-%H%M%S).sql + +# Restore (drops + recreates) +podman exec -i vestige-pg psql -U postgres -v ON_ERROR_STOP=1 \ + -c 'DROP DATABASE IF EXISTS vestige;' \ + -c 'CREATE DATABASE vestige OWNER vestige ENCODING UTF8 TEMPLATE template0;' +PGPASSWORD="$(cat ~/.vestige_pg_pw)" psql -h 127.0.0.1 -U vestige -d vestige < vestige-DUMP.sql +``` + +The named volume `vestige-pgdata` persists outside the container; the +container can be `podman rm`'d and recreated without losing data, as +long as the volume stays in place. + +--- + ## Teardown Destroys the cluster and all data in it: ```sh -sudo systemctl disable --now postgresql -sudo pacman -Rns postgresql postgresql-libs -sudo rm -rf /var/lib/postgres -rm -f ~/.vestige_pg_pw +podman stop vestige-pg +podman rm vestige-pg +podman volume rm vestige-pgdata +podman rmi docker.io/pgvector/pgvector:pg18 +rm -f ~/.vestige_pg_pw ~/.vestige_pg_superpw ``` +`enable-linger` and the user systemd unit can be undone with +`sudo loginctl disable-linger "$USER"` and +`systemctl --user disable podman-restart.service` if you turned them on. + +--- + +## Notes for Phase 2 + +- `pgvector` is preinstalled in the image; the `CREATE EXTENSION vector` + in step 7 above makes it available inside the `vestige` DB. The + extension must be loaded BEFORE `sqlx::migrate!` runs the Phase 2 + migration that declares typed `Vector` columns, otherwise the + migration fails. +- Testcontainer-based Phase 2 integration tests use the same + `pgvector/pgvector:pg18` image and spin up fresh containers per run; + they are independent of this long-lived cluster. This cluster exists + for `sqlx prepare`, `cargo run -- migrate --to postgres`, and manual + poking. +- `sqlx prepare` needs `DATABASE_URL` pointed at this cluster with + `vestige` migrations already applied. Run from `crates/vestige-core/`. + --- ## Out of scope for this doc -- TLS, client-cert auth, non-localhost access. Phase 3 exposes the Vestige HTTP API over the network, not Postgres directly. -- Backups, PITR, WAL archiving. For dev data: `pg_dump -h 127.0.0.1 -U vestige vestige > vestige.sql`. -- Replication, PgBouncer, tuned `postgresql.conf`. Defaults are fine for Phase 2 development. -- Making this the canonical Vestige backend. By default Vestige still uses SQLite; this cluster exists so the `postgres-backend` feature can be built and tested locally. +- TLS, client-cert auth, non-localhost access. Phase 3 exposes the + Vestige HTTP API over the network, not Postgres directly. +- PITR, WAL archiving, replication, PgBouncer, tuned `postgresql.conf`. + Defaults are fine for Phase 2 development. +- Native (non-container) Postgres install. The prior version of this + doc covered native Arch packaging; superseded by ADR 0002's hybrid + decision. +- Making this the canonical Vestige backend. By default Vestige still + uses SQLite; this cluster exists so the `postgres-backend` feature + can be built and tested locally. From 3df930ca7e5d55352d71d97ea855bd8c26336be4 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Wed, 27 May 2026 20:00:46 -0500 Subject: [PATCH 07/38] Fix data-dir permission preservation --- crates/vestige-mcp/src/main.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/crates/vestige-mcp/src/main.rs b/crates/vestige-mcp/src/main.rs index 52524ba..916c441 100644 --- a/crates/vestige-mcp/src/main.rs +++ b/crates/vestige-mcp/src/main.rs @@ -256,12 +256,13 @@ fn prepare_storage_path(data_dir: Option) -> io::Result } // Only create if it doesn't exist (avoids "File exists" error on existing directories) - if !data_dir.exists() { + let created = !data_dir.exists(); + if created { fs::create_dir_all(&data_dir)?; } #[cfg(unix)] - { + if created { use std::os::unix::fs::PermissionsExt; let _ = fs::set_permissions(&data_dir, fs::Permissions::from_mode(0o700)); } @@ -594,6 +595,23 @@ mod tests { assert_eq!(db_path, Some(data_dir.join(DATABASE_FILE))); } + #[cfg(unix)] + #[test] + fn prepare_storage_path_preserves_existing_data_dir_permissions() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let data_dir = temp.path().join("shared"); + fs::create_dir_all(&data_dir).unwrap(); + fs::set_permissions(&data_dir, fs::Permissions::from_mode(0o755)).unwrap(); + + let db_path = prepare_storage_path(Some(data_dir.clone())).unwrap(); + let mode = fs::metadata(&data_dir).unwrap().permissions().mode() & 0o777; + + assert_eq!(db_path, Some(data_dir.join(DATABASE_FILE))); + assert_eq!(mode, 0o755); + } + #[test] fn expand_tilde_expands_current_users_home_only() { let home = BaseDirs::new().unwrap().home_dir().to_path_buf(); From 8c18231a20596528773a6b1d1d5d04a57632f1eb Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 28 May 2026 13:34:09 -0500 Subject: [PATCH 08/38] docs: keep README focused on v2.1.23 --- README.md | 112 ------------------------------------------------------ 1 file changed, 112 deletions(-) diff --git a/README.md b/README.md index 674eebb..f747715 100644 --- a/README.md +++ b/README.md @@ -33,118 +33,6 @@ observable, and harder to spoof. - **Safer batch writes.** `smart_ingest` batch mode now keeps caller-separated items separate by default and returns merge previews when an existing memory is mutated. - **Opt-in NVIDIA acceleration path.** Qwen3 embedding builds expose CUDA/cuDNN feature flags for contributors and users with CUDA-capable hosts. -## What's New in v2.1.22 "Sanhedrin Receipts" - -v2.1.22 makes the optional Sanhedrin hook accountable enough to trust in daily -agent work. Vetoes now leave local receipts, verification claims need real -command evidence, and users can appeal stale or over-strict blocks from the -dashboard. - -- **Receipt Lock.** Claims like "tests passed", "build is green", or "lint is clean" are blocked unless the current transcript contains a matching successful command receipt. -- **Screenshotable veto receipts.** Sanhedrin writes `~/.vestige/sanhedrin/latest.json` and `latest.html` with Claim -> Verdict -> Precedent -> Fix -> Appeal. -- **Dashboard Verdict Bar.** The dashboard shows PASS, NOTE, CAUTION, VETO, or APPEALED globally, expands into the receipt, and records stale/wrong/too-strict appeals. -- **Claim ledger.** Claim-mode Sanhedrin output now maps every extracted claim into structured JSON instead of treating the whole draft as one blob. -- **Appeal training.** Appeals are saved to `appeals.jsonl` and suppress future vetoes for the same claim fingerprint. - -## What's New in v2.1.21 "Agent-Neutral Hardening" - -v2.1.21 tightens Vestige for normal use across MCP-compatible agents, without -making Claude Code companion tooling part of the default path. - -- **Agent-neutral default.** Stdio MCP remains the default transport; optional HTTP MCP is explicit with `--http`, `--http-port`, or `VESTIGE_HTTP_ENABLED=1`. -- **Safer destructive actions.** `memory(action="delete")` now requires `confirm=true`, matching `purge`, and the legacy `delete_knowledge` shim forwards that confirmation instead of bypassing it. -- **Portable sync repair.** Merge imports preserve purge tombstones, avoid `INSERT OR REPLACE` cascades, rebuild the vector index from a clean state, and write portable archive temp files with private Unix permissions. -- **Release/package cleanup.** Release builds check the embedded dashboard before packaging, publish checksums, and the npm installer rejects targets that do not have release assets. -- **Any-agent memory protocol.** The setup docs now include a short agent-agnostic memory protocol for Claude Code, Codex, Cursor, VS Code, Xcode, JetBrains, Windsurf, and other MCP clients. - -## What's New in v2.1.2 "Honest Memory" - -v2.1.2 makes Vestige easier to trust in everyday work: literal lookups stay literal, purge really removes content, contradictions are inspectable, and updates no longer require a curl reinstall flow. - -- **Concrete search mode.** Quoted strings, env vars, UUIDs, paths, and code identifiers now take a keyword/literal path that skips HyDE, semantic fusion, FSRS reweighting, competition, and spreading activation. Exact things like `OPENAI_API_KEY`, `mlx_lm.server`, and migration IDs land first. -- **Irreversible purge.** `memory(action="purge", confirm=true)` permanently removes memory content and embeddings, scrubs insight JSON references, detaches temporal-summary children, prunes graph edges, and keeps only a non-content deletion tombstone for sync/audit. -- **First-class contradiction inspection.** New `contradictions` MCP tool surfaces trust-weighted disagreements directly instead of hiding them inside `deep_reference`. -- **Simple update flow.** `vestige update` refreshes binaries. Claude Code Cognitive Sandwich companion files are opt-in with `vestige update --sandwich-companion` or `vestige sandwich install`. -- **Pro waitlist preview.** `/dashboard/waitlist` adds a local-first Solo Pro and Team Pro early-access surface. `VITE_WAITLIST_ENDPOINT` and `VITE_SUPPORT_BOT_ENDPOINT` are opt-in dashboard env vars, so no signup data is captured unless endpoints are configured. - -## What's New in v2.1.1 "Portable Sync" - -v2.1.1 focuses on the biggest post-launch ask: move memories between machines without losing cognitive state. It also adds opt-in Qwen3 embeddings for higher-recall local retrieval. - -- **Exact portable archives.** `vestige portable-export` / `vestige portable-import` preserve IDs, FSRS state, graph edges, suppression state, audit rows, and embedding blobs for Vestige-to-Vestige device transfer. -- **Sync-safe merge storage.** `vestige portable-import --merge` and `vestige sync ` merge non-empty databases, apply delete tombstones, keep newer local memories, rebuild FTS, and push through a pluggable portable-sync backend. v2.1.1 ships the file backend for Dropbox, iCloud, Syncthing, Git, and shared folders. -- **Qwen3 embeddings.** Build with `qwen3-embeddings`, set `VESTIGE_EMBEDDING_MODEL=qwen3-0.6b`, and run `vestige consolidate` to re-embed existing memories. `vestige health` reports mixed-model stores before search quality is affected. -- **Model-aware retrieval.** Vestige now avoids comparing Qwen and Nomic vectors in the same search/dedup path. - -## What's New in v2.1.0 "Cognitive Sandwich Goes Local" - -v2.1.0 adds an opt-in Claude Code hook harness around the existing Vestige MCP server. The MCP tool surface and database schema stay backward compatible, while preflight hooks can inject trusted memory context before Claude answers. The heavyweight Sanhedrin verifier is optional and can be enabled separately. - -- **Optional Sanhedrin Executioner.** The post-response verifier is off by default. Users can enable it with an OpenAI-compatible endpoint on x86/Linux/Intel Mac, or add `--with-launchd` on Apple Silicon to run the local MLX Qwen backend. -- **One-command Cognitive Sandwich installer.** `vestige sandwich install` stages hook files and agents by default, removes old Vestige hook wiring, and leaves all Claude Code hook layers plus the 19 GB model path opt-in. -- **Pulse hook backed by `/api/changelog`.** Fresh dream and connection events can be injected into the next Claude Code prompt context without blocking the prompt. -- **`VESTIGE_DATA_DIR` support.** `--data-dir` now has an env-var fallback, tilde expansion, secure directory creation, and clear precedence docs. -- **NPM release wrapper fixed.** `vestige-mcp-server@2.1.0` now downloads binaries from the matching `v2.1.0` GitHub release tag instead of an old hardcoded release. - -## What's New in v2.0.9 "Autopilot" - -Autopilot flips Vestige from passive memory library to **self-managing cognitive surface**. Same 24 MCP tools, zero schema changes — but the moment you upgrade, 14 previously dormant cognitive primitives start firing on live events without any tool call from your client. - -- **One supervised backend task subscribes to the 20-event WebSocket bus** and routes six event classes into the cognitive engine: `MemoryCreated` triggers synaptic-tagging PRP + predictive-access records, `SearchPerformed` warms the speculative-retrieval model, `MemoryPromoted` fires activation spread, `MemorySuppressed` emits the Rac1 cascade wave, high-importance `ImportanceScored` (>0.85) auto-promotes, and `Heartbeat` rate-limit-fires `find_duplicates` on large DBs. **The engine mutex is never held across `.await`, so MCP dispatch is never starved.** -- **Panic-resilient supervisors.** Both background tasks run inside an outer supervisor loop — if one handler panics on a bad memory, the supervisor respawns it in 5 s instead of losing every future event. -- **Fully backward compatible.** No new MCP tools. No schema migration. Existing v2.0.8 databases open without a single step. Opt out with `VESTIGE_AUTOPILOT_ENABLED=0` if you want the passive-library contract back. -- **3,091 LOC of orphan v1.0 tool code removed** — nine superseded modules (`checkpoint`, `codebase`, `consolidate`, `ingest`, `intentions`, `knowledge`, `recall`, plus helpers) verified zero non-test callers before deletion. Tool surface unchanged. - -## What's New in v2.0.8 "Pulse" - -v2.0.8 wires the dashboard through to the cognitive engine. Eight new surfaces expose the reasoning stack visually — every one was MCP-only before. - -- **Reasoning Theater (`/reasoning`)** — `Cmd+K` Ask palette over the 8-stage `deep_reference` pipeline (hybrid retrieval → cross-encoder rerank → spreading activation → FSRS-6 trust → temporal supersession → contradiction analysis → relation assessment → template reasoning chain). Evidence cards, confidence meter, contradiction geodesic arcs, superseded-memory lineage, evolution timeline. **Zero LLM calls, 100% local.** -- **Pulse InsightToast** — real-time toasts for `DreamCompleted`, `ConsolidationCompleted`, `ConnectionDiscovered`, promote/demote/suppress/unsuppress, `Rac1CascadeSwept`. Rate-limited, auto-dismiss, click-to-dismiss. -- **Memory Birth Ritual (Terrarium)** — new memories materialize in the 3D graph on every `MemoryCreated`: elastic scale-in, quadratic Bezier flight path, glow sprite fade-in, Newton's Cradle docking recoil. 60-frame sequence, zero-alloc math. -- **7 more dashboard surfaces** — `/duplicates`, `/dreams`, `/schedule`, `/importance`, `/activation`, `/contradictions`, `/patterns`. Left nav expanded 8 → 16 with single-key shortcuts. -- **Intel Mac (`x86_64-apple-darwin`) support restored** via the `ort-dynamic` Cargo feature + Homebrew `onnxruntime`. Microsoft deprecated x86_64 macOS prebuilts; the dynamic-link path sidesteps that permanently. **Closes #41.** -- **Contradiction-detection false positives eliminated** — four thresholds tightened so adjacent-domain memories no longer flag as conflicts. On an FSRS-6 query this collapses false contradictions 12 → 0 without regressing legitimate test cases. - -## What's New in v2.0.7 "Visible" - -Hygiene release closing two UI gaps and finishing schema cleanup. No breaking changes, no user-data migrations. - -- **`POST /api/memories/{id}/suppress` + `/unsuppress` HTTP endpoints** — dashboard can trigger Anderson 2025 SIF + Rac1 cascade without dropping to raw MCP. `suppressionCount`, `retrievalPenalty`, `reversibleUntil`, `labileWindowHours` all in response. Suppress button joins Promote / Demote / Delete on the Memories page. -- **Uptime in the sidebar footer** — the `Heartbeat` event has carried `uptime_secs` since v2.0.5 but was never rendered. Now shows as `up 3d 4h` / `up 18m` / `up 47s`. -- **`execute_export` panic fix** — unreachable match arm replaced with a clean "unsupported export format" error instead of unwinding through the MCP dispatcher. -- **`predict` surfaces `predict_degraded: true`** on lock poisoning instead of silently returning empty vecs. `memory_changelog` honors `start` / `end` bounds. `intention` check honors `include_snoozed`. -- **Migration V11** — drops dead `knowledge_edges` + `compressed_memories` tables (added speculatively in V4, never used). - -## What's New in v2.0.6 "Composer" - -v2.0.6 is a polish release that makes the existing cognitive stack finally *feel* alive in the dashboard and stays out of your way on the prompt side. - -- **Six live graph reactions, not one** — `MemorySuppressed`, `MemoryUnsuppressed`, `Rac1CascadeSwept`, `Connected`, `ConsolidationStarted`, and `ImportanceScored` now light the 3D graph in real time. v2.0.5 shipped `suppress` but the graph was silent when you called it; consolidation and importance scoring have been silent since v2.0.0. No longer. -- **Intentions page actually works** — fixes a long-standing bug where every intention rendered as "normal priority" (type/schema drift between backend and frontend) and context/time triggers surfaced as raw JSON. -- **Opt-in composition mandate** — the new MCP `instructions` string stays minimal by default. Opt in to the full Composing / Never-composed / Recommendation composition protocol with `VESTIGE_SYSTEM_PROMPT_MODE=full` when you want it, and nothing is imposed on your sessions when you don't. - -## What's New in v2.0.5 "Intentional Amnesia" - -**The first shipped AI memory system with top-down inhibitory control over retrieval.** Other systems implement passive decay — memories fade if you don't touch them. Vestige v2.0.5 also implements *active* suppression: the new **`suppress`** tool compounds a retrieval penalty on every call (up to 80%), a background Rac1 worker fades co-activated neighbors over 72 hours, and the whole thing is reversible within a 24-hour labile window. **Never deletes.** The memory is inhibited, not erased. - -Ebbinghaus 1885 models what happens to memories you don't touch. Anderson 2025 models what happens when you actively want to stop thinking about one. Every other AI memory system implements the first. Vestige is the first to ship the second. - -Based on [Anderson et al. 2025](https://www.nature.com/articles/s41583-025-00929-y) (Suppression-Induced Forgetting, *Nat Rev Neurosci*) and [Cervantes-Sandoval et al. 2020](https://pmc.ncbi.nlm.nih.gov/articles/PMC7477079/) (Rac1 synaptic cascade). - -

-Earlier releases (v2.0 "Cognitive Leap" → v2.0.4 "Deep Reference") - -- **v2.0.4 — `deep_reference` Tool** — 8-stage cognitive reasoning pipeline with FSRS-6 trust scoring, intent classification, spreading activation, contradiction analysis, and pre-built reasoning chains. Token budgets raised 10K → 100K. CORS tightened. -- **v2.0 — 3D Memory Dashboard** — SvelteKit + Three.js neural visualization with real-time WebSocket events, bloom post-processing, force-directed graph layout. -- **v2.0 — WebSocket Event Bus** — Every cognitive operation broadcasts events: memory creation, search, dreaming, consolidation, retention decay. -- **v2.0 — HyDE Query Expansion** — Template-based Hypothetical Document Embeddings for dramatically improved search quality on conceptual queries. -- **v2.0 — Nomic v2 MoE (experimental)** — fastembed 5.11 with optional Nomic Embed Text v2 MoE (475M params, 8 experts) + Metal GPU acceleration. -- **v2.0 — Command Palette** — `Cmd+K` navigation, keyboard shortcuts, responsive mobile layout, PWA installable. -- **v2.0 — FSRS Decay Visualization** — SVG retention curves with predicted decay at 1d/7d/30d. - -
- --- ## Quick Start From cd496e562cafb38faf6b2af426d0640d08189538 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 28 May 2026 16:37:33 -0500 Subject: [PATCH 09/38] docs: add Glama ownership metadata --- glama.json | 6 ++++++ server.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 glama.json diff --git a/glama.json b/glama.json new file mode 100644 index 0000000..ecca9c4 --- /dev/null +++ b/glama.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://glama.ai/mcp/schemas/server.json", + "maintainers": [ + "samvallad33" + ] +} diff --git a/server.json b/server.json index 7a3d8dd..e11c5a4 100644 --- a/server.json +++ b/server.json @@ -2,7 +2,7 @@ "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.samvallad33/vestige", "title": "Vestige", - "description": "Local-first cognitive memory server for AI agents with SQLite, smart ingest, and portable sync.", + "description": "Local-first cognitive memory and accountability MCP server for AI agents. Uses local SQLite by default, exposes memory/search tools, and adds Receipt Lock to check verification claims against command receipts without requiring cloud services.", "repository": { "url": "https://github.com/samvallad33/vestige", "source": "github" From c9f457dd94dc558e2f8e5aad58a67d9f8b5e3ca9 Mon Sep 17 00:00:00 2001 From: ShalokShalom Date: Tue, 2 Jun 2026 16:06:08 +0200 Subject: [PATCH 10/38] Update CONFIGURATION.md with Opencode TUI details Added configuration instructions for Opencode TUI/Desktop. --- docs/CONFIGURATION.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 1205f70..b80c4e1 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -141,6 +141,20 @@ Add to `%APPDATA%\Claude\claude_desktop_config.json`: } ``` +### Opencode TUI/Desktop + +You can put it at various different locations. I recommend `opencode.json` in the project folder. + +```json +{ + "mcpServers": { + "vestige": { + "command": "vestige-mcp" + } + } +} +``` + --- ## Custom Data Directory From d8de7daf11c66e6b0741942f03362b7fb5015019 Mon Sep 17 00:00:00 2001 From: ShalokShalom Date: Tue, 2 Jun 2026 16:08:39 +0200 Subject: [PATCH 11/38] Update CONFIGURATION.md Add link to specific options to place the config file --- docs/CONFIGURATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index b80c4e1..64b9c06 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -143,7 +143,7 @@ Add to `%APPDATA%\Claude\claude_desktop_config.json`: ### Opencode TUI/Desktop -You can put it at various different locations. I recommend `opencode.json` in the project folder. +You can put it at [various different](https://opencode.ai/docs/config/#locations) locations. I recommend `opencode.json` in the project folder. ```json { From 00511948ffd345e17b3f12a858114fc35375f502 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Tue, 2 Jun 2026 12:38:18 -0500 Subject: [PATCH 12/38] Add developer launch kit for Vestige v2.1.23 Dual-wave marketing assets (Receipt Lock + cognitive memory), GitHub Pages landing, comparison doc, ready-to-post copy, growth-engine scripts, and a dedicated marketing Vestige data-dir setup path. Co-authored-by: Cursor --- .github/workflows/pages.yml | 46 +++++ CLAUDE.md | 3 +- README.md | 59 ++++++- docs/LAUNCH_STATS.md | 88 +++++++++ docs/comparison.md | 82 +++++++++ docs/launch/blog-post.md | 2 +- docs/launch/demo-script.md | 12 +- docs/launch/receipt-lock.md | 167 ++++++++++++++++++ docs/launch/reddit-cross-reference.md | 25 +-- docs/launch/show-hn.md | 38 ++-- docs/marketing/LAUNCH_NOW.md | 35 ++++ docs/marketing/README.md | 33 ++++ docs/marketing/assets/.gitkeep | 0 docs/marketing/assets/CAPTURE.md | 40 +++++ .../assets/dashboard-placeholder.svg | 10 ++ docs/marketing/demo-video-storyboard.md | 32 ++++ .../growth-engine/MARKETING-CLAUDE.md | 69 ++++++++ docs/marketing/growth-engine/README.md | 84 +++++++++ docs/marketing/mcp-registries.md | 57 ++++++ docs/marketing/metrics-tracker.md | 66 +++++++ .../ready-to-post/hn-first-comment.txt | 32 ++++ docs/marketing/ready-to-post/hn-title.txt | 1 + .../reddit-experienceddevs-title.txt | 1 + .../ready-to-post/reddit-experienceddevs.md | 19 ++ docs/marketing/ready-to-post/x-thread.txt | 18 ++ docs/marketing/wave-a-launch.md | 47 +++++ docs/marketing/wave-b-launch.md | 45 +++++ docs/website/index.html | 82 +++++++++ package.json | 3 +- packages/vestige-mcp-npm/README.md | 28 ++- packages/vestige-mcp-npm/package.json | 1 + scripts/marketing/preflight.sh | 47 +++++ scripts/marketing/seed-baseline-memories.sh | 29 +++ scripts/marketing/setup-marketing-instance.sh | 44 +++++ 34 files changed, 1303 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/pages.yml create mode 100644 docs/LAUNCH_STATS.md create mode 100644 docs/comparison.md create mode 100644 docs/launch/receipt-lock.md create mode 100644 docs/marketing/LAUNCH_NOW.md create mode 100644 docs/marketing/README.md create mode 100644 docs/marketing/assets/.gitkeep create mode 100644 docs/marketing/assets/CAPTURE.md create mode 100644 docs/marketing/assets/dashboard-placeholder.svg create mode 100644 docs/marketing/demo-video-storyboard.md create mode 100644 docs/marketing/growth-engine/MARKETING-CLAUDE.md create mode 100644 docs/marketing/growth-engine/README.md create mode 100644 docs/marketing/mcp-registries.md create mode 100644 docs/marketing/metrics-tracker.md create mode 100644 docs/marketing/ready-to-post/hn-first-comment.txt create mode 100644 docs/marketing/ready-to-post/hn-title.txt create mode 100644 docs/marketing/ready-to-post/reddit-experienceddevs-title.txt create mode 100644 docs/marketing/ready-to-post/reddit-experienceddevs.md create mode 100644 docs/marketing/ready-to-post/x-thread.txt create mode 100644 docs/marketing/wave-a-launch.md create mode 100644 docs/marketing/wave-b-launch.md create mode 100644 docs/website/index.html create mode 100755 scripts/marketing/preflight.sh create mode 100755 scripts/marketing/seed-baseline-memories.sh create mode 100755 scripts/marketing/setup-marketing-instance.sh diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..d30a8f3 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,46 @@ +name: Deploy GitHub Pages + +on: + push: + branches: [main] + paths: + - 'docs/website/**' + - 'docs/marketing/assets/**' + - '.github/workflows/pages.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v4 + + - name: Prepare site root + run: | + mkdir -p _site + cp -r docs/website/* _site/ + mkdir -p _site/assets + cp -r docs/marketing/assets/* _site/assets/ 2>/dev/null || true + # Fix asset paths for Pages (no parent ../) + sed -i 's|../marketing/assets/|assets/|g' _site/index.html || true + + - uses: actions/configure-pages@v5 + + - uses: actions/upload-pages-artifact@v3 + with: + path: _site + + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 4ee5762..c9034d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,8 @@ dashboard embedded into the release binary. The core product promise is: ## Working Rules - Prefer source evidence over memory. Use `rg`, tests, and nearby code before - making claims about behavior. + making claims about behavior. This public repo guidance does not override + private user-level memory protocols loaded outside the repository. - Keep release changes scoped. Do not rewrite unrelated modules during a version/tag cleanup unless the release gate requires it. - Preserve local-first behavior. Heavy models, Sanhedrin-style verifier hooks, diff --git a/README.md b/README.md index f747715..fb612ad 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Vestige -### Local cognitive memory for MCP-compatible AI agents. +### Local memory and receipts for MCP-compatible AI agents. [![GitHub stars](https://img.shields.io/github/stars/samvallad33/vestige?style=social)](https://github.com/samvallad33/vestige) [![Release](https://img.shields.io/github/v/release/samvallad33/vestige)](https://github.com/samvallad33/vestige/releases/latest) @@ -10,11 +10,23 @@ [![License](https://img.shields.io/badge/license-AGPL--3.0-blue)](LICENSE) [![MCP Compatible](https://img.shields.io/badge/MCP-compatible-green)](https://modelcontextprotocol.io) -**Your agent forgets project decisions between sessions. Vestige gives it local, inspectable memory.** +**Your coding agent forgets yesterday and can lie about today.** Vestige gives it a local brain — FSRS-6 memory that learns and forgets — plus **Receipt Lock** that blocks "tests passed" unless a real command receipt exists. -Built on proven memory and retrieval ideas — FSRS-6 spaced repetition, prediction error gating, synaptic tagging, spreading activation, and memory consolidation — all running in a single Rust binary with a local dashboard. 100% local. Zero cloud. +```bash +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user +``` -[Quick Start](#quick-start) | [Dashboard](#-3d-memory-dashboard) | [How It Works](#-the-cognitive-science-stack) | [Tools](#-25-mcp-tools) | [Docs](docs/) +| | | +|---|---| +| **Receipt Lock** — optional hook layer; vetoes unverified "green build" claims | **3D dashboard** — `vestige dashboard` → `localhost:3927` | +| ![Receipt Lock demo](docs/marketing/assets/receipt-lock.gif) · [capture guide](docs/marketing/assets/CAPTURE.md) | ![Memory dashboard](docs/marketing/assets/dashboard-placeholder.svg) | + +*Replace placeholders with GIFs from [`CAPTURE.md`](docs/marketing/assets/CAPTURE.md) before Wave B launch.* + +**v2.1.23** · ~86K LOC Rust · **25** MCP tools · **30** cognitive modules · **1,200+** tests · **22MB** binary · 100% local · AGPL-3.0 + +[Quick Start](#quick-start) | [Receipt Lock](#receipt-lock) | [Dashboard](#-3d-memory-dashboard) | [Compare vs RAG](docs/comparison.md) | [Launch stats](docs/LAUNCH_STATS.md) | [Docs](docs/) @@ -55,6 +67,43 @@ codex mcp add vestige -- vestige-mcp # → "You prefer TypeScript over JavaScript." ``` +## Receipt Lock + +Coding agents often finish with confident summaries like "tests passed" or +"the build is green." Receipt Lock checks those operational claims against +structured command receipts from the current transcript before they become part +of the final answer. + +If the agent claims verification happened but no matching successful command +receipt exists, Vestige can block the claim and write an inspectable local +receipt instead of letting the agent invent a clean ending. + +Receipt Lock is optional and works through the Claude Code Cognitive Sandwich +hook layer: + +```bash +# Install the local memory server first +npm install -g vestige-mcp-server@latest + +# Add normal MCP memory +claude mcp add vestige vestige-mcp -s user + +# Optional: enable Receipt Lock / Sanhedrin hooks +vestige sandwich install --enable-sanhedrin + +# Optional: point Sanhedrin at any OpenAI-compatible endpoint +vestige sandwich install \ + --enable-sanhedrin \ + --sanhedrin-endpoint=http://127.0.0.1:11434/v1/chat/completions \ + --sanhedrin-model=qwen2.5:14b +``` + +Receipts are local: + +- Latest JSON: `~/.vestige/sanhedrin/latest.json` +- Latest HTML: `~/.vestige/sanhedrin/latest.html` +- Command ledger: `~/.vestige/sanhedrin/command-receipts.jsonl` +
Other platforms & install methods @@ -422,6 +471,8 @@ vestige dashboard # Open 3D dashboard in browser | [CLAUDE.md Setup](docs/CLAUDE-SETUP.md) | Templates for proactive memory | | [Configuration](docs/CONFIGURATION.md) | CLI commands, environment variables | | [Integrations](docs/integrations/) | Codex, Xcode, Cursor, VS Code, JetBrains, Windsurf | +| [Comparison vs RAG/Mem0](docs/comparison.md) | When to use Vestige | +| [Marketing kit](docs/marketing/README.md) | Launch waves, growth engine, metrics | | [Changelog](CHANGELOG.md) | Version history | --- diff --git a/docs/LAUNCH_STATS.md b/docs/LAUNCH_STATS.md new file mode 100644 index 0000000..4bd1d92 --- /dev/null +++ b/docs/LAUNCH_STATS.md @@ -0,0 +1,88 @@ +# Vestige Launch Stats (Single Source of Truth) + +**Last verified:** 2026-06-02 +**Use this file** when updating README, launch posts, npm README, and landing page. Do not invent numbers elsewhere. + +## Release + +| Field | Value | +|-------|-------| +| Version | **v2.1.23** ("Receipt Lock Hardening") | +| npm package | `vestige-mcp-server@latest` | +| Install | `npm install -g vestige-mcp-server@latest` | +| MCP connect (Claude Code) | `claude mcp add vestige vestige-mcp -s user` | +| Optional Receipt Lock | `vestige sandwich install --enable-sanhedrin` | +| License | AGPL-3.0-only | +| Repo | https://github.com/samvallad33/vestige | +| Homepage (marketing) | https://samvallad33.github.io/vestige/ (GitHub Pages) | + +## Author + +| Field | Value | +|-------|-------| +| Name | Sam Valladares | +| Age | **22** (solo developer) | +| GitHub stars (2026-06-02) | **542** | +| Forks | **55** | + +## Codebase (run to refresh) + +```bash +# Rust LOC (crates + tests) +find crates tests -name '*.rs' | xargs wc -l | tail -1 + +# MCP tool count (must match server assertion) +rg 'name: "' crates/vestige-mcp/src/server.rs | wc -l + +# Tests +cargo test --workspace --no-fail-fast 2>&1 | tail -3 +``` + +| Metric | Current value | Notes | +|--------|---------------|-------| +| Rust LOC | **~86,000** | `crates/` + `tests/` `.rs` files | +| MCP tools | **25** | Verified in `server.rs` (`tools.len() == 25`) | +| Cognitive modules | **30** | Per README architecture | +| Rust tests | **1,200+** | CHANGELOG v2.1.0: 1,229 passing; re-run before major launch | +| Dashboard tests | **171** | Vitest in `apps/dashboard` | +| Release binary | **~22MB** | Single binary, embedded SvelteKit dashboard | +| Embedding model | Nomic Embed Text v1.5 (~130MB first-run download) | + +## Install (canonical — no `sudo mv`) + +The npm package registers global bins via `postinstall`. **Do not** tell users to `sudo mv vestige-mcp` unless manual binary install failed. + +```bash +npm install -g vestige-mcp-server@latest +vestige health +claude mcp add vestige vestige-mcp -s user +``` + +If `vestige-mcp` is not on PATH after install: + +```bash +npm prefix -g # e.g. /usr/local or ~/.npm-global +# Ensure that path/bin is in your shell PATH +``` + +Manual binary placement (optional): + +```bash +vestige update --install-dir /usr/local/bin +``` + +## Messaging guardrails + +- Lead Wave A with **Receipt Lock** (agents overclaim "tests passed"). +- Close Wave B with **cognitive memory** (FSRS-6, dreaming, 3D dashboard). +- Never: "revolutionary", "game-changer", "AI-powered", competitor bashing. +- Always: honest neuroscience (faithful implementations vs engineering heuristics). + +## North-star metrics + +Track weekly (see `docs/marketing/metrics-tracker.md`): + +1. **npm downloads** (`npm view vestige-mcp-server` / npmjs.com stats) +2. **GitHub stars delta** +3. **Inbound issues/DMs** mentioning install +4. **Referral source** (HN, Reddit, X, registry) diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 0000000..9a78a3e --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,82 @@ +# Vestige vs Mem0 vs RAG vs Native AI Memory + +Canonical comparison for launch posts and arguments. Grounded in [SCIENCE.md](SCIENCE.md) and [LAUNCH_STATS.md](LAUNCH_STATS.md). + +## One-line thesis + +**RAG is retrieval. Native memory is a black box. Mem0 is a strong cloud memory API. Vestige is a local cognitive system that forgets, strengthens, dreams, and can block unverified agent claims.** + +## Comparison table + +| Capability | RAG / vector DB | Native AI memory (Claude, ChatGPT) | Mem0 | Vestige | +|------------|-----------------|-------------------------------------|------|---------| +| **Runs local** | Often cloud embeddings | Cloud only | Cloud API (local option limited) | **100% local** default | +| **You own the data** | Your infra | Vendor | Vendor / API | **SQLite on your disk** | +| **Forgetting curve** | None — equal weight forever | Opaque | Categories + metadata | **FSRS-6** power-law decay | +| **Duplicate handling** | Manual | Opaque | Some dedup | **Prediction Error Gating** on ingest | +| **Retrieval strengthens memory** | No | Unknown | Partial | **Testing Effect** on every search | +| **Offline consolidation** | No | No | No | **`dream`** — replay + connect | +| **Contradiction awareness** | Returns both chunks | No | Some products | **`deep_reference` / `contradictions`** | +| **Active suppression** | Delete only | No | Delete | **`suppress`** — inhibited, not erased | +| **Agent overclaim guard** | No | No | No | **Receipt Lock** (optional Sanhedrin hooks) | +| **Visualization** | None | None | Dashboard (cloud) | **3D graph** + WebSocket events | +| **Protocol** | Custom | Proprietary | API + MCP | **MCP** (25 tools) | +| **License** | Varies | Proprietary | Apache / commercial | **AGPL-3.0** (local use = free) | + +## When to use what + +### Use RAG when + +- You have a fixed document corpus (PDFs, wiki, codebase index). +- You need one-shot Q&A over static content. +- You do not need memory lifecycle or session continuity. + +### Use Mem0 when + +- You want a hosted memory API with minimal setup. +- Team sync and cloud dashboards are acceptable. +- You do not need FSRS decay or local-only air-gapped deploy. + +### Use native Claude/ChatGPT memory when + +- Casual personal context is enough. +- You do not need inspectable storage, decay curves, or contradiction tooling. + +### Use Vestige when + +- You run **Claude Code, Cursor, Codex, or any MCP client** daily. +- Context bloat from "remember everything" hurts retrieval quality. +- **Contradicting memories** have burned you (config changed, lib upgraded). +- You want **Receipt Lock** so agents cannot fake "tests passed." +- **Privacy / air-gapped** matters — embeddings run locally via ONNX. + +## Honest limitations (Vestige) + +- **AGPL-3.0**: hosting as a service without source disclosure is not allowed. +- **First-run download**: ~130MB embedding model (then offline). +- **Receipt Lock** requires optional Claude Code Cognitive Sandwich hooks + a verifier endpoint for Sanhedrin. +- **Neuroscience modules** mix faithful implementations and engineering heuristics — see [SCIENCE.md](SCIENCE.md) for citations vs approximations. +- **Solo project**: no enterprise SLA; GitHub issues are the support channel. + +## Receipt Lock (Vestige-only) + +Coding agents often end sessions with: + +> "All tests passed. Build is green. Ready to merge." + +Receipt Lock checks those **operational claims** against structured command receipts from the transcript. No matching successful receipt → claim blocked, local veto receipt written under `~/.vestige/sanhedrin/`. + +```bash +vestige sandwich install --enable-sanhedrin +``` + +Details: [README Receipt Lock section](../README.md#receipt-lock). + +## Install + +```bash +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user +``` + +Full stats: [LAUNCH_STATS.md](LAUNCH_STATS.md) · Repo: https://github.com/samvallad33/vestige diff --git a/docs/launch/blog-post.md b/docs/launch/blog-post.md index 886bb5a..08178ba 100644 --- a/docs/launch/blog-post.md +++ b/docs/launch/blog-post.md @@ -6,7 +6,7 @@ Every conversation starts from zero. You explain your project structure, your pr Vestige is an open-source Rust MCP server that gives AI agents persistent memory modeled on real neuroscience. Not metaphorical neuroscience. Actual published algorithms from Ebbinghaus (1885), Collins & Loftus (1975), Bjork & Bjork (1992), Frey & Morris (1997), and the FSRS-6 spaced repetition scheduler trained on 700 million Anki reviews. -77,840+ lines of Rust. 29 cognitive modules. 734 tests. Single binary deployment with an embedded SvelteKit dashboard. AGPL-3.0 licensed. +~86,000 lines of Rust. 30 cognitive modules. 1,200+ tests. v2.1.23 adds Receipt Lock for unverified agent claims. Single 22MB binary with embedded SvelteKit dashboard. AGPL-3.0 licensed. Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md). Here is how we built it. diff --git a/docs/launch/demo-script.md b/docs/launch/demo-script.md index 4740dc4..162efb8 100644 --- a/docs/launch/demo-script.md +++ b/docs/launch/demo-script.md @@ -1,4 +1,6 @@ -# Vestige v2.0 "Cognitive Leap" — MCP Dev Summit NYC Demo Script +# Vestige v2.1.23 — Demo Script (Conference + Launch Video) + +> Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md) · Wave A hook: [receipt-lock.md](receipt-lock.md) **Event:** MCP Dev Summit NYC, April 1-3, 2026 **Presenter:** Sam Valladares @@ -15,7 +17,7 @@ - [ ] Phone hotspot configured as backup (embedding model already cached = no network needed) ### Software -- [ ] Vestige v2.0 binary installed: `vestige-mcp --version` shows `2.0.0` +- [ ] Vestige binary installed: `vestige-mcp --version` shows `2.1.23` (or latest) - [ ] Claude Code installed and authenticated - [ ] Terminal font size: 18pt minimum (audience readability) - [ ] Browser zoom: 150% for dashboard views @@ -205,7 +207,7 @@ claude mcp add vestige vestige-mcp -s user ### [2:50-3:00] Close -> Vestige v2.0, "Cognitive Leap." Open source, AGPL-3.0. The repo is `samvallad33/vestige`. Come talk to me if you want to see the neuroscience under the hood. +> Vestige v2.1.23. Open source, AGPL-3.0. The repo is `samvallad33/vestige`. Come talk to me if you want to see the neuroscience under the hood. --- @@ -391,9 +393,9 @@ vestige-mcp --version # One command to install ``` -> This is what I've been building for the past three months. I'm one person, I'm twenty-one years old, and I believe this is how AI memory should work — grounded in real science, running locally, open source. +> This is what I've been building. I'm one person, I'm twenty-two years old, and I believe this is how AI memory should work — grounded in real science, running locally, open source. > -> Vestige v2.0, "Cognitive Leap." The repo is `github.com/samvallad33/vestige`. The dashboard is running at `localhost:3927`. I'll be around all three days — come find me if you want to talk about FSRS, or synaptic tagging, or why I think every AI assistant on the planet should have a forgetting curve. +> Vestige v2.1.23. The repo is `github.com/samvallad33/vestige`. The dashboard is running at `localhost:3927`. I'll be around all three days — come find me if you want to talk about FSRS, or synaptic tagging, or why I think every AI assistant on the planet should have a forgetting curve. > > Thank you. diff --git a/docs/launch/receipt-lock.md b/docs/launch/receipt-lock.md new file mode 100644 index 0000000..a27df5a --- /dev/null +++ b/docs/launch/receipt-lock.md @@ -0,0 +1,167 @@ +# Wave A Launch — Receipt Lock (v2.1.23) + +Primary viral hook. Post **before** the memory/science Show HN wave. Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md). + +--- + +## Hacker News — Show HN + +### Title (≤80 chars) + +``` +Show HN: Vestige – blocks coding agents from claiming "tests passed" without receipts +``` + +### First comment (body) + +``` +Hi HN, + +Your coding agent probably ends sessions with something like "all tests passed" or +"the build is green." I kept trusting that — until it wasn't true. + +I built Receipt Lock in Vestige (an MCP memory server I maintain). Before operational +claims become part of the final answer, Vestige checks them against structured +command receipts from the current transcript. No matching successful receipt → the +claim can be blocked and a local veto receipt is written (JSON + HTML under +~/.vestige/sanhedrin/). + +**What it is:** Optional Claude Code Cognitive Sandwich hooks + local MCP server. +Not cloud. Not "trust me bro" logging — inspectable receipts on disk. + +**Install (memory server — required base):** +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user + +**Enable Receipt Lock (optional):** +vestige sandwich install --enable-sanhedrin + +Sanhedrin verifier can point at any OpenAI-compatible endpoint (Ollama, MLX, hosted API). + +**And it also does real memory:** FSRS-6 spaced repetition, prediction error gating, +memory dreaming, 3D dashboard at localhost:3927. ~86K LOC Rust, 25 MCP tools, 1,200+ +tests, 22MB binary. 100% local after first embedding download. + +I'm 22, solo, AGPL-3.0. Repo: https://github.com/samvallad33/vestige +Comparison: https://github.com/samvallad33/vestige/blob/main/docs/comparison.md + +Happy to discuss false positive tuning, Sanhedrin presets, or why receipts beat vibes. +``` + +--- + +## r/ExperiencedDevs + +### Title + +``` +My coding agent kept saying "tests passed" when they hadn't. I added a receipt check before the summary ships. +``` + +### Body + +```markdown +**TL;DR:** Vestige Receipt Lock checks operational claims ("tests passed", "build green", "lint clean") against structured command receipts from the transcript. No receipt → block + local veto artifact. + +**The failure mode:** Agent runs partial checks, or hallucinates a green ending. You merge. CI breaks. You've seen this. + +**The fix:** Optional hooks (`vestige sandwich install --enable-sanhedrin`) + MCP memory server. When the model tries to assert verification without evidence, Vestige can veto and write `~/.vestige/sanhedrin/latest.html` so you can inspect what happened. + +**Not a replacement for CI.** It's a last-mile guard on *agent-authored* summaries in Claude Code. + +**Stack:** Rust, local, MCP. Same project also does FSRS-6 cognitive memory (decay, dreaming, contradiction tools) — I'll post that angle separately if people want the science side. + +```bash +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user +vestige sandwich install --enable-sanhedrin +``` + +GitHub: https://github.com/samvallad33/vestige + +What false positives are you seeing with agent verification claims? Curious if this matches your workflow. +``` + +--- + +## r/programming + +### Title + +``` +Open-source guard: coding agents can't claim "tests passed" without command receipts (local MCP, Rust) +``` + +### Body — use r/ExperiencedDevs body; add: + +```markdown +License: AGPL-3.0. v2.1.23. Stats: ~86K LOC, 25 tools, 22MB binary. +``` + +--- + +## X / Twitter thread (8 posts) + +1. Your coding agent ends with "tests passed." Did it run tests? Or did it summarize hope? + +2. I ship Receipt Lock in Vestige — checks operational claims against command receipts from the transcript. + +3. No matching successful receipt → claim blocked. Local veto receipt: `~/.vestige/sanhedrin/latest.html` + +4. Optional hooks. Local MCP server. Not cloud analytics. + +5. ```bash + npm i -g vestige-mcp-server@latest + claude mcp add vestige vestige-mcp -s user + vestige sandwich install --enable-sanhedrin + ``` + +6. Same binary also does FSRS-6 memory — decay, dreaming, 3D brain viz. Thread on that tomorrow. + +7. 22yo solo dev. AGPL. https://github.com/samvallad33/vestige + +8. What's the worst "green build" lie your agent told you? Reply — building the FAQ from real stories. + +--- + +## Lobste.rs + +### Title + +``` +Vestige Receipt Lock: local MCP guard against unverified "tests passed" agent claims +``` + +### Tags + +`rust` `programming` `security` + +### Body + +Use HN first comment (shorter). Link comparison.md. + +--- + +## Engagement playbook (Wave A) + +| Window | Action | +|--------|--------| +| 0–3h | Reply every comment within 30 min | +| Tone | Technical, humble, no "revolutionary" | +| Competitors | Acknowledge Mem0/Cursor memory; don't bash | +| CTA | Install + link comparison.md | +| Next | Schedule Wave B 48h after Wave A peaks | + +### DO NOT + +- "Game-changer" / "AI-powered" / "paradigm shift" +- Disparage Mem0 or Claude native memory +- Promise Receipt Lock replaces CI + +--- + +## Timing + +- **HN / Lobsters:** Tuesday or Wednesday, 8–10 AM US Eastern +- **Reddit:** Same day, +1–2h after HN +- **X:** Pin thread during HN peak diff --git a/docs/launch/reddit-cross-reference.md b/docs/launch/reddit-cross-reference.md index eae7aaf..d03f5f6 100644 --- a/docs/launch/reddit-cross-reference.md +++ b/docs/launch/reddit-cross-reference.md @@ -1,4 +1,6 @@ -# Reddit Launch Posts — cross_reference Tool +# Reddit Launch Posts — cross_reference / deep_reference (v2.1.23) + +> Canonical install: [LAUNCH_STATS.md](../LAUNCH_STATS.md) — **no `sudo mv`**; use `npm install -g vestige-mcp-server@latest` ## Post 1: r/ClaudeAI (Primary) @@ -78,8 +80,8 @@ Memory systems need to be SMARTER, not just bigger. That's what Vestige does — - **cross_reference** — the new tool that catches contradictions before they become wrong answers ### Stats: -- 22 MCP tools -- 746 tests, 0 failures +- 25 MCP tools +- 1,200+ tests - Zero `unsafe` code - Clean security audit (0 findings — AgentAudit verified) - Single 22MB Rust binary — no Docker, no PostgreSQL, no cloud @@ -87,9 +89,8 @@ Memory systems need to be SMARTER, not just bigger. That's what Vestige does — ### Install (30 seconds): ```bash -# macOS Apple Silicon -npm install -g vestige-mcp-server -sudo mv vestige-mcp /usr/local/bin/ +npm install -g vestige-mcp-server@latest +vestige health claude mcp add vestige vestige-mcp -s user ``` @@ -162,12 +163,12 @@ The AI sees the conflict. Picks the right one. Every time. **100% local. Your data never leaves your machine.** ```bash -npm install -g vestige-mcp-server -sudo mv vestige-mcp /usr/local/bin/ +npm install -g vestige-mcp-server@latest +vestige health claude mcp add vestige vestige-mcp -s user ``` -746 tests. Zero unsafe code. Clean security audit. AGPL-3.0. +1,200+ tests. Zero unsafe code. AGPL-3.0. GitHub: https://github.com/samvallad33/vestige @@ -175,7 +176,7 @@ GitHub: https://github.com/samvallad33/vestige ## Post 3: r/rust (Optional, technical audience) -**Title:** `I built a 22MB Rust binary that gives AI agents a brain — FSRS-6, 29 cognitive modules, 3D dashboard, and a new contradiction detection tool. 746 tests, zero unsafe.` +**Title:** `I built a 22MB Rust binary that gives AI agents a brain — FSRS-6, 30 cognitive modules, Receipt Lock, 25 MCP tools. ~86K LOC, zero unsafe.` --- @@ -188,7 +189,7 @@ The latest addition: `cross_reference` — pairwise contradiction detection acro - No runtime, no GC pauses during real-time search - `tokio::sync::Mutex` for the cognitive engine, `std::sync::Mutex` for SQLite reader/writer split - Zero `unsafe` blocks in the entire codebase -- `cargo test` runs 746 tests in 11 seconds +- `cargo test` runs 1,200+ tests across the workspace **Architecture:** ``` @@ -213,7 +214,7 @@ SQLite WAL + FTS5 + USearch HNSW Clean security audit. Parameterized SQL everywhere. CSP headers on the dashboard. Constant-time auth comparison (`subtle::ConstantTimeEq`). File permissions 0o600/0o700. GitHub: https://github.com/samvallad33/vestige -AGPL-3.0 | 746 tests | 79K+ LOC +AGPL-3.0 | 1,200+ tests | ~86K LOC --- diff --git a/docs/launch/show-hn.md b/docs/launch/show-hn.md index 8cc5a95..adc58ab 100644 --- a/docs/launch/show-hn.md +++ b/docs/launch/show-hn.md @@ -1,4 +1,7 @@ -# Vestige v2.0 Launch — Show HN + Cross-Posts +# Vestige v2.1.23 Launch — Show HN + Cross-Posts (Wave B: Memory) + +> **Wave A (Receipt Lock)** posts live in [receipt-lock.md](receipt-lock.md). Run Wave A first. +> Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md) --- @@ -7,7 +10,7 @@ ### Title (76 chars) ``` -Show HN: Vestige – FSRS-6 spaced repetition as long-term memory for AI agents +Show HN: Vestige v2.1.23 – FSRS-6 memory for AI agents + local Receipt Lock ``` ### Body (first comment) @@ -57,7 +60,7 @@ retrieval. Written in Rust, 100% local, single 22MB binary. discover hidden connections and synthesize insights. Inspired by hippocampal replay during sleep. -**v2.0 adds:** +**Dashboard (since v2.0, still core):** - 3D neural visualization dashboard (SvelteKit + Three.js) — watch memories pulse when accessed, burst particles on creation, golden flash lines when @@ -74,7 +77,10 @@ retrieval. Written in Rust, 100% local, single 22MB binary. embedded via Rust's `include_dir!` macro. No Docker, no Node runtime, no external services. -**Numbers:** 77,840 lines of Rust, 734 tests, 29 cognitive modules, 21 MCP +**v2.1.23 adds Receipt Lock:** optional hooks that block operational claims like +"tests passed" unless matching command receipts exist in the transcript. + +**Numbers:** ~86,000 lines of Rust, 1,200+ tests, 30 cognitive modules, 25 MCP tools, search under 50ms for 1000 memories (SQLite FTS5 + USearch HNSW). **What it is NOT:** This is not RAG. RAG treats memory as a static database — @@ -87,7 +93,7 @@ The embedding model (Nomic Embed Text v1.5) runs locally via ONNX. After the first-run model download (~130MB), there are zero network requests. No telemetry, no analytics, no phoning home. -I've been using this daily for 2 months and the experience is genuinely different. +I've been using this daily and the experience is genuinely different. Claude remembers my coding patterns, my architectural decisions, my preferences. New sessions start with context instead of a blank slate. @@ -275,12 +281,12 @@ surprising and useful. ### r/rust -**Title:** `Vestige v2.0 — 77K LOC Rust memory system with FSRS-6, HNSW, Axum WebSockets, and an embedded SvelteKit dashboard in a 22MB binary` +**Title:** `Vestige v2.1.23 — ~86K LOC Rust memory system with FSRS-6, Receipt Lock, and a 22MB binary` **Body:** ```markdown -I've been building Vestige for the past few months and just shipped v2.0. It's +I've been building Vestige and just shipped v2.1.23. It's a cognitive memory system for AI agents that implements neuroscience-backed memory algorithms in pure Rust. @@ -314,7 +320,7 @@ memory algorithms in pure Rust. - **Release profile**: `lto = true`, `codegen-units = 1`, `opt-level = "z"`, `strip = true` gets the binary down to 22MB including embedded assets. -- **734 tests**: 352 core + 378 mcp + 4 doctests. Zero warnings. +- **1,200+ tests** across workspace. Zero warnings on release gates. **Architecture:** @@ -357,7 +363,7 @@ Happy to discuss any of the Rust architecture decisions. ### r/ClaudeAI -**Title:** `Vestige v2.0 "Cognitive Leap" — give Claude real long-term memory with neuroscience-backed forgetting, a 3D dashboard, and 21 MCP tools` +**Title:** `Vestige v2.1.23 — give Claude real long-term memory (FSRS-6) + optional Receipt Lock for fake "tests passed" claims` **Body:** @@ -385,7 +391,7 @@ locally on your machine. strength model, testing effect, synaptic tagging, spreading activation, context-dependent retrieval, memory dreaming. -**v2.0 new features:** +**Highlights:** - **3D Memory Dashboard** at localhost:3927/dashboard — watch Claude's mind in real-time. Memories pulse when accessed, burst particles on creation, golden @@ -401,7 +407,8 @@ locally on your machine. **Setup (2 minutes):** ```bash -npm install -g vestige-mcp-server +npm install -g vestige-mcp-server@latest +vestige health claude mcp add vestige vestige-mcp -s user ``` @@ -417,7 +424,7 @@ on Project X ended with a tricky race condition in the WebSocket handler. It's the difference between talking to someone with amnesia vs. someone who actually knows you. -21 MCP tools. 77,840 lines of Rust. 734 tests. Works with Claude Code, Claude +25 MCP tools. ~86,000 lines of Rust. 1,200+ tests. Works with Claude Code, Claude Desktop, Cursor, VS Code Copilot, JetBrains, Windsurf, and Xcode. Source: https://github.com/samvallad33/vestige @@ -429,7 +436,7 @@ Happy to answer questions or help with setup. ### r/LocalLLaMA -**Title:** `Vestige v2.0 — local-first AI memory server with FSRS-6 spaced repetition, ONNX embeddings, and zero cloud dependency (77K LOC Rust, 22MB binary)` +**Title:** `Vestige v2.1.23 — local-first AI memory with FSRS-6, Receipt Lock, zero cloud (~86K LOC Rust, 22MB binary)` **Body:** @@ -482,7 +489,7 @@ algorithms: - 3D force-directed memory graph with real-time WebSocket events - HyDE query expansion (template-based hypothetical document embeddings) - FSRS decay visualization with retention curves -- 734 tests, 29 cognitive modules, 21 tools +- 1,200+ tests, 30 cognitive modules, 25 tools - fastembed 5.11 with feature flags for Nomic v2 MoE + Qwen3 reranker **Performance:** @@ -535,8 +542,7 @@ This is a solo project — feedback, issues, and contributions are very welcome. implementations, some are engineering heuristics inspired by research) 3. 100% local, zero cloud — this is a feature, not a limitation 4. The 3D dashboard is a genuine exploration tool, not just eye candy -5. FSRS-6 is the differentiator — no other AI memory system uses real spaced - repetition +5. FSRS-6 + Receipt Lock — spaced repetition memory and optional agent claim verification ### What NOT to Say diff --git a/docs/marketing/LAUNCH_NOW.md b/docs/marketing/LAUNCH_NOW.md new file mode 100644 index 0000000..7e2b887 --- /dev/null +++ b/docs/marketing/LAUNCH_NOW.md @@ -0,0 +1,35 @@ +# LAUNCH NOW — Wave A (Receipt Lock) + +**Date started:** 2026-06-02 +**Copy-paste from:** [ready-to-post/](ready-to-post/) + +## 5-minute sequence + +1. Run preflight: + ```bash + ./scripts/marketing/preflight.sh + ``` + +2. **Hacker News** → https://news.ycombinator.com/submit + - URL: `https://github.com/samvallad33/vestige` + - Title: paste `ready-to-post/hn-title.txt` + - Post link, then immediately paste `ready-to-post/hn-first-comment.txt` as first comment + +3. **X** — paste `ready-to-post/x-thread.txt` (one tweet per numbered block) + +4. **r/ExperiencedDevs** — title from `reddit-experienceddevs-title.txt`, body from `reddit-experienceddevs.md` + +5. **r/programming** — same body + line: `License: AGPL-3.0. v2.1.23. ~86K LOC, 25 tools, 22MB binary.` + +6. Log URLs in [metrics-tracker.md](metrics-tracker.md) + +## After posting (30 min SLA on comments) + +```bash +vestige ingest "Wave A posted YYYY-MM-DD on HN Reddit X. Hook: agent fake tests passed. Log URLs in metrics-tracker." \ + --data-dir ~/.vestige-marketing --tags marketing,wave-a,vestige-launch +``` + +## 48h later → Wave B + +[wave-b-launch.md](wave-b-launch.md) + [show-hn.md](../launch/show-hn.md) memory angle diff --git a/docs/marketing/README.md b/docs/marketing/README.md new file mode 100644 index 0000000..75acb1e --- /dev/null +++ b/docs/marketing/README.md @@ -0,0 +1,33 @@ +# Vestige Marketing Kit + +Everything needed to run the dual-wave launch and weekly growth loop. + +## Start here + +**Posting today?** → [LAUNCH_NOW.md](LAUNCH_NOW.md) (copy-paste from [ready-to-post/](ready-to-post/)) + +1. [LAUNCH_STATS.md](../LAUNCH_STATS.md) — canonical version, stats, install +2. [comparison.md](../comparison.md) — vs Mem0 / RAG / native memory +3. [launch/receipt-lock.md](../launch/receipt-lock.md) — **Wave A** copy (post first) +4. [launch/show-hn.md](../launch/show-hn.md) — **Wave B** copy +5. [wave-a-launch.md](wave-a-launch.md) / [wave-b-launch.md](wave-b-launch.md) — execution checklists + +## Assets + +| Path | Purpose | +|------|---------| +| [website/index.html](../website/index.html) | GitHub Pages landing | +| [assets/CAPTURE.md](assets/CAPTURE.md) | GIF/video capture | +| [demo-video-storyboard.md](demo-video-storyboard.md) | 60–90s video beats | + +## Ongoing + +| Path | Purpose | +|------|---------| +| [growth-engine/](growth-engine/) | Vestige-powered marketing agent setup | +| [metrics-tracker.md](metrics-tracker.md) | Weekly npm / stars / hooks | +| [mcp-registries.md](mcp-registries.md) | Directory submission packet | + +## Deploy landing page + +Push to `main` → GitHub Actions workflow `.github/workflows/pages.yml` publishes `docs/website/` to **https://samvallad33.github.io/vestige/** (enable Pages: Settings → Pages → GitHub Actions). diff --git a/docs/marketing/assets/.gitkeep b/docs/marketing/assets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/marketing/assets/CAPTURE.md b/docs/marketing/assets/CAPTURE.md new file mode 100644 index 0000000..7f45315 --- /dev/null +++ b/docs/marketing/assets/CAPTURE.md @@ -0,0 +1,40 @@ +# Demo GIF / Video Capture Guide + +Record these on a machine with Vestige v2.1.23 installed and ~20 pre-loaded memories (see `docs/launch/demo-script.md` pre-load section). + +## Prerequisites + +```bash +npm install -g vestige-mcp-server@latest +vestige health +claude mcp add vestige vestige-mcp -s user +open http://localhost:3927/dashboard +``` + +## Assets to produce + +| File | Duration | What to show | +|------|----------|----------------| +| `receipt-lock.gif` | 8–12s loop | Agent claims "tests passed" → Sanhedrin veto → `~/.vestige/sanhedrin/latest.html` receipt | +| `dashboard-dream.gif` | 10–15s loop | Graph view → trigger dream in Claude → purple dream mode, golden connection lines | +| `memory-born.gif` | 5–8s | Feed tab: `MemoryCreated` WebSocket event + new node burst on graph | +| `demo-full.mp4` | 60–90s | Full script: `docs/launch/demo-script.md` Version 2 (3-minute cut to 90s) | + +## macOS capture (recommended) + +```bash +# Screen recording → convert to GIF (install: brew install ffmpeg) +ffmpeg -f avfoundation -i "1" -t 12 -vf "fps=10,scale=1280:-1" -y /tmp/vestige-rec.mov +ffmpeg -i /tmp/vestige-rec.mov -vf "fps=8,scale=960:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 docs/marketing/assets/dashboard-dream.gif +``` + +## Static fallback + +If GIFs are not ready for launch, export one PNG from the dashboard graph view: + +```bash +# Browser screenshot → save as: +docs/marketing/assets/dashboard-static.png +``` + +Commit GIFs when ready; README and landing page reference these paths. diff --git a/docs/marketing/assets/dashboard-placeholder.svg b/docs/marketing/assets/dashboard-placeholder.svg new file mode 100644 index 0000000..4a50448 --- /dev/null +++ b/docs/marketing/assets/dashboard-placeholder.svg @@ -0,0 +1,10 @@ + + + Vestige 3D Memory Dashboard + Run vestige dashboard  capture GIF per docs/marketing/assets/CAPTURE.md + + + + + + diff --git a/docs/marketing/demo-video-storyboard.md b/docs/marketing/demo-video-storyboard.md new file mode 100644 index 0000000..98f2942 --- /dev/null +++ b/docs/marketing/demo-video-storyboard.md @@ -0,0 +1,32 @@ +# Demo Video Storyboard (60–90s) + +For `docs/marketing/assets/demo-full.mp4` and GIF exports. Full script: [demo-script.md](../../launch/demo-script.md) Version 2. + +## Beat sheet + +| Time | Visual | Audio / text overlay | +|------|--------|----------------------| +| 0:00–0:08 | 3D dashboard graph, nodes pulsing | "Your AI forgets everything between sessions." | +| 0:08–0:18 | Claude Code: "Remember I prefer TypeScript" → Feed: MemoryCreated | "Vestige stores it with prediction error gating — not a dumb bucket." | +| 0:18–0:30 | New session: "What are my language preferences?" → correct answer | "New session. Same brain." | +| 0:30–0:45 | Search → graph nodes pulse blue (spreading activation) | "Search runs a 7-stage cognitive pipeline." | +| 0:45–0:58 | Dream mode: purple wash, golden edges | "Dream consolidation finds connections you never typed." | +| 0:58–1:15 | Terminal: agent says "tests passed" → veto / sanhedrin HTML | "Receipt Lock: no receipt, no claim." | +| 1:15–1:25 | Terminal: install commands | `npm install -g vestige-mcp-server@latest` | +| 1:25–1:30 | Logo / github.com/samvallad33/vestige | "v2.1.23 · local · AGPL" | + +## Export targets + +| Asset | Path | +|-------|------| +| Full video | `docs/marketing/assets/demo-full.mp4` | +| Dashboard loop | `docs/marketing/assets/dashboard-dream.gif` | +| Receipt Lock loop | `docs/marketing/assets/receipt-lock.gif` | +| Memory create | `docs/marketing/assets/memory-born.gif` | + +Capture commands: [assets/CAPTURE.md](assets/CAPTURE.md) + +## Wave usage + +- **Wave A:** Ship `receipt-lock.gif` + beats 0:58–1:15 first +- **Wave B:** Ship `dashboard-dream.gif` + full `demo-full.mp4` diff --git a/docs/marketing/growth-engine/MARKETING-CLAUDE.md b/docs/marketing/growth-engine/MARKETING-CLAUDE.md new file mode 100644 index 0000000..ef3aae0 --- /dev/null +++ b/docs/marketing/growth-engine/MARKETING-CLAUDE.md @@ -0,0 +1,69 @@ +# Vestige Marketing Agent Protocol + +You are the marketing operator for **Vestige** (v2.1.23). You have access to a **dedicated** Vestige MCP instance (`vestige-marketing` / `VESTIGE_DATA_DIR=~/.vestige-marketing`). Never confuse this with the user's dev memory. + +## Product facts (do not invent stats) + +Read [docs/LAUNCH_STATS.md](../../LAUNCH_STATS.md) before drafting. Current anchors: + +- ~86K LOC Rust, 25 MCP tools, 30 cognitive modules, 1,200+ tests, 22MB binary +- Install: `npm install -g vestige-mcp-server@latest` + `claude mcp add vestige vestige-mcp -s user` +- Wave A hook: **Receipt Lock** — blocks "tests passed" without command receipts +- Wave B product: **FSRS-6 cognitive memory**, dreaming, 3D dashboard +- Comparison: [docs/comparison.md](../../comparison.md) +- Author: Sam Valladares, 22, solo, AGPL-3.0 + +## Session start + +1. `session_context` with query: `vestige marketing launch hooks objections` +2. `deep_reference` if drafting factual claims about features or competitors +3. `contradictions` if messaging might conflict with prior brand guidelines + +## Voice + +- Technical, humble, specific — never "revolutionary", "game-changer", "AI-powered" +- Lead with **pain** (agent amnesia, fake green builds, contradicting memories) +- Reveal **tool** second +- Acknowledge Mem0, native Claude memory, RAG honestly — do not bash +- Neuroscience: cite real papers; admit heuristics where approximate + +## On user feedback + +- Winning hook / post → `memory` promote on that memory +- Flopped angle → `suppress` (not delete) +- New objection → `smart_ingest` with tags `marketing, objection` +- User correction → `smart_ingest` + demote wrong memory if needed + +## Weekly deliverables + +When asked for "weekly content": + +1. **One long-form** (800–1200 words): expand top objection OR one cognitive module OR Receipt Lock story +2. **3–5 short posts** (X/LinkedIn): each ≤280 chars or ≤2 short paragraphs +3. **One Reddit draft** (technical, humble title — pain first) +4. **Metrics summary** paragraph for ingest after user fills tracker + +## Channels (user posts manually) + +You draft only. User sends all posts and DMs to avoid bans and keep authenticity. + +| Channel | Style | +|---------|-------| +| HN | Show HN title ≤80 chars; first comment = full body; science-first | +| Reddit | Personal story + JSON output + install block; no "introducing my startup" | +| X | 8–12 tweet thread; hook tweet must stand alone | +| LinkedIn | Professional, link comparison.md | + +## End of week + +``` +dream with focus on marketing memories tagged vestige-launch from the last 7 days. +Return: top 3 hooks to promote, top 2 to suppress, one recommended post for next week. +``` + +## Hard rules + +- Do not claim Vestige replaces CI/CD or enterprise memory suites +- Do not fabricate download numbers — use metrics-tracker.md only +- Do not tell users `sudo mv` for install unless manual binary path failed +- Always include GitHub link: https://github.com/samvallad33/vestige diff --git a/docs/marketing/growth-engine/README.md b/docs/marketing/growth-engine/README.md new file mode 100644 index 0000000..b2b4af5 --- /dev/null +++ b/docs/marketing/growth-engine/README.md @@ -0,0 +1,84 @@ +# Vestige Marketing Growth Engine + +Repeatable weekly loop: Vestige remembers what worked, Claude Code drafts what’s next, you approve and post manually. + +## One-time setup + +### 1. Dedicated marketing memory store + +```bash +mkdir -p ~/.vestige-marketing +``` + +Add a **second** MCP server entry (do not mix with dev memory): + +```bash +claude mcp add vestige-marketing vestige-mcp -s user \ + --env VESTIGE_DATA_DIR=$HOME/.vestige-marketing +``` + +If your client does not support env on `mcp add`, use a wrapper script: + +```bash +# ~/bin/vestige-mcp-marketing +#!/bin/bash +export VESTIGE_DATA_DIR="$HOME/.vestige-marketing" +exec vestige-mcp "$@" +``` + +### 2. Copy marketing agent instructions + +```bash +cp docs/marketing/growth-engine/MARKETING-CLAUDE.md ~/vestige-marketing-CLAUDE.md +``` + +In Claude Code for marketing sessions, include that file (or paste into project instructions). + +### 3. Seed baseline memories + +Open Claude Code with `vestige-marketing` connected and run: + +``` +Read docs/LAUNCH_STATS.md, docs/comparison.md, and docs/marketing/metrics-tracker.md. +smart_ingest each as separate marketing baseline memories with tags: marketing, baseline, vestige-launch. +``` + +## Weekly loop (≈2 hours) + +| Step | Who | Action | +|------|-----|--------| +| Mon AM | You | Fill `metrics-tracker.md` row | +| Mon | Agent | `session_context` with query "vestige marketing launch" | +| Mon | Agent | Draft 1 long-form + 3–5 shorts from last week's `promote`d hooks | +| Mon | You | Edit and post manually (HN/Reddit/X/LinkedIn) | +| Fri | You | Log engagement URLs + numbers | +| Fri | Agent | `smart_ingest` weekly metrics + objections | +| Fri | Agent | `dream` on tag `vestige-launch` for next week's angles | + +## Tool cheat sheet + +| Goal | Tool | +|------|------| +| Load brand voice + past wins | `session_context` | +| Save post results | `smart_ingest` | +| Recall winning hooks | `search` / `deep_reference` | +| Retire dead angles | `suppress` | +| Boost viral hook | `memory` action=promote | +| Weekly strategy | `dream` | + +## Dogfood story (meta-content) + +> "I use Vestige to market Vestige — marketing memories live in a separate data dir, FSRS promotes hooks that converted, suppress kills angles that flopped." + +Post this on X after Week 2 if metrics show engagement. + +## Files + +| File | Purpose | +|------|---------| +| [MARKETING-CLAUDE.md](MARKETING-CLAUDE.md) | Agent protocol | +| [../metrics-tracker.md](../metrics-tracker.md) | Weekly numbers | +| [../wave-a-launch.md](../wave-a-launch.md) | Receipt Lock execution | +| [../wave-b-launch.md](../wave-b-launch.md) | Memory wave execution | +| [../../launch/receipt-lock.md](../../launch/receipt-lock.md) | Wave A copy | +| [../../comparison.md](../../comparison.md) | Argument anchor | diff --git a/docs/marketing/mcp-registries.md b/docs/marketing/mcp-registries.md new file mode 100644 index 0000000..aaa7a98 --- /dev/null +++ b/docs/marketing/mcp-registries.md @@ -0,0 +1,57 @@ +# MCP Registry & Directory Submissions + +Passive install channel — update listings whenever v2.1.x ships. Check off as you submit. + +## Submission packet (reuse everywhere) + +| Field | Value | +|-------|-------| +| Name | Vestige | +| Slug | `io.github.samvallad33/vestige` (npm `mcpName`) | +| Description | Local cognitive memory for MCP agents — FSRS-6 spaced repetition, prediction error gating, memory dreaming, 3D dashboard, optional Receipt Lock for agent verification claims. | +| Install | `npm install -g vestige-mcp-server@latest` then `claude mcp add vestige vestige-mcp -s user` | +| Repo | https://github.com/samvallad33/vestige | +| Homepage | https://samvallad33.github.io/vestige/ | +| License | AGPL-3.0-only | +| Transport | stdio (default); HTTP opt-in | +| Version | 2.1.23 | +| Tags | memory, mcp, claude, cursor, local-first, fsrs, neuroscience, rust | + +## Registries + +| Directory | URL | Status | Notes | +|-----------|-----|--------|-------| +| Glama | https://glama.ai/mcp/servers | [ ] Submit / refresh | Ownership metadata in repo (`cd496e5`) | +| mcp.so | https://mcp.so | [ ] Submit | Use submission packet | +| Smithery | https://smithery.ai | [ ] Submit | npm package + stdio command | +| PulseMCP | https://www.pulsemcp.com | [ ] Submit | | +| Awesome MCP Servers | https://github.com/punkpeye/awesome-mcp-servers | [ ] PR | Add under Memory / Knowledge | +| modelcontextprotocol/servers | https://github.com/modelcontextprotocol/servers | [ ] PR if accepted | Follow their CONTRIBUTING | +| Cursor directory | docs/integrations/cursor.md | [x] Doc exists | Link from Cursor forum / Discord | +| VS Code marketplace | N/A for MCP stdio | [ ] N/A | Use integrations/vscode.md in posts | + +## Awesome-MCP PR snippet + +```markdown +### Vestige +- **Description:** Local cognitive memory — FSRS-6 decay, dreaming, contradiction tools, optional Receipt Lock +- **Install:** `npm install -g vestige-mcp-server@latest` +- **Command:** `vestige-mcp` +- **Repo:** https://github.com/samvallad33/vestige +``` + +## After each listing goes live + +```bash +# Ingest into marketing Vestige +smart_ingest: "Listed Vestige on [REGISTRY] at [URL]. Version 2.1.23." +tags: marketing, registry, vestige-launch +``` + +## Editor-specific posts (optional) + +| Community | Action | +|-----------|--------| +| Cursor Discord #showcase | Link comparison.md + 30s dashboard GIF | +| Claude Code GitHub discussions | Receipt Lock angle + install | +| r/mcp | Neutral "new server" post after Wave B | diff --git a/docs/marketing/metrics-tracker.md b/docs/marketing/metrics-tracker.md new file mode 100644 index 0000000..d995ba9 --- /dev/null +++ b/docs/marketing/metrics-tracker.md @@ -0,0 +1,66 @@ +# Vestige Growth Metrics Tracker + +**North star:** weekly `vestige-mcp-server` npm installs + evidence of active MCP connections (issues, "it works" posts). + +Update every **Monday**. Feed summary into marketing Vestige via `smart_ingest`. + +## How to fetch numbers + +```bash +# npm weekly downloads (approximate) +npm view vestige-mcp-server + +# GitHub stars +gh api repos/samvallad33/vestige --jq .stargazers_count + +# Optional: npm download chart +# https://www.npmjs.com/package/vestige-mcp-server +``` + +## Weekly log template + +Copy a row per week: + +| Week ending | npm downloads (total) | Stars | Stars Δ | Top channel | Top hook | Installs anecdote | Notes | +|-------------|----------------------|-------|---------|-------------|----------|-------------------|-------| +| 2026-06-02 | TBD | 542 | 0 | pre-launch | Receipt Lock / fake tests passed | setup complete | marketing instance seeded, ready for Wave A | +| 2026-06-09 | | | | | | | post Wave A week 1 | + +## Per-post log template + +| Date | Wave | Channel | Post URL | Engagement | Stars Δ (48h) | Objections | Action | +|------|------|---------|----------|------------|---------------|------------|--------| +| | A | HN | | | | | | + +## Objection → content flywheel + +When the same objection appears 3+ times, promote to permanent doc: + +| Objection | Response doc | +|-----------|----------------| +| "Isn't this just RAG?" | [comparison.md](../comparison.md) | +| "Claude has memory now" | comparison.md + Receipt Lock section | +| "AGPL?" | README + HN FAQ in show-hn.md | +| "77K LOC over-engineered" | show-hn.md FAQ | +| "FSRS gimmick?" | [SCIENCE.md](../SCIENCE.md) | + +## Agent ingest prompt (weekly) + +``` +smart_ingest: Vestige marketing week ending YYYY-MM-DD. +npm: X total (ΔY). Stars: N (ΔZ). +Best channel: [HN/Reddit/X]. +Best hook: [phrase]. +Top objection: [text]. +Next week: [one action]. +tags: marketing, metrics, vestige-launch +``` + +## Goals (first 8 weeks) + +| Milestone | Target | +|-----------|--------| +| Wave A HN front page | 100+ points | +| Stars | 542 → 800+ | +| npm weekly downloads | 2× baseline | +| Registry listings | 5+ MCP directories | diff --git a/docs/marketing/ready-to-post/hn-first-comment.txt b/docs/marketing/ready-to-post/hn-first-comment.txt new file mode 100644 index 0000000..ba24fb5 --- /dev/null +++ b/docs/marketing/ready-to-post/hn-first-comment.txt @@ -0,0 +1,32 @@ +Hi HN, + +Your coding agent probably ends sessions with something like "all tests passed" or +"the build is green." I kept trusting that — until it wasn't true. + +I built Receipt Lock in Vestige (an MCP memory server I maintain). Before operational +claims become part of the final answer, Vestige checks them against structured +command receipts from the current transcript. No matching successful receipt → the +claim can be blocked and a local veto receipt is written (JSON + HTML under +~/.vestige/sanhedrin/). + +**What it is:** Optional Claude Code Cognitive Sandwich hooks + local MCP server. +Not cloud. Not "trust me bro" logging — inspectable receipts on disk. + +**Install (memory server — required base):** +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user + +**Enable Receipt Lock (optional):** +vestige sandwich install --enable-sanhedrin + +Sanhedrin verifier can point at any OpenAI-compatible endpoint (Ollama, MLX, hosted API). + +**And it also does real memory:** FSRS-6 spaced repetition, prediction error gating, +memory dreaming, 3D dashboard at localhost:3927. ~86K LOC Rust, 25 MCP tools, 1,200+ +tests, 22MB binary. 100% local after first embedding download. + +I'm 22, solo, AGPL-3.0. Repo: https://github.com/samvallad33/vestige +Comparison: https://github.com/samvallad33/vestige/blob/main/docs/comparison.md +Landing: https://samvallad33.github.io/vestige/ + +Happy to discuss false positive tuning, Sanhedrin presets, or why receipts beat vibes. diff --git a/docs/marketing/ready-to-post/hn-title.txt b/docs/marketing/ready-to-post/hn-title.txt new file mode 100644 index 0000000..f01f5d8 --- /dev/null +++ b/docs/marketing/ready-to-post/hn-title.txt @@ -0,0 +1 @@ +Show HN: Vestige – blocks coding agents from claiming "tests passed" without receipts diff --git a/docs/marketing/ready-to-post/reddit-experienceddevs-title.txt b/docs/marketing/ready-to-post/reddit-experienceddevs-title.txt new file mode 100644 index 0000000..3aab4ba --- /dev/null +++ b/docs/marketing/ready-to-post/reddit-experienceddevs-title.txt @@ -0,0 +1 @@ +My coding agent kept saying "tests passed" when they hadn't. I added a receipt check before the summary ships. diff --git a/docs/marketing/ready-to-post/reddit-experienceddevs.md b/docs/marketing/ready-to-post/reddit-experienceddevs.md new file mode 100644 index 0000000..410f87b --- /dev/null +++ b/docs/marketing/ready-to-post/reddit-experienceddevs.md @@ -0,0 +1,19 @@ +**TL;DR:** Vestige Receipt Lock checks operational claims ("tests passed", "build green", "lint clean") against structured command receipts from the transcript. No receipt → block + local veto artifact. + +**The failure mode:** Agent runs partial checks, or hallucinates a green ending. You merge. CI breaks. You've seen this. + +**The fix:** Optional hooks (`vestige sandwich install --enable-sanhedrin`) + MCP memory server. When the model tries to assert verification without evidence, Vestige can veto and write `~/.vestige/sanhedrin/latest.html` so you can inspect what happened. + +**Not a replacement for CI.** It's a last-mile guard on *agent-authored* summaries in Claude Code. + +**Stack:** Rust, local, MCP. Same project also does FSRS-6 cognitive memory (decay, dreaming, contradiction tools) — I'll post that angle separately if people want the science side. + +```bash +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user +vestige sandwich install --enable-sanhedrin +``` + +GitHub: https://github.com/samvallad33/vestige + +What false positives are you seeing with agent verification claims? Curious if this matches your workflow. diff --git a/docs/marketing/ready-to-post/x-thread.txt b/docs/marketing/ready-to-post/x-thread.txt new file mode 100644 index 0000000..cc81fec --- /dev/null +++ b/docs/marketing/ready-to-post/x-thread.txt @@ -0,0 +1,18 @@ +1/8 Your coding agent ends with "tests passed." Did it run tests? Or did it summarize hope? + +2/8 I ship Receipt Lock in Vestige — checks operational claims against command receipts from the transcript. + +3/8 No matching successful receipt → claim blocked. Local veto receipt: ~/.vestige/sanhedrin/latest.html + +4/8 Optional hooks. Local MCP server. Not cloud analytics. + +5/8 +npm i -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user +vestige sandwich install --enable-sanhedrin + +6/8 Same binary also does FSRS-6 memory — decay, dreaming, 3D brain viz. Thread on that tomorrow. + +7/8 22yo solo dev. AGPL. https://github.com/samvallad33/vestige + +8/8 What's the worst "green build" lie your agent told you? Reply — building the FAQ from real stories. diff --git a/docs/marketing/wave-a-launch.md b/docs/marketing/wave-a-launch.md new file mode 100644 index 0000000..74303d6 --- /dev/null +++ b/docs/marketing/wave-a-launch.md @@ -0,0 +1,47 @@ +# Wave A — Execution Checklist + +Copy-paste from [docs/launch/receipt-lock.md](../launch/receipt-lock.md). **You post manually.** + +## Pre-flight + +- [x] `docs/LAUNCH_STATS.md` numbers match README +- [x] `vestige health` passes on your machine +- [ ] GitHub Pages live: https://samvallad33.github.io/vestige/ (after `git push` + Pages enabled) +- [x] GIFs captured OR placeholder SVG acceptable for Wave A +- [x] Marketing Vestige seeded: `./scripts/marketing/setup-marketing-instance.sh` +- [ ] Ingest Wave A URLs after posting (see LAUNCH_NOW.md) + +## Day 1 — HN + Lobsters + +| Time (ET) | Channel | Artifact | +|-----------|---------|----------| +| 8:00 AM Tue/Wed | Hacker News Show HN | Title + first comment from `receipt-lock.md` | +| +30 min | Lobste.rs | Shorter HN body | +| 8:00–11:00 AM | HN comments | 30-min reply SLA | + +## Day 1–2 — Reddit + X + +| Time | Channel | Artifact | +|------|---------|----------| +| +1h | r/ExperiencedDevs | Full post in `receipt-lock.md` | +| +2h | r/programming | Same + stats line | +| +0h (parallel) | X thread | 8 tweets in `receipt-lock.md` | +| Pin | X profile | Thread link during HN peak | + +## Metrics to log (→ `metrics-tracker.md`) + +| Field | Value | +|-------|-------| +| Date | | +| Channel | | +| URL | | +| Upvotes / points | | +| Comments | | +| npm downloads (week delta) | | +| Stars delta (48h) | | +| Top objection | | +| Winning hook phrase | | + +## After Wave A + +Wait **48h** from HN peak, then run [wave-b-launch.md](wave-b-launch.md). diff --git a/docs/marketing/wave-b-launch.md b/docs/marketing/wave-b-launch.md new file mode 100644 index 0000000..8f1084e --- /dev/null +++ b/docs/marketing/wave-b-launch.md @@ -0,0 +1,45 @@ +# Wave B — Execution Checklist + +Memory / neuroscience product wave. Cross-link Wave A HN thread if it performed. + +Sources: [show-hn.md](../launch/show-hn.md) (refresh before posting), [demo-script.md](../launch/demo-script.md), [comparison.md](../comparison.md). + +## Pre-flight + +- [ ] Dashboard GIF live (`docs/marketing/assets/dashboard-dream.gif`) +- [ ] 20 pre-loaded memories per demo-script pre-demo checklist +- [ ] Wave A metrics logged in `metrics-tracker.md` + +## Day 3 — Show HN (memory angle) OR skip if Wave A HN was same week + +If Wave A used Show HN title for Receipt Lock, **do not** second Show HN same week. Use Reddit + X only for Wave B. + +| Channel | Focus | +|---------|-------| +| r/ClaudeAI | MCP memory, FSRS-6, dashboard, 2-min install | +| r/LocalLLaMA | Local-first, zero cloud, ONNX embeddings | +| r/rust | Architecture, 86K LOC, 25 tools, zero unsafe | + +## Day 4–5 — Content + +| Channel | Artifact | +|---------|----------| +| X | 10-tweet thread: "RAG is not memory" + FSRS + dream GIF | +| LinkedIn | Link to comparison.md + dashboard GIF | +| Blog | Optional: publish refreshed `docs/launch/blog-post.md` on dev.to / personal site | + +## r/ClaudeAI title (v2.1.23) + +``` +Vestige v2.1.23 — FSRS-6 memory for Claude Code + optional Receipt Lock when agents fake "tests passed" +``` + +Body: Use r/ClaudeAI section from refreshed `show-hn.md` + install block from LAUNCH_STATS. + +## Cross-link line (if Wave A performed) + +> Earlier this week I posted about Receipt Lock (agents claiming tests passed without receipts). Same project — the memory engine underneath: [link] + +## Metrics + +Same table as wave-a-launch.md. Tag `wave=B` in marketing Vestige ingest. diff --git a/docs/website/index.html b/docs/website/index.html new file mode 100644 index 0000000..7246339 --- /dev/null +++ b/docs/website/index.html @@ -0,0 +1,82 @@ + + + + + + Vestige — Local memory and receipts for AI agents + + + + + + +
+

+ v2.1.23 + 25 MCP tools + ~86K LOC Rust + AGPL-3.0 +

+ +

Your agent forgets yesterday.
It can lie about today.

+

Vestige is a local MCP memory server: real FSRS-6 forgetting and consolidation, plus optional Receipt Lock that blocks “tests passed” without command receipts.

+ + 
npm install -g vestige-mcp-server@latest
+claude mcp add vestige vestige-mcp -s user
+ GitHub → + +

Wave A: Receipt Lock

+
Coding agents finish with confident summaries. Vestige checks operational claims against structured command receipts before they become your final answer.
+

If the agent claims verification happened but no matching successful receipt exists, the claim can be blocked and an inspectable local veto is written to ~/.vestige/sanhedrin/.

+ 
vestige sandwich install --enable-sanhedrin
+

Receipt Lock docs

+ +

Wave B: A brain, not a bucket

+ Vestige 3D memory dashboard +

Memories decay on FSRS-6 curves. Search strengthens them (Testing Effect). dream consolidates offline. The 3D dashboard shows pulses, connections, and dream replay in real time.

+

vestige dashboard → localhost:3927/dashboard

+ +

Vestige vs the rest

+ + + + + + + + + +
RAGNative AI memoryVestige
ForgettingNoneOpaqueFSRS-6
Local / privateVariesCloud100% local
ContradictionsBoth chunksNodeep_reference
Fake “tests passed”N/AN/AReceipt Lock
VisualizationNoneNone3D graph
+

Full comparison (Mem0, RAG, native)

+ +

Install

+ 
npm install -g vestige-mcp-server@latest
+vestige health
+claude mcp add vestige vestige-mcp -s user
+

Also works with Codex, Cursor, VS Code Copilot, JetBrains, Windsurf, Xcode. See integration guides.

+ + +
+ + diff --git a/package.json b/package.json index 9d759a6..1057869 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "license": "AGPL-3.0-only", "repository": { "type": "git", - "url": "https://github.com/samvallad33/vestige" + "url": "https://github.com/samvallad33/vestige", + "homepage": "https://samvallad33.github.io/vestige/" }, "scripts": { "build:mcp": "cargo build --release --package vestige-mcp", diff --git a/packages/vestige-mcp-npm/README.md b/packages/vestige-mcp-npm/README.md index 98e6575..b7a01c2 100644 --- a/packages/vestige-mcp-npm/README.md +++ b/packages/vestige-mcp-npm/README.md @@ -1,8 +1,12 @@ # vestige-mcp-server -Vestige MCP Server - A synthetic hippocampus for AI assistants. +**v2.1.23** — Vestige MCP Server: local cognitive memory and optional Receipt Lock for MCP-compatible AI agents. -Built on 130 years of cognitive science research, Vestige provides biologically-inspired memory that decays, strengthens, and consolidates like the human mind. +- **Memory:** FSRS-6 spaced repetition, prediction error gating, dreaming, 3D dashboard +- **Receipt Lock:** blocks "tests passed" / "build green" without command receipts (optional hooks) +- **Stats:** ~86K LOC Rust · 25 tools · 1,200+ tests · 22MB binary · 100% local + +Homepage: https://samvallad33.github.io/vestige/ · Repo: https://github.com/samvallad33/vestige ## Installation @@ -54,6 +58,25 @@ codex mcp add vestige -- vestige-mcp Then restart your MCP client. +## Optional Receipt Lock for Claude Code + +Receipt Lock is part of Vestige's optional Cognitive Sandwich hook layer. Normal +MCP memory stays lightweight and local. If you want claim checking for summaries +like "tests passed" or "lint is clean," enable Sanhedrin and point it at any +OpenAI-compatible chat endpoint: + +```bash +vestige sandwich install --enable-sanhedrin + +vestige sandwich install \ + --enable-sanhedrin \ + --sanhedrin-endpoint=http://127.0.0.1:11434/v1/chat/completions \ + --sanhedrin-model=qwen2.5:14b +``` + +If a claim is missing command evidence, Vestige writes local receipts under +`~/.vestige/sanhedrin/` so the veto is inspectable instead of opaque. + ## Usage with Claude Desktop Add to your Claude Desktop configuration: @@ -86,6 +109,7 @@ vestige sandwich install # Manage optional Claude Code hook files ## Features - **FSRS-6 Algorithm**: State-of-the-art spaced repetition for optimal memory retention +- **Receipt Lock**: Optional command-receipt checking for test/build/lint/typecheck claims - **Dual-Strength Memory**: Bjork & Bjork (1992) - Storage + Retrieval strength model - **Synaptic Tagging**: Memories become important retroactively (Frey & Morris 1997) - **Semantic Search**: Local embeddings via nomic-embed-text-v1.5 (768 dimensions) diff --git a/packages/vestige-mcp-npm/package.json b/packages/vestige-mcp-npm/package.json index 1ff860f..272681e 100644 --- a/packages/vestige-mcp-npm/package.json +++ b/packages/vestige-mcp-npm/package.json @@ -29,6 +29,7 @@ "type": "git", "url": "git+https://github.com/samvallad33/vestige.git" }, + "homepage": "https://samvallad33.github.io/vestige/", "engines": { "node": ">=18" }, diff --git a/scripts/marketing/preflight.sh b/scripts/marketing/preflight.sh new file mode 100755 index 0000000..590464a --- /dev/null +++ b/scripts/marketing/preflight.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Pre-launch checks before Wave A posts. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "${ROOT}" + +FAIL=0 +pass() { echo "OK $1"; } +fail() { echo "FAIL $1"; FAIL=1; } + +echo "=== Vestige Launch Preflight ===" + +command -v vestige-mcp >/dev/null && pass "vestige-mcp on PATH" || fail "vestige-mcp not found" +command -v npm >/dev/null && pass "npm available" || fail "npm missing" + +VER="$(vestige-mcp --version 2>/dev/null || true)" +[[ "${VER}" == *"2.1."* ]] && pass "version ${VER}" || fail "unexpected version: ${VER}" + +vestige health >/dev/null 2>&1 && pass "vestige health" || fail "vestige health failed" + +[[ -f docs/LAUNCH_STATS.md ]] && pass "LAUNCH_STATS.md" || fail "missing LAUNCH_STATS.md" +[[ -f docs/launch/receipt-lock.md ]] && pass "receipt-lock.md" || fail "missing receipt-lock.md" +[[ -f docs/website/index.html ]] && pass "landing page source" || fail "missing website" +[[ -f .github/workflows/pages.yml ]] && pass "pages workflow" || fail "missing pages workflow" + +if curl -sf --max-time 5 "https://samvallad33.github.io/vestige/" >/dev/null 2>&1; then + pass "GitHub Pages live" +else + echo "WARN GitHub Pages not live yet — push main and enable Pages → GitHub Actions" +fi + +MARKETING_DIR="${HOME}/.vestige-marketing" +if [[ -d "${MARKETING_DIR}" ]]; then + pass "marketing data dir exists" +else + echo "WARN run scripts/marketing/setup-marketing-instance.sh" +fi + +echo "" +if [[ "${FAIL}" -eq 0 ]]; then + echo "Preflight PASSED — ready for Wave A" + exit 0 +else + echo "Preflight FAILED — fix items above" + exit 1 +fi diff --git a/scripts/marketing/seed-baseline-memories.sh b/scripts/marketing/seed-baseline-memories.sh new file mode 100755 index 0000000..9b2a384 --- /dev/null +++ b/scripts/marketing/seed-baseline-memories.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Seed marketing Vestige with launch facts (separate from dev memory). +set -euo pipefail + +MARKETING_DIR="${VESTIGE_MARKETING_DIR:-$HOME/.vestige-marketing}" +TAGS="marketing,baseline,vestige-launch" + +ingest() { + vestige ingest "$1" --data-dir "${MARKETING_DIR}" --tags "${TAGS}" --node-type note +} + +echo "Seeding into ${MARKETING_DIR}..." + +ingest "Vestige v2.1.23 launch stats: ~86K LOC Rust, 25 MCP tools, 30 cognitive modules, 1200+ tests, 22MB binary, AGPL-3.0, npm vestige-mcp-server@latest, homepage samvallad33.github.io/vestige" + +ingest "Wave A hook Receipt Lock: block operational claims like tests passed or build green unless matching command receipts exist. Optional vestige sandwich install --enable-sanhedrin. Veto receipts at ~/.vestige/sanhedrin/" + +ingest "Wave B product: FSRS-6 spaced repetition memory, prediction error gating, memory dreaming, 3D dashboard localhost:3927, deep_reference contradictions, 100 percent local after embedding download" + +ingest "Canonical install: npm install -g vestige-mcp-server@latest && vestige health && claude mcp add vestige vestige-mcp -s user. Do NOT tell users sudo mv unless manual binary install failed." + +ingest "Messaging guardrails: no revolutionary game-changer AI-powered. Acknowledge Mem0 RAG native Claude memory honestly. Lead pain first tool second. Author Sam Valladares age 22 solo." + +ingest "North star metric: weekly npm installs vestige-mcp-server and active MCP connections not stars alone. Track in docs/marketing/metrics-tracker.md" + +ingest "Comparison anchor docs/comparison.md: RAG is retrieval, Vestige is cognitive lifecycle with forgetting consolidation Receipt Lock. Mem0 is cloud API Vestige is local AGPL." + +vestige stats --data-dir "${MARKETING_DIR}" 2>/dev/null || true +echo "Baseline seed complete." diff --git a/scripts/marketing/setup-marketing-instance.sh b/scripts/marketing/setup-marketing-instance.sh new file mode 100755 index 0000000..e465c06 --- /dev/null +++ b/scripts/marketing/setup-marketing-instance.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# One-time setup: dedicated Vestige store + Claude Code MCP entry for marketing. +set -euo pipefail + +MARKETING_DIR="${VESTIGE_MARKETING_DIR:-$HOME/.vestige-marketing}" +BIN_DIR="${HOME}/.local/bin" +WRAPPER="${BIN_DIR}/vestige-mcp-marketing" + +echo "==> Marketing data dir: ${MARKETING_DIR}" +mkdir -p "${MARKETING_DIR}" + +if ! command -v vestige-mcp >/dev/null 2>&1; then + echo "Install Vestige first: npm install -g vestige-mcp-server@latest" + exit 1 +fi + +mkdir -p "${BIN_DIR}" +cat > "${WRAPPER}" < Wrapper: ${WRAPPER}" + +if command -v claude >/dev/null 2>&1; then + if claude mcp list 2>/dev/null | grep -q vestige-marketing; then + echo "==> claude mcp: vestige-marketing already registered" + else + claude mcp add vestige-marketing "${WRAPPER}" -s user + echo "==> Added: claude mcp add vestige-marketing ${WRAPPER} -s user" + fi +else + echo "==> Claude Code not found — register manually:" + echo " claude mcp add vestige-marketing ${WRAPPER} -s user" +fi + +echo "==> Seeding baseline memories..." +"$(dirname "$0")/seed-baseline-memories.sh" + +echo "" +echo "Done. Open Claude Code with MARKETING-CLAUDE.md:" +echo " cp docs/marketing/growth-engine/MARKETING-CLAUDE.md ~/vestige-marketing-CLAUDE.md" +echo " vestige health --data-dir ${MARKETING_DIR}" From 67f9550e6c32361bfc8af15675644404eea833db Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Tue, 2 Jun 2026 12:39:26 -0500 Subject: [PATCH 13/38] Fix Pages workflow enablement and add Lobsters launch copy Use configure-pages enablement so the landing deploy can succeed without manual repo setup first. Co-authored-by: Cursor --- .github/workflows/pages.yml | 2 ++ docs/marketing/LAUNCH_NOW.md | 14 +++++++++----- docs/marketing/ready-to-post/lobsters.md | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 docs/marketing/ready-to-post/lobsters.md diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d30a8f3..2d54012 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -37,6 +37,8 @@ jobs: sed -i 's|../marketing/assets/|assets/|g' _site/index.html || true - uses: actions/configure-pages@v5 + with: + enablement: true - uses: actions/upload-pages-artifact@v3 with: diff --git a/docs/marketing/LAUNCH_NOW.md b/docs/marketing/LAUNCH_NOW.md index 7e2b887..d3c7739 100644 --- a/docs/marketing/LAUNCH_NOW.md +++ b/docs/marketing/LAUNCH_NOW.md @@ -10,18 +10,22 @@ ./scripts/marketing/preflight.sh ``` -2. **Hacker News** → https://news.ycombinator.com/submit +2. **Enable GitHub Pages** (one-time, if preflight warns): Repo → Settings → Pages → Build: **GitHub Actions**. Or re-run deploy after the workflow with `enablement: true` lands. + +3. **Hacker News** → https://news.ycombinator.com/submit - URL: `https://github.com/samvallad33/vestige` - Title: paste `ready-to-post/hn-title.txt` - Post link, then immediately paste `ready-to-post/hn-first-comment.txt` as first comment -3. **X** — paste `ready-to-post/x-thread.txt` (one tweet per numbered block) +4. **Lobste.rs** — paste `ready-to-post/lobsters.md` -4. **r/ExperiencedDevs** — title from `reddit-experienceddevs-title.txt`, body from `reddit-experienceddevs.md` +5. **X** — paste `ready-to-post/x-thread.txt` (one tweet per numbered block) -5. **r/programming** — same body + line: `License: AGPL-3.0. v2.1.23. ~86K LOC, 25 tools, 22MB binary.` +6. **r/ExperiencedDevs** — title from `reddit-experienceddevs-title.txt`, body from `reddit-experienceddevs.md` -6. Log URLs in [metrics-tracker.md](metrics-tracker.md) +7. **r/programming** — same body + line: `License: AGPL-3.0. v2.1.23. ~86K LOC, 25 tools, 22MB binary.` + +8. Log URLs in [metrics-tracker.md](metrics-tracker.md) ## After posting (30 min SLA on comments) diff --git a/docs/marketing/ready-to-post/lobsters.md b/docs/marketing/ready-to-post/lobsters.md new file mode 100644 index 0000000..7fdc9fe --- /dev/null +++ b/docs/marketing/ready-to-post/lobsters.md @@ -0,0 +1,20 @@ +Vestige Receipt Lock: local MCP guard against unverified "tests passed" agent claims + +Tags: rust, programming, security + +--- + +Your coding agent probably ends sessions with "all tests passed" or "the build is green." + +Vestige (MCP memory server, Rust, local) adds optional Receipt Lock: operational claims are checked against structured command receipts from the transcript. No matching successful receipt → claim can be blocked; inspectable veto at ~/.vestige/sanhedrin/latest.html + +```bash +npm install -g vestige-mcp-server@latest +claude mcp add vestige vestige-mcp -s user +vestige sandwich install --enable-sanhedrin +``` + +Same binary also does FSRS-6 cognitive memory (decay, dreaming, 3D dashboard). v2.1.23, ~86K LOC, 25 tools, AGPL-3.0. + +https://github.com/samvallad33/vestige +https://github.com/samvallad33/vestige/blob/main/docs/comparison.md From 6a37586c5f13b14c47097ba5c1d628a4d4b0fede Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Tue, 2 Jun 2026 12:40:48 -0500 Subject: [PATCH 14/38] Revert "Fix Pages workflow enablement and add Lobsters launch copy" This reverts commit 67f9550e6c32361bfc8af15675644404eea833db. --- .github/workflows/pages.yml | 2 -- docs/marketing/LAUNCH_NOW.md | 14 +++++--------- docs/marketing/ready-to-post/lobsters.md | 20 -------------------- 3 files changed, 5 insertions(+), 31 deletions(-) delete mode 100644 docs/marketing/ready-to-post/lobsters.md diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 2d54012..d30a8f3 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -37,8 +37,6 @@ jobs: sed -i 's|../marketing/assets/|assets/|g' _site/index.html || true - uses: actions/configure-pages@v5 - with: - enablement: true - uses: actions/upload-pages-artifact@v3 with: diff --git a/docs/marketing/LAUNCH_NOW.md b/docs/marketing/LAUNCH_NOW.md index d3c7739..7e2b887 100644 --- a/docs/marketing/LAUNCH_NOW.md +++ b/docs/marketing/LAUNCH_NOW.md @@ -10,22 +10,18 @@ ./scripts/marketing/preflight.sh ``` -2. **Enable GitHub Pages** (one-time, if preflight warns): Repo → Settings → Pages → Build: **GitHub Actions**. Or re-run deploy after the workflow with `enablement: true` lands. - -3. **Hacker News** → https://news.ycombinator.com/submit +2. **Hacker News** → https://news.ycombinator.com/submit - URL: `https://github.com/samvallad33/vestige` - Title: paste `ready-to-post/hn-title.txt` - Post link, then immediately paste `ready-to-post/hn-first-comment.txt` as first comment -4. **Lobste.rs** — paste `ready-to-post/lobsters.md` +3. **X** — paste `ready-to-post/x-thread.txt` (one tweet per numbered block) -5. **X** — paste `ready-to-post/x-thread.txt` (one tweet per numbered block) +4. **r/ExperiencedDevs** — title from `reddit-experienceddevs-title.txt`, body from `reddit-experienceddevs.md` -6. **r/ExperiencedDevs** — title from `reddit-experienceddevs-title.txt`, body from `reddit-experienceddevs.md` +5. **r/programming** — same body + line: `License: AGPL-3.0. v2.1.23. ~86K LOC, 25 tools, 22MB binary.` -7. **r/programming** — same body + line: `License: AGPL-3.0. v2.1.23. ~86K LOC, 25 tools, 22MB binary.` - -8. Log URLs in [metrics-tracker.md](metrics-tracker.md) +6. Log URLs in [metrics-tracker.md](metrics-tracker.md) ## After posting (30 min SLA on comments) diff --git a/docs/marketing/ready-to-post/lobsters.md b/docs/marketing/ready-to-post/lobsters.md deleted file mode 100644 index 7fdc9fe..0000000 --- a/docs/marketing/ready-to-post/lobsters.md +++ /dev/null @@ -1,20 +0,0 @@ -Vestige Receipt Lock: local MCP guard against unverified "tests passed" agent claims - -Tags: rust, programming, security - ---- - -Your coding agent probably ends sessions with "all tests passed" or "the build is green." - -Vestige (MCP memory server, Rust, local) adds optional Receipt Lock: operational claims are checked against structured command receipts from the transcript. No matching successful receipt → claim can be blocked; inspectable veto at ~/.vestige/sanhedrin/latest.html - -```bash -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user -vestige sandwich install --enable-sanhedrin -``` - -Same binary also does FSRS-6 cognitive memory (decay, dreaming, 3D dashboard). v2.1.23, ~86K LOC, 25 tools, AGPL-3.0. - -https://github.com/samvallad33/vestige -https://github.com/samvallad33/vestige/blob/main/docs/comparison.md From a355da99a6f93e52d349c33ca95debe63aa56c9a Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Tue, 2 Jun 2026 12:40:48 -0500 Subject: [PATCH 15/38] Revert "Add developer launch kit for Vestige v2.1.23" This reverts commit 00511948ffd345e17b3f12a858114fc35375f502. --- .github/workflows/pages.yml | 46 ----- CLAUDE.md | 3 +- README.md | 59 +------ docs/LAUNCH_STATS.md | 88 --------- docs/comparison.md | 82 --------- docs/launch/blog-post.md | 2 +- docs/launch/demo-script.md | 12 +- docs/launch/receipt-lock.md | 167 ------------------ docs/launch/reddit-cross-reference.md | 25 ++- docs/launch/show-hn.md | 38 ++-- docs/marketing/LAUNCH_NOW.md | 35 ---- docs/marketing/README.md | 33 ---- docs/marketing/assets/.gitkeep | 0 docs/marketing/assets/CAPTURE.md | 40 ----- .../assets/dashboard-placeholder.svg | 10 -- docs/marketing/demo-video-storyboard.md | 32 ---- .../growth-engine/MARKETING-CLAUDE.md | 69 -------- docs/marketing/growth-engine/README.md | 84 --------- docs/marketing/mcp-registries.md | 57 ------ docs/marketing/metrics-tracker.md | 66 ------- .../ready-to-post/hn-first-comment.txt | 32 ---- docs/marketing/ready-to-post/hn-title.txt | 1 - .../reddit-experienceddevs-title.txt | 1 - .../ready-to-post/reddit-experienceddevs.md | 19 -- docs/marketing/ready-to-post/x-thread.txt | 18 -- docs/marketing/wave-a-launch.md | 47 ----- docs/marketing/wave-b-launch.md | 45 ----- docs/website/index.html | 82 --------- package.json | 3 +- packages/vestige-mcp-npm/README.md | 28 +-- packages/vestige-mcp-npm/package.json | 1 - scripts/marketing/preflight.sh | 47 ----- scripts/marketing/seed-baseline-memories.sh | 29 --- scripts/marketing/setup-marketing-instance.sh | 44 ----- 34 files changed, 42 insertions(+), 1303 deletions(-) delete mode 100644 .github/workflows/pages.yml delete mode 100644 docs/LAUNCH_STATS.md delete mode 100644 docs/comparison.md delete mode 100644 docs/launch/receipt-lock.md delete mode 100644 docs/marketing/LAUNCH_NOW.md delete mode 100644 docs/marketing/README.md delete mode 100644 docs/marketing/assets/.gitkeep delete mode 100644 docs/marketing/assets/CAPTURE.md delete mode 100644 docs/marketing/assets/dashboard-placeholder.svg delete mode 100644 docs/marketing/demo-video-storyboard.md delete mode 100644 docs/marketing/growth-engine/MARKETING-CLAUDE.md delete mode 100644 docs/marketing/growth-engine/README.md delete mode 100644 docs/marketing/mcp-registries.md delete mode 100644 docs/marketing/metrics-tracker.md delete mode 100644 docs/marketing/ready-to-post/hn-first-comment.txt delete mode 100644 docs/marketing/ready-to-post/hn-title.txt delete mode 100644 docs/marketing/ready-to-post/reddit-experienceddevs-title.txt delete mode 100644 docs/marketing/ready-to-post/reddit-experienceddevs.md delete mode 100644 docs/marketing/ready-to-post/x-thread.txt delete mode 100644 docs/marketing/wave-a-launch.md delete mode 100644 docs/marketing/wave-b-launch.md delete mode 100644 docs/website/index.html delete mode 100755 scripts/marketing/preflight.sh delete mode 100755 scripts/marketing/seed-baseline-memories.sh delete mode 100755 scripts/marketing/setup-marketing-instance.sh diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml deleted file mode 100644 index d30a8f3..0000000 --- a/.github/workflows/pages.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Deploy GitHub Pages - -on: - push: - branches: [main] - paths: - - 'docs/website/**' - - 'docs/marketing/assets/**' - - '.github/workflows/pages.yml' - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - deploy: - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - uses: actions/checkout@v4 - - - name: Prepare site root - run: | - mkdir -p _site - cp -r docs/website/* _site/ - mkdir -p _site/assets - cp -r docs/marketing/assets/* _site/assets/ 2>/dev/null || true - # Fix asset paths for Pages (no parent ../) - sed -i 's|../marketing/assets/|assets/|g' _site/index.html || true - - - uses: actions/configure-pages@v5 - - - uses: actions/upload-pages-artifact@v3 - with: - path: _site - - - id: deployment - uses: actions/deploy-pages@v4 diff --git a/CLAUDE.md b/CLAUDE.md index c9034d6..4ee5762 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,8 +18,7 @@ dashboard embedded into the release binary. The core product promise is: ## Working Rules - Prefer source evidence over memory. Use `rg`, tests, and nearby code before - making claims about behavior. This public repo guidance does not override - private user-level memory protocols loaded outside the repository. + making claims about behavior. - Keep release changes scoped. Do not rewrite unrelated modules during a version/tag cleanup unless the release gate requires it. - Preserve local-first behavior. Heavy models, Sanhedrin-style verifier hooks, diff --git a/README.md b/README.md index fb612ad..f747715 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Vestige -### Local memory and receipts for MCP-compatible AI agents. +### Local cognitive memory for MCP-compatible AI agents. [![GitHub stars](https://img.shields.io/github/stars/samvallad33/vestige?style=social)](https://github.com/samvallad33/vestige) [![Release](https://img.shields.io/github/v/release/samvallad33/vestige)](https://github.com/samvallad33/vestige/releases/latest) @@ -10,23 +10,11 @@ [![License](https://img.shields.io/badge/license-AGPL--3.0-blue)](LICENSE) [![MCP Compatible](https://img.shields.io/badge/MCP-compatible-green)](https://modelcontextprotocol.io) -**Your coding agent forgets yesterday and can lie about today.** Vestige gives it a local brain — FSRS-6 memory that learns and forgets — plus **Receipt Lock** that blocks "tests passed" unless a real command receipt exists. +**Your agent forgets project decisions between sessions. Vestige gives it local, inspectable memory.** -```bash -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user -``` +Built on proven memory and retrieval ideas — FSRS-6 spaced repetition, prediction error gating, synaptic tagging, spreading activation, and memory consolidation — all running in a single Rust binary with a local dashboard. 100% local. Zero cloud. -| | | -|---|---| -| **Receipt Lock** — optional hook layer; vetoes unverified "green build" claims | **3D dashboard** — `vestige dashboard` → `localhost:3927` | -| ![Receipt Lock demo](docs/marketing/assets/receipt-lock.gif) · [capture guide](docs/marketing/assets/CAPTURE.md) | ![Memory dashboard](docs/marketing/assets/dashboard-placeholder.svg) | - -*Replace placeholders with GIFs from [`CAPTURE.md`](docs/marketing/assets/CAPTURE.md) before Wave B launch.* - -**v2.1.23** · ~86K LOC Rust · **25** MCP tools · **30** cognitive modules · **1,200+** tests · **22MB** binary · 100% local · AGPL-3.0 - -[Quick Start](#quick-start) | [Receipt Lock](#receipt-lock) | [Dashboard](#-3d-memory-dashboard) | [Compare vs RAG](docs/comparison.md) | [Launch stats](docs/LAUNCH_STATS.md) | [Docs](docs/) +[Quick Start](#quick-start) | [Dashboard](#-3d-memory-dashboard) | [How It Works](#-the-cognitive-science-stack) | [Tools](#-25-mcp-tools) | [Docs](docs/) @@ -67,43 +55,6 @@ codex mcp add vestige -- vestige-mcp # → "You prefer TypeScript over JavaScript." ``` -## Receipt Lock - -Coding agents often finish with confident summaries like "tests passed" or -"the build is green." Receipt Lock checks those operational claims against -structured command receipts from the current transcript before they become part -of the final answer. - -If the agent claims verification happened but no matching successful command -receipt exists, Vestige can block the claim and write an inspectable local -receipt instead of letting the agent invent a clean ending. - -Receipt Lock is optional and works through the Claude Code Cognitive Sandwich -hook layer: - -```bash -# Install the local memory server first -npm install -g vestige-mcp-server@latest - -# Add normal MCP memory -claude mcp add vestige vestige-mcp -s user - -# Optional: enable Receipt Lock / Sanhedrin hooks -vestige sandwich install --enable-sanhedrin - -# Optional: point Sanhedrin at any OpenAI-compatible endpoint -vestige sandwich install \ - --enable-sanhedrin \ - --sanhedrin-endpoint=http://127.0.0.1:11434/v1/chat/completions \ - --sanhedrin-model=qwen2.5:14b -``` - -Receipts are local: - -- Latest JSON: `~/.vestige/sanhedrin/latest.json` -- Latest HTML: `~/.vestige/sanhedrin/latest.html` -- Command ledger: `~/.vestige/sanhedrin/command-receipts.jsonl` -
Other platforms & install methods @@ -471,8 +422,6 @@ vestige dashboard # Open 3D dashboard in browser | [CLAUDE.md Setup](docs/CLAUDE-SETUP.md) | Templates for proactive memory | | [Configuration](docs/CONFIGURATION.md) | CLI commands, environment variables | | [Integrations](docs/integrations/) | Codex, Xcode, Cursor, VS Code, JetBrains, Windsurf | -| [Comparison vs RAG/Mem0](docs/comparison.md) | When to use Vestige | -| [Marketing kit](docs/marketing/README.md) | Launch waves, growth engine, metrics | | [Changelog](CHANGELOG.md) | Version history | --- diff --git a/docs/LAUNCH_STATS.md b/docs/LAUNCH_STATS.md deleted file mode 100644 index 4bd1d92..0000000 --- a/docs/LAUNCH_STATS.md +++ /dev/null @@ -1,88 +0,0 @@ -# Vestige Launch Stats (Single Source of Truth) - -**Last verified:** 2026-06-02 -**Use this file** when updating README, launch posts, npm README, and landing page. Do not invent numbers elsewhere. - -## Release - -| Field | Value | -|-------|-------| -| Version | **v2.1.23** ("Receipt Lock Hardening") | -| npm package | `vestige-mcp-server@latest` | -| Install | `npm install -g vestige-mcp-server@latest` | -| MCP connect (Claude Code) | `claude mcp add vestige vestige-mcp -s user` | -| Optional Receipt Lock | `vestige sandwich install --enable-sanhedrin` | -| License | AGPL-3.0-only | -| Repo | https://github.com/samvallad33/vestige | -| Homepage (marketing) | https://samvallad33.github.io/vestige/ (GitHub Pages) | - -## Author - -| Field | Value | -|-------|-------| -| Name | Sam Valladares | -| Age | **22** (solo developer) | -| GitHub stars (2026-06-02) | **542** | -| Forks | **55** | - -## Codebase (run to refresh) - -```bash -# Rust LOC (crates + tests) -find crates tests -name '*.rs' | xargs wc -l | tail -1 - -# MCP tool count (must match server assertion) -rg 'name: "' crates/vestige-mcp/src/server.rs | wc -l - -# Tests -cargo test --workspace --no-fail-fast 2>&1 | tail -3 -``` - -| Metric | Current value | Notes | -|--------|---------------|-------| -| Rust LOC | **~86,000** | `crates/` + `tests/` `.rs` files | -| MCP tools | **25** | Verified in `server.rs` (`tools.len() == 25`) | -| Cognitive modules | **30** | Per README architecture | -| Rust tests | **1,200+** | CHANGELOG v2.1.0: 1,229 passing; re-run before major launch | -| Dashboard tests | **171** | Vitest in `apps/dashboard` | -| Release binary | **~22MB** | Single binary, embedded SvelteKit dashboard | -| Embedding model | Nomic Embed Text v1.5 (~130MB first-run download) | - -## Install (canonical — no `sudo mv`) - -The npm package registers global bins via `postinstall`. **Do not** tell users to `sudo mv vestige-mcp` unless manual binary install failed. - -```bash -npm install -g vestige-mcp-server@latest -vestige health -claude mcp add vestige vestige-mcp -s user -``` - -If `vestige-mcp` is not on PATH after install: - -```bash -npm prefix -g # e.g. /usr/local or ~/.npm-global -# Ensure that path/bin is in your shell PATH -``` - -Manual binary placement (optional): - -```bash -vestige update --install-dir /usr/local/bin -``` - -## Messaging guardrails - -- Lead Wave A with **Receipt Lock** (agents overclaim "tests passed"). -- Close Wave B with **cognitive memory** (FSRS-6, dreaming, 3D dashboard). -- Never: "revolutionary", "game-changer", "AI-powered", competitor bashing. -- Always: honest neuroscience (faithful implementations vs engineering heuristics). - -## North-star metrics - -Track weekly (see `docs/marketing/metrics-tracker.md`): - -1. **npm downloads** (`npm view vestige-mcp-server` / npmjs.com stats) -2. **GitHub stars delta** -3. **Inbound issues/DMs** mentioning install -4. **Referral source** (HN, Reddit, X, registry) diff --git a/docs/comparison.md b/docs/comparison.md deleted file mode 100644 index 9a78a3e..0000000 --- a/docs/comparison.md +++ /dev/null @@ -1,82 +0,0 @@ -# Vestige vs Mem0 vs RAG vs Native AI Memory - -Canonical comparison for launch posts and arguments. Grounded in [SCIENCE.md](SCIENCE.md) and [LAUNCH_STATS.md](LAUNCH_STATS.md). - -## One-line thesis - -**RAG is retrieval. Native memory is a black box. Mem0 is a strong cloud memory API. Vestige is a local cognitive system that forgets, strengthens, dreams, and can block unverified agent claims.** - -## Comparison table - -| Capability | RAG / vector DB | Native AI memory (Claude, ChatGPT) | Mem0 | Vestige | -|------------|-----------------|-------------------------------------|------|---------| -| **Runs local** | Often cloud embeddings | Cloud only | Cloud API (local option limited) | **100% local** default | -| **You own the data** | Your infra | Vendor | Vendor / API | **SQLite on your disk** | -| **Forgetting curve** | None — equal weight forever | Opaque | Categories + metadata | **FSRS-6** power-law decay | -| **Duplicate handling** | Manual | Opaque | Some dedup | **Prediction Error Gating** on ingest | -| **Retrieval strengthens memory** | No | Unknown | Partial | **Testing Effect** on every search | -| **Offline consolidation** | No | No | No | **`dream`** — replay + connect | -| **Contradiction awareness** | Returns both chunks | No | Some products | **`deep_reference` / `contradictions`** | -| **Active suppression** | Delete only | No | Delete | **`suppress`** — inhibited, not erased | -| **Agent overclaim guard** | No | No | No | **Receipt Lock** (optional Sanhedrin hooks) | -| **Visualization** | None | None | Dashboard (cloud) | **3D graph** + WebSocket events | -| **Protocol** | Custom | Proprietary | API + MCP | **MCP** (25 tools) | -| **License** | Varies | Proprietary | Apache / commercial | **AGPL-3.0** (local use = free) | - -## When to use what - -### Use RAG when - -- You have a fixed document corpus (PDFs, wiki, codebase index). -- You need one-shot Q&A over static content. -- You do not need memory lifecycle or session continuity. - -### Use Mem0 when - -- You want a hosted memory API with minimal setup. -- Team sync and cloud dashboards are acceptable. -- You do not need FSRS decay or local-only air-gapped deploy. - -### Use native Claude/ChatGPT memory when - -- Casual personal context is enough. -- You do not need inspectable storage, decay curves, or contradiction tooling. - -### Use Vestige when - -- You run **Claude Code, Cursor, Codex, or any MCP client** daily. -- Context bloat from "remember everything" hurts retrieval quality. -- **Contradicting memories** have burned you (config changed, lib upgraded). -- You want **Receipt Lock** so agents cannot fake "tests passed." -- **Privacy / air-gapped** matters — embeddings run locally via ONNX. - -## Honest limitations (Vestige) - -- **AGPL-3.0**: hosting as a service without source disclosure is not allowed. -- **First-run download**: ~130MB embedding model (then offline). -- **Receipt Lock** requires optional Claude Code Cognitive Sandwich hooks + a verifier endpoint for Sanhedrin. -- **Neuroscience modules** mix faithful implementations and engineering heuristics — see [SCIENCE.md](SCIENCE.md) for citations vs approximations. -- **Solo project**: no enterprise SLA; GitHub issues are the support channel. - -## Receipt Lock (Vestige-only) - -Coding agents often end sessions with: - -> "All tests passed. Build is green. Ready to merge." - -Receipt Lock checks those **operational claims** against structured command receipts from the transcript. No matching successful receipt → claim blocked, local veto receipt written under `~/.vestige/sanhedrin/`. - -```bash -vestige sandwich install --enable-sanhedrin -``` - -Details: [README Receipt Lock section](../README.md#receipt-lock). - -## Install - -```bash -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user -``` - -Full stats: [LAUNCH_STATS.md](LAUNCH_STATS.md) · Repo: https://github.com/samvallad33/vestige diff --git a/docs/launch/blog-post.md b/docs/launch/blog-post.md index 08178ba..886bb5a 100644 --- a/docs/launch/blog-post.md +++ b/docs/launch/blog-post.md @@ -6,7 +6,7 @@ Every conversation starts from zero. You explain your project structure, your pr Vestige is an open-source Rust MCP server that gives AI agents persistent memory modeled on real neuroscience. Not metaphorical neuroscience. Actual published algorithms from Ebbinghaus (1885), Collins & Loftus (1975), Bjork & Bjork (1992), Frey & Morris (1997), and the FSRS-6 spaced repetition scheduler trained on 700 million Anki reviews. -~86,000 lines of Rust. 30 cognitive modules. 1,200+ tests. v2.1.23 adds Receipt Lock for unverified agent claims. Single 22MB binary with embedded SvelteKit dashboard. AGPL-3.0 licensed. Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md). +77,840+ lines of Rust. 29 cognitive modules. 734 tests. Single binary deployment with an embedded SvelteKit dashboard. AGPL-3.0 licensed. Here is how we built it. diff --git a/docs/launch/demo-script.md b/docs/launch/demo-script.md index 162efb8..4740dc4 100644 --- a/docs/launch/demo-script.md +++ b/docs/launch/demo-script.md @@ -1,6 +1,4 @@ -# Vestige v2.1.23 — Demo Script (Conference + Launch Video) - -> Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md) · Wave A hook: [receipt-lock.md](receipt-lock.md) +# Vestige v2.0 "Cognitive Leap" — MCP Dev Summit NYC Demo Script **Event:** MCP Dev Summit NYC, April 1-3, 2026 **Presenter:** Sam Valladares @@ -17,7 +15,7 @@ - [ ] Phone hotspot configured as backup (embedding model already cached = no network needed) ### Software -- [ ] Vestige binary installed: `vestige-mcp --version` shows `2.1.23` (or latest) +- [ ] Vestige v2.0 binary installed: `vestige-mcp --version` shows `2.0.0` - [ ] Claude Code installed and authenticated - [ ] Terminal font size: 18pt minimum (audience readability) - [ ] Browser zoom: 150% for dashboard views @@ -207,7 +205,7 @@ claude mcp add vestige vestige-mcp -s user ### [2:50-3:00] Close -> Vestige v2.1.23. Open source, AGPL-3.0. The repo is `samvallad33/vestige`. Come talk to me if you want to see the neuroscience under the hood. +> Vestige v2.0, "Cognitive Leap." Open source, AGPL-3.0. The repo is `samvallad33/vestige`. Come talk to me if you want to see the neuroscience under the hood. --- @@ -393,9 +391,9 @@ vestige-mcp --version # One command to install ``` -> This is what I've been building. I'm one person, I'm twenty-two years old, and I believe this is how AI memory should work — grounded in real science, running locally, open source. +> This is what I've been building for the past three months. I'm one person, I'm twenty-one years old, and I believe this is how AI memory should work — grounded in real science, running locally, open source. > -> Vestige v2.1.23. The repo is `github.com/samvallad33/vestige`. The dashboard is running at `localhost:3927`. I'll be around all three days — come find me if you want to talk about FSRS, or synaptic tagging, or why I think every AI assistant on the planet should have a forgetting curve. +> Vestige v2.0, "Cognitive Leap." The repo is `github.com/samvallad33/vestige`. The dashboard is running at `localhost:3927`. I'll be around all three days — come find me if you want to talk about FSRS, or synaptic tagging, or why I think every AI assistant on the planet should have a forgetting curve. > > Thank you. diff --git a/docs/launch/receipt-lock.md b/docs/launch/receipt-lock.md deleted file mode 100644 index a27df5a..0000000 --- a/docs/launch/receipt-lock.md +++ /dev/null @@ -1,167 +0,0 @@ -# Wave A Launch — Receipt Lock (v2.1.23) - -Primary viral hook. Post **before** the memory/science Show HN wave. Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md). - ---- - -## Hacker News — Show HN - -### Title (≤80 chars) - -``` -Show HN: Vestige – blocks coding agents from claiming "tests passed" without receipts -``` - -### First comment (body) - -``` -Hi HN, - -Your coding agent probably ends sessions with something like "all tests passed" or -"the build is green." I kept trusting that — until it wasn't true. - -I built Receipt Lock in Vestige (an MCP memory server I maintain). Before operational -claims become part of the final answer, Vestige checks them against structured -command receipts from the current transcript. No matching successful receipt → the -claim can be blocked and a local veto receipt is written (JSON + HTML under -~/.vestige/sanhedrin/). - -**What it is:** Optional Claude Code Cognitive Sandwich hooks + local MCP server. -Not cloud. Not "trust me bro" logging — inspectable receipts on disk. - -**Install (memory server — required base):** -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user - -**Enable Receipt Lock (optional):** -vestige sandwich install --enable-sanhedrin - -Sanhedrin verifier can point at any OpenAI-compatible endpoint (Ollama, MLX, hosted API). - -**And it also does real memory:** FSRS-6 spaced repetition, prediction error gating, -memory dreaming, 3D dashboard at localhost:3927. ~86K LOC Rust, 25 MCP tools, 1,200+ -tests, 22MB binary. 100% local after first embedding download. - -I'm 22, solo, AGPL-3.0. Repo: https://github.com/samvallad33/vestige -Comparison: https://github.com/samvallad33/vestige/blob/main/docs/comparison.md - -Happy to discuss false positive tuning, Sanhedrin presets, or why receipts beat vibes. -``` - ---- - -## r/ExperiencedDevs - -### Title - -``` -My coding agent kept saying "tests passed" when they hadn't. I added a receipt check before the summary ships. -``` - -### Body - -```markdown -**TL;DR:** Vestige Receipt Lock checks operational claims ("tests passed", "build green", "lint clean") against structured command receipts from the transcript. No receipt → block + local veto artifact. - -**The failure mode:** Agent runs partial checks, or hallucinates a green ending. You merge. CI breaks. You've seen this. - -**The fix:** Optional hooks (`vestige sandwich install --enable-sanhedrin`) + MCP memory server. When the model tries to assert verification without evidence, Vestige can veto and write `~/.vestige/sanhedrin/latest.html` so you can inspect what happened. - -**Not a replacement for CI.** It's a last-mile guard on *agent-authored* summaries in Claude Code. - -**Stack:** Rust, local, MCP. Same project also does FSRS-6 cognitive memory (decay, dreaming, contradiction tools) — I'll post that angle separately if people want the science side. - -```bash -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user -vestige sandwich install --enable-sanhedrin -``` - -GitHub: https://github.com/samvallad33/vestige - -What false positives are you seeing with agent verification claims? Curious if this matches your workflow. -``` - ---- - -## r/programming - -### Title - -``` -Open-source guard: coding agents can't claim "tests passed" without command receipts (local MCP, Rust) -``` - -### Body — use r/ExperiencedDevs body; add: - -```markdown -License: AGPL-3.0. v2.1.23. Stats: ~86K LOC, 25 tools, 22MB binary. -``` - ---- - -## X / Twitter thread (8 posts) - -1. Your coding agent ends with "tests passed." Did it run tests? Or did it summarize hope? - -2. I ship Receipt Lock in Vestige — checks operational claims against command receipts from the transcript. - -3. No matching successful receipt → claim blocked. Local veto receipt: `~/.vestige/sanhedrin/latest.html` - -4. Optional hooks. Local MCP server. Not cloud analytics. - -5. ```bash - npm i -g vestige-mcp-server@latest - claude mcp add vestige vestige-mcp -s user - vestige sandwich install --enable-sanhedrin - ``` - -6. Same binary also does FSRS-6 memory — decay, dreaming, 3D brain viz. Thread on that tomorrow. - -7. 22yo solo dev. AGPL. https://github.com/samvallad33/vestige - -8. What's the worst "green build" lie your agent told you? Reply — building the FAQ from real stories. - ---- - -## Lobste.rs - -### Title - -``` -Vestige Receipt Lock: local MCP guard against unverified "tests passed" agent claims -``` - -### Tags - -`rust` `programming` `security` - -### Body - -Use HN first comment (shorter). Link comparison.md. - ---- - -## Engagement playbook (Wave A) - -| Window | Action | -|--------|--------| -| 0–3h | Reply every comment within 30 min | -| Tone | Technical, humble, no "revolutionary" | -| Competitors | Acknowledge Mem0/Cursor memory; don't bash | -| CTA | Install + link comparison.md | -| Next | Schedule Wave B 48h after Wave A peaks | - -### DO NOT - -- "Game-changer" / "AI-powered" / "paradigm shift" -- Disparage Mem0 or Claude native memory -- Promise Receipt Lock replaces CI - ---- - -## Timing - -- **HN / Lobsters:** Tuesday or Wednesday, 8–10 AM US Eastern -- **Reddit:** Same day, +1–2h after HN -- **X:** Pin thread during HN peak diff --git a/docs/launch/reddit-cross-reference.md b/docs/launch/reddit-cross-reference.md index d03f5f6..eae7aaf 100644 --- a/docs/launch/reddit-cross-reference.md +++ b/docs/launch/reddit-cross-reference.md @@ -1,6 +1,4 @@ -# Reddit Launch Posts — cross_reference / deep_reference (v2.1.23) - -> Canonical install: [LAUNCH_STATS.md](../LAUNCH_STATS.md) — **no `sudo mv`**; use `npm install -g vestige-mcp-server@latest` +# Reddit Launch Posts — cross_reference Tool ## Post 1: r/ClaudeAI (Primary) @@ -80,8 +78,8 @@ Memory systems need to be SMARTER, not just bigger. That's what Vestige does — - **cross_reference** — the new tool that catches contradictions before they become wrong answers ### Stats: -- 25 MCP tools -- 1,200+ tests +- 22 MCP tools +- 746 tests, 0 failures - Zero `unsafe` code - Clean security audit (0 findings — AgentAudit verified) - Single 22MB Rust binary — no Docker, no PostgreSQL, no cloud @@ -89,8 +87,9 @@ Memory systems need to be SMARTER, not just bigger. That's what Vestige does — ### Install (30 seconds): ```bash -npm install -g vestige-mcp-server@latest -vestige health +# macOS Apple Silicon +npm install -g vestige-mcp-server +sudo mv vestige-mcp /usr/local/bin/ claude mcp add vestige vestige-mcp -s user ``` @@ -163,12 +162,12 @@ The AI sees the conflict. Picks the right one. Every time. **100% local. Your data never leaves your machine.** ```bash -npm install -g vestige-mcp-server@latest -vestige health +npm install -g vestige-mcp-server +sudo mv vestige-mcp /usr/local/bin/ claude mcp add vestige vestige-mcp -s user ``` -1,200+ tests. Zero unsafe code. AGPL-3.0. +746 tests. Zero unsafe code. Clean security audit. AGPL-3.0. GitHub: https://github.com/samvallad33/vestige @@ -176,7 +175,7 @@ GitHub: https://github.com/samvallad33/vestige ## Post 3: r/rust (Optional, technical audience) -**Title:** `I built a 22MB Rust binary that gives AI agents a brain — FSRS-6, 30 cognitive modules, Receipt Lock, 25 MCP tools. ~86K LOC, zero unsafe.` +**Title:** `I built a 22MB Rust binary that gives AI agents a brain — FSRS-6, 29 cognitive modules, 3D dashboard, and a new contradiction detection tool. 746 tests, zero unsafe.` --- @@ -189,7 +188,7 @@ The latest addition: `cross_reference` — pairwise contradiction detection acro - No runtime, no GC pauses during real-time search - `tokio::sync::Mutex` for the cognitive engine, `std::sync::Mutex` for SQLite reader/writer split - Zero `unsafe` blocks in the entire codebase -- `cargo test` runs 1,200+ tests across the workspace +- `cargo test` runs 746 tests in 11 seconds **Architecture:** ``` @@ -214,7 +213,7 @@ SQLite WAL + FTS5 + USearch HNSW Clean security audit. Parameterized SQL everywhere. CSP headers on the dashboard. Constant-time auth comparison (`subtle::ConstantTimeEq`). File permissions 0o600/0o700. GitHub: https://github.com/samvallad33/vestige -AGPL-3.0 | 1,200+ tests | ~86K LOC +AGPL-3.0 | 746 tests | 79K+ LOC --- diff --git a/docs/launch/show-hn.md b/docs/launch/show-hn.md index adc58ab..8cc5a95 100644 --- a/docs/launch/show-hn.md +++ b/docs/launch/show-hn.md @@ -1,7 +1,4 @@ -# Vestige v2.1.23 Launch — Show HN + Cross-Posts (Wave B: Memory) - -> **Wave A (Receipt Lock)** posts live in [receipt-lock.md](receipt-lock.md). Run Wave A first. -> Stats: [LAUNCH_STATS.md](../LAUNCH_STATS.md) +# Vestige v2.0 Launch — Show HN + Cross-Posts --- @@ -10,7 +7,7 @@ ### Title (76 chars) ``` -Show HN: Vestige v2.1.23 – FSRS-6 memory for AI agents + local Receipt Lock +Show HN: Vestige – FSRS-6 spaced repetition as long-term memory for AI agents ``` ### Body (first comment) @@ -60,7 +57,7 @@ retrieval. Written in Rust, 100% local, single 22MB binary. discover hidden connections and synthesize insights. Inspired by hippocampal replay during sleep. -**Dashboard (since v2.0, still core):** +**v2.0 adds:** - 3D neural visualization dashboard (SvelteKit + Three.js) — watch memories pulse when accessed, burst particles on creation, golden flash lines when @@ -77,10 +74,7 @@ retrieval. Written in Rust, 100% local, single 22MB binary. embedded via Rust's `include_dir!` macro. No Docker, no Node runtime, no external services. -**v2.1.23 adds Receipt Lock:** optional hooks that block operational claims like -"tests passed" unless matching command receipts exist in the transcript. - -**Numbers:** ~86,000 lines of Rust, 1,200+ tests, 30 cognitive modules, 25 MCP +**Numbers:** 77,840 lines of Rust, 734 tests, 29 cognitive modules, 21 MCP tools, search under 50ms for 1000 memories (SQLite FTS5 + USearch HNSW). **What it is NOT:** This is not RAG. RAG treats memory as a static database — @@ -93,7 +87,7 @@ The embedding model (Nomic Embed Text v1.5) runs locally via ONNX. After the first-run model download (~130MB), there are zero network requests. No telemetry, no analytics, no phoning home. -I've been using this daily and the experience is genuinely different. +I've been using this daily for 2 months and the experience is genuinely different. Claude remembers my coding patterns, my architectural decisions, my preferences. New sessions start with context instead of a blank slate. @@ -281,12 +275,12 @@ surprising and useful. ### r/rust -**Title:** `Vestige v2.1.23 — ~86K LOC Rust memory system with FSRS-6, Receipt Lock, and a 22MB binary` +**Title:** `Vestige v2.0 — 77K LOC Rust memory system with FSRS-6, HNSW, Axum WebSockets, and an embedded SvelteKit dashboard in a 22MB binary` **Body:** ```markdown -I've been building Vestige and just shipped v2.1.23. It's +I've been building Vestige for the past few months and just shipped v2.0. It's a cognitive memory system for AI agents that implements neuroscience-backed memory algorithms in pure Rust. @@ -320,7 +314,7 @@ memory algorithms in pure Rust. - **Release profile**: `lto = true`, `codegen-units = 1`, `opt-level = "z"`, `strip = true` gets the binary down to 22MB including embedded assets. -- **1,200+ tests** across workspace. Zero warnings on release gates. +- **734 tests**: 352 core + 378 mcp + 4 doctests. Zero warnings. **Architecture:** @@ -363,7 +357,7 @@ Happy to discuss any of the Rust architecture decisions. ### r/ClaudeAI -**Title:** `Vestige v2.1.23 — give Claude real long-term memory (FSRS-6) + optional Receipt Lock for fake "tests passed" claims` +**Title:** `Vestige v2.0 "Cognitive Leap" — give Claude real long-term memory with neuroscience-backed forgetting, a 3D dashboard, and 21 MCP tools` **Body:** @@ -391,7 +385,7 @@ locally on your machine. strength model, testing effect, synaptic tagging, spreading activation, context-dependent retrieval, memory dreaming. -**Highlights:** +**v2.0 new features:** - **3D Memory Dashboard** at localhost:3927/dashboard — watch Claude's mind in real-time. Memories pulse when accessed, burst particles on creation, golden @@ -407,8 +401,7 @@ locally on your machine. **Setup (2 minutes):** ```bash -npm install -g vestige-mcp-server@latest -vestige health +npm install -g vestige-mcp-server claude mcp add vestige vestige-mcp -s user ``` @@ -424,7 +417,7 @@ on Project X ended with a tricky race condition in the WebSocket handler. It's the difference between talking to someone with amnesia vs. someone who actually knows you. -25 MCP tools. ~86,000 lines of Rust. 1,200+ tests. Works with Claude Code, Claude +21 MCP tools. 77,840 lines of Rust. 734 tests. Works with Claude Code, Claude Desktop, Cursor, VS Code Copilot, JetBrains, Windsurf, and Xcode. Source: https://github.com/samvallad33/vestige @@ -436,7 +429,7 @@ Happy to answer questions or help with setup. ### r/LocalLLaMA -**Title:** `Vestige v2.1.23 — local-first AI memory with FSRS-6, Receipt Lock, zero cloud (~86K LOC Rust, 22MB binary)` +**Title:** `Vestige v2.0 — local-first AI memory server with FSRS-6 spaced repetition, ONNX embeddings, and zero cloud dependency (77K LOC Rust, 22MB binary)` **Body:** @@ -489,7 +482,7 @@ algorithms: - 3D force-directed memory graph with real-time WebSocket events - HyDE query expansion (template-based hypothetical document embeddings) - FSRS decay visualization with retention curves -- 1,200+ tests, 30 cognitive modules, 25 tools +- 734 tests, 29 cognitive modules, 21 tools - fastembed 5.11 with feature flags for Nomic v2 MoE + Qwen3 reranker **Performance:** @@ -542,7 +535,8 @@ This is a solo project — feedback, issues, and contributions are very welcome. implementations, some are engineering heuristics inspired by research) 3. 100% local, zero cloud — this is a feature, not a limitation 4. The 3D dashboard is a genuine exploration tool, not just eye candy -5. FSRS-6 + Receipt Lock — spaced repetition memory and optional agent claim verification +5. FSRS-6 is the differentiator — no other AI memory system uses real spaced + repetition ### What NOT to Say diff --git a/docs/marketing/LAUNCH_NOW.md b/docs/marketing/LAUNCH_NOW.md deleted file mode 100644 index 7e2b887..0000000 --- a/docs/marketing/LAUNCH_NOW.md +++ /dev/null @@ -1,35 +0,0 @@ -# LAUNCH NOW — Wave A (Receipt Lock) - -**Date started:** 2026-06-02 -**Copy-paste from:** [ready-to-post/](ready-to-post/) - -## 5-minute sequence - -1. Run preflight: - ```bash - ./scripts/marketing/preflight.sh - ``` - -2. **Hacker News** → https://news.ycombinator.com/submit - - URL: `https://github.com/samvallad33/vestige` - - Title: paste `ready-to-post/hn-title.txt` - - Post link, then immediately paste `ready-to-post/hn-first-comment.txt` as first comment - -3. **X** — paste `ready-to-post/x-thread.txt` (one tweet per numbered block) - -4. **r/ExperiencedDevs** — title from `reddit-experienceddevs-title.txt`, body from `reddit-experienceddevs.md` - -5. **r/programming** — same body + line: `License: AGPL-3.0. v2.1.23. ~86K LOC, 25 tools, 22MB binary.` - -6. Log URLs in [metrics-tracker.md](metrics-tracker.md) - -## After posting (30 min SLA on comments) - -```bash -vestige ingest "Wave A posted YYYY-MM-DD on HN Reddit X. Hook: agent fake tests passed. Log URLs in metrics-tracker." \ - --data-dir ~/.vestige-marketing --tags marketing,wave-a,vestige-launch -``` - -## 48h later → Wave B - -[wave-b-launch.md](wave-b-launch.md) + [show-hn.md](../launch/show-hn.md) memory angle diff --git a/docs/marketing/README.md b/docs/marketing/README.md deleted file mode 100644 index 75acb1e..0000000 --- a/docs/marketing/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Vestige Marketing Kit - -Everything needed to run the dual-wave launch and weekly growth loop. - -## Start here - -**Posting today?** → [LAUNCH_NOW.md](LAUNCH_NOW.md) (copy-paste from [ready-to-post/](ready-to-post/)) - -1. [LAUNCH_STATS.md](../LAUNCH_STATS.md) — canonical version, stats, install -2. [comparison.md](../comparison.md) — vs Mem0 / RAG / native memory -3. [launch/receipt-lock.md](../launch/receipt-lock.md) — **Wave A** copy (post first) -4. [launch/show-hn.md](../launch/show-hn.md) — **Wave B** copy -5. [wave-a-launch.md](wave-a-launch.md) / [wave-b-launch.md](wave-b-launch.md) — execution checklists - -## Assets - -| Path | Purpose | -|------|---------| -| [website/index.html](../website/index.html) | GitHub Pages landing | -| [assets/CAPTURE.md](assets/CAPTURE.md) | GIF/video capture | -| [demo-video-storyboard.md](demo-video-storyboard.md) | 60–90s video beats | - -## Ongoing - -| Path | Purpose | -|------|---------| -| [growth-engine/](growth-engine/) | Vestige-powered marketing agent setup | -| [metrics-tracker.md](metrics-tracker.md) | Weekly npm / stars / hooks | -| [mcp-registries.md](mcp-registries.md) | Directory submission packet | - -## Deploy landing page - -Push to `main` → GitHub Actions workflow `.github/workflows/pages.yml` publishes `docs/website/` to **https://samvallad33.github.io/vestige/** (enable Pages: Settings → Pages → GitHub Actions). diff --git a/docs/marketing/assets/.gitkeep b/docs/marketing/assets/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/marketing/assets/CAPTURE.md b/docs/marketing/assets/CAPTURE.md deleted file mode 100644 index 7f45315..0000000 --- a/docs/marketing/assets/CAPTURE.md +++ /dev/null @@ -1,40 +0,0 @@ -# Demo GIF / Video Capture Guide - -Record these on a machine with Vestige v2.1.23 installed and ~20 pre-loaded memories (see `docs/launch/demo-script.md` pre-load section). - -## Prerequisites - -```bash -npm install -g vestige-mcp-server@latest -vestige health -claude mcp add vestige vestige-mcp -s user -open http://localhost:3927/dashboard -``` - -## Assets to produce - -| File | Duration | What to show | -|------|----------|----------------| -| `receipt-lock.gif` | 8–12s loop | Agent claims "tests passed" → Sanhedrin veto → `~/.vestige/sanhedrin/latest.html` receipt | -| `dashboard-dream.gif` | 10–15s loop | Graph view → trigger dream in Claude → purple dream mode, golden connection lines | -| `memory-born.gif` | 5–8s | Feed tab: `MemoryCreated` WebSocket event + new node burst on graph | -| `demo-full.mp4` | 60–90s | Full script: `docs/launch/demo-script.md` Version 2 (3-minute cut to 90s) | - -## macOS capture (recommended) - -```bash -# Screen recording → convert to GIF (install: brew install ffmpeg) -ffmpeg -f avfoundation -i "1" -t 12 -vf "fps=10,scale=1280:-1" -y /tmp/vestige-rec.mov -ffmpeg -i /tmp/vestige-rec.mov -vf "fps=8,scale=960:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 docs/marketing/assets/dashboard-dream.gif -``` - -## Static fallback - -If GIFs are not ready for launch, export one PNG from the dashboard graph view: - -```bash -# Browser screenshot → save as: -docs/marketing/assets/dashboard-static.png -``` - -Commit GIFs when ready; README and landing page reference these paths. diff --git a/docs/marketing/assets/dashboard-placeholder.svg b/docs/marketing/assets/dashboard-placeholder.svg deleted file mode 100644 index 4a50448..0000000 --- a/docs/marketing/assets/dashboard-placeholder.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - Vestige 3D Memory Dashboard - Run vestige dashboard  capture GIF per docs/marketing/assets/CAPTURE.md - - - - - - diff --git a/docs/marketing/demo-video-storyboard.md b/docs/marketing/demo-video-storyboard.md deleted file mode 100644 index 98f2942..0000000 --- a/docs/marketing/demo-video-storyboard.md +++ /dev/null @@ -1,32 +0,0 @@ -# Demo Video Storyboard (60–90s) - -For `docs/marketing/assets/demo-full.mp4` and GIF exports. Full script: [demo-script.md](../../launch/demo-script.md) Version 2. - -## Beat sheet - -| Time | Visual | Audio / text overlay | -|------|--------|----------------------| -| 0:00–0:08 | 3D dashboard graph, nodes pulsing | "Your AI forgets everything between sessions." | -| 0:08–0:18 | Claude Code: "Remember I prefer TypeScript" → Feed: MemoryCreated | "Vestige stores it with prediction error gating — not a dumb bucket." | -| 0:18–0:30 | New session: "What are my language preferences?" → correct answer | "New session. Same brain." | -| 0:30–0:45 | Search → graph nodes pulse blue (spreading activation) | "Search runs a 7-stage cognitive pipeline." | -| 0:45–0:58 | Dream mode: purple wash, golden edges | "Dream consolidation finds connections you never typed." | -| 0:58–1:15 | Terminal: agent says "tests passed" → veto / sanhedrin HTML | "Receipt Lock: no receipt, no claim." | -| 1:15–1:25 | Terminal: install commands | `npm install -g vestige-mcp-server@latest` | -| 1:25–1:30 | Logo / github.com/samvallad33/vestige | "v2.1.23 · local · AGPL" | - -## Export targets - -| Asset | Path | -|-------|------| -| Full video | `docs/marketing/assets/demo-full.mp4` | -| Dashboard loop | `docs/marketing/assets/dashboard-dream.gif` | -| Receipt Lock loop | `docs/marketing/assets/receipt-lock.gif` | -| Memory create | `docs/marketing/assets/memory-born.gif` | - -Capture commands: [assets/CAPTURE.md](assets/CAPTURE.md) - -## Wave usage - -- **Wave A:** Ship `receipt-lock.gif` + beats 0:58–1:15 first -- **Wave B:** Ship `dashboard-dream.gif` + full `demo-full.mp4` diff --git a/docs/marketing/growth-engine/MARKETING-CLAUDE.md b/docs/marketing/growth-engine/MARKETING-CLAUDE.md deleted file mode 100644 index ef3aae0..0000000 --- a/docs/marketing/growth-engine/MARKETING-CLAUDE.md +++ /dev/null @@ -1,69 +0,0 @@ -# Vestige Marketing Agent Protocol - -You are the marketing operator for **Vestige** (v2.1.23). You have access to a **dedicated** Vestige MCP instance (`vestige-marketing` / `VESTIGE_DATA_DIR=~/.vestige-marketing`). Never confuse this with the user's dev memory. - -## Product facts (do not invent stats) - -Read [docs/LAUNCH_STATS.md](../../LAUNCH_STATS.md) before drafting. Current anchors: - -- ~86K LOC Rust, 25 MCP tools, 30 cognitive modules, 1,200+ tests, 22MB binary -- Install: `npm install -g vestige-mcp-server@latest` + `claude mcp add vestige vestige-mcp -s user` -- Wave A hook: **Receipt Lock** — blocks "tests passed" without command receipts -- Wave B product: **FSRS-6 cognitive memory**, dreaming, 3D dashboard -- Comparison: [docs/comparison.md](../../comparison.md) -- Author: Sam Valladares, 22, solo, AGPL-3.0 - -## Session start - -1. `session_context` with query: `vestige marketing launch hooks objections` -2. `deep_reference` if drafting factual claims about features or competitors -3. `contradictions` if messaging might conflict with prior brand guidelines - -## Voice - -- Technical, humble, specific — never "revolutionary", "game-changer", "AI-powered" -- Lead with **pain** (agent amnesia, fake green builds, contradicting memories) -- Reveal **tool** second -- Acknowledge Mem0, native Claude memory, RAG honestly — do not bash -- Neuroscience: cite real papers; admit heuristics where approximate - -## On user feedback - -- Winning hook / post → `memory` promote on that memory -- Flopped angle → `suppress` (not delete) -- New objection → `smart_ingest` with tags `marketing, objection` -- User correction → `smart_ingest` + demote wrong memory if needed - -## Weekly deliverables - -When asked for "weekly content": - -1. **One long-form** (800–1200 words): expand top objection OR one cognitive module OR Receipt Lock story -2. **3–5 short posts** (X/LinkedIn): each ≤280 chars or ≤2 short paragraphs -3. **One Reddit draft** (technical, humble title — pain first) -4. **Metrics summary** paragraph for ingest after user fills tracker - -## Channels (user posts manually) - -You draft only. User sends all posts and DMs to avoid bans and keep authenticity. - -| Channel | Style | -|---------|-------| -| HN | Show HN title ≤80 chars; first comment = full body; science-first | -| Reddit | Personal story + JSON output + install block; no "introducing my startup" | -| X | 8–12 tweet thread; hook tweet must stand alone | -| LinkedIn | Professional, link comparison.md | - -## End of week - -``` -dream with focus on marketing memories tagged vestige-launch from the last 7 days. -Return: top 3 hooks to promote, top 2 to suppress, one recommended post for next week. -``` - -## Hard rules - -- Do not claim Vestige replaces CI/CD or enterprise memory suites -- Do not fabricate download numbers — use metrics-tracker.md only -- Do not tell users `sudo mv` for install unless manual binary path failed -- Always include GitHub link: https://github.com/samvallad33/vestige diff --git a/docs/marketing/growth-engine/README.md b/docs/marketing/growth-engine/README.md deleted file mode 100644 index b2b4af5..0000000 --- a/docs/marketing/growth-engine/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Vestige Marketing Growth Engine - -Repeatable weekly loop: Vestige remembers what worked, Claude Code drafts what’s next, you approve and post manually. - -## One-time setup - -### 1. Dedicated marketing memory store - -```bash -mkdir -p ~/.vestige-marketing -``` - -Add a **second** MCP server entry (do not mix with dev memory): - -```bash -claude mcp add vestige-marketing vestige-mcp -s user \ - --env VESTIGE_DATA_DIR=$HOME/.vestige-marketing -``` - -If your client does not support env on `mcp add`, use a wrapper script: - -```bash -# ~/bin/vestige-mcp-marketing -#!/bin/bash -export VESTIGE_DATA_DIR="$HOME/.vestige-marketing" -exec vestige-mcp "$@" -``` - -### 2. Copy marketing agent instructions - -```bash -cp docs/marketing/growth-engine/MARKETING-CLAUDE.md ~/vestige-marketing-CLAUDE.md -``` - -In Claude Code for marketing sessions, include that file (or paste into project instructions). - -### 3. Seed baseline memories - -Open Claude Code with `vestige-marketing` connected and run: - -``` -Read docs/LAUNCH_STATS.md, docs/comparison.md, and docs/marketing/metrics-tracker.md. -smart_ingest each as separate marketing baseline memories with tags: marketing, baseline, vestige-launch. -``` - -## Weekly loop (≈2 hours) - -| Step | Who | Action | -|------|-----|--------| -| Mon AM | You | Fill `metrics-tracker.md` row | -| Mon | Agent | `session_context` with query "vestige marketing launch" | -| Mon | Agent | Draft 1 long-form + 3–5 shorts from last week's `promote`d hooks | -| Mon | You | Edit and post manually (HN/Reddit/X/LinkedIn) | -| Fri | You | Log engagement URLs + numbers | -| Fri | Agent | `smart_ingest` weekly metrics + objections | -| Fri | Agent | `dream` on tag `vestige-launch` for next week's angles | - -## Tool cheat sheet - -| Goal | Tool | -|------|------| -| Load brand voice + past wins | `session_context` | -| Save post results | `smart_ingest` | -| Recall winning hooks | `search` / `deep_reference` | -| Retire dead angles | `suppress` | -| Boost viral hook | `memory` action=promote | -| Weekly strategy | `dream` | - -## Dogfood story (meta-content) - -> "I use Vestige to market Vestige — marketing memories live in a separate data dir, FSRS promotes hooks that converted, suppress kills angles that flopped." - -Post this on X after Week 2 if metrics show engagement. - -## Files - -| File | Purpose | -|------|---------| -| [MARKETING-CLAUDE.md](MARKETING-CLAUDE.md) | Agent protocol | -| [../metrics-tracker.md](../metrics-tracker.md) | Weekly numbers | -| [../wave-a-launch.md](../wave-a-launch.md) | Receipt Lock execution | -| [../wave-b-launch.md](../wave-b-launch.md) | Memory wave execution | -| [../../launch/receipt-lock.md](../../launch/receipt-lock.md) | Wave A copy | -| [../../comparison.md](../../comparison.md) | Argument anchor | diff --git a/docs/marketing/mcp-registries.md b/docs/marketing/mcp-registries.md deleted file mode 100644 index aaa7a98..0000000 --- a/docs/marketing/mcp-registries.md +++ /dev/null @@ -1,57 +0,0 @@ -# MCP Registry & Directory Submissions - -Passive install channel — update listings whenever v2.1.x ships. Check off as you submit. - -## Submission packet (reuse everywhere) - -| Field | Value | -|-------|-------| -| Name | Vestige | -| Slug | `io.github.samvallad33/vestige` (npm `mcpName`) | -| Description | Local cognitive memory for MCP agents — FSRS-6 spaced repetition, prediction error gating, memory dreaming, 3D dashboard, optional Receipt Lock for agent verification claims. | -| Install | `npm install -g vestige-mcp-server@latest` then `claude mcp add vestige vestige-mcp -s user` | -| Repo | https://github.com/samvallad33/vestige | -| Homepage | https://samvallad33.github.io/vestige/ | -| License | AGPL-3.0-only | -| Transport | stdio (default); HTTP opt-in | -| Version | 2.1.23 | -| Tags | memory, mcp, claude, cursor, local-first, fsrs, neuroscience, rust | - -## Registries - -| Directory | URL | Status | Notes | -|-----------|-----|--------|-------| -| Glama | https://glama.ai/mcp/servers | [ ] Submit / refresh | Ownership metadata in repo (`cd496e5`) | -| mcp.so | https://mcp.so | [ ] Submit | Use submission packet | -| Smithery | https://smithery.ai | [ ] Submit | npm package + stdio command | -| PulseMCP | https://www.pulsemcp.com | [ ] Submit | | -| Awesome MCP Servers | https://github.com/punkpeye/awesome-mcp-servers | [ ] PR | Add under Memory / Knowledge | -| modelcontextprotocol/servers | https://github.com/modelcontextprotocol/servers | [ ] PR if accepted | Follow their CONTRIBUTING | -| Cursor directory | docs/integrations/cursor.md | [x] Doc exists | Link from Cursor forum / Discord | -| VS Code marketplace | N/A for MCP stdio | [ ] N/A | Use integrations/vscode.md in posts | - -## Awesome-MCP PR snippet - -```markdown -### Vestige -- **Description:** Local cognitive memory — FSRS-6 decay, dreaming, contradiction tools, optional Receipt Lock -- **Install:** `npm install -g vestige-mcp-server@latest` -- **Command:** `vestige-mcp` -- **Repo:** https://github.com/samvallad33/vestige -``` - -## After each listing goes live - -```bash -# Ingest into marketing Vestige -smart_ingest: "Listed Vestige on [REGISTRY] at [URL]. Version 2.1.23." -tags: marketing, registry, vestige-launch -``` - -## Editor-specific posts (optional) - -| Community | Action | -|-----------|--------| -| Cursor Discord #showcase | Link comparison.md + 30s dashboard GIF | -| Claude Code GitHub discussions | Receipt Lock angle + install | -| r/mcp | Neutral "new server" post after Wave B | diff --git a/docs/marketing/metrics-tracker.md b/docs/marketing/metrics-tracker.md deleted file mode 100644 index d995ba9..0000000 --- a/docs/marketing/metrics-tracker.md +++ /dev/null @@ -1,66 +0,0 @@ -# Vestige Growth Metrics Tracker - -**North star:** weekly `vestige-mcp-server` npm installs + evidence of active MCP connections (issues, "it works" posts). - -Update every **Monday**. Feed summary into marketing Vestige via `smart_ingest`. - -## How to fetch numbers - -```bash -# npm weekly downloads (approximate) -npm view vestige-mcp-server - -# GitHub stars -gh api repos/samvallad33/vestige --jq .stargazers_count - -# Optional: npm download chart -# https://www.npmjs.com/package/vestige-mcp-server -``` - -## Weekly log template - -Copy a row per week: - -| Week ending | npm downloads (total) | Stars | Stars Δ | Top channel | Top hook | Installs anecdote | Notes | -|-------------|----------------------|-------|---------|-------------|----------|-------------------|-------| -| 2026-06-02 | TBD | 542 | 0 | pre-launch | Receipt Lock / fake tests passed | setup complete | marketing instance seeded, ready for Wave A | -| 2026-06-09 | | | | | | | post Wave A week 1 | - -## Per-post log template - -| Date | Wave | Channel | Post URL | Engagement | Stars Δ (48h) | Objections | Action | -|------|------|---------|----------|------------|---------------|------------|--------| -| | A | HN | | | | | | - -## Objection → content flywheel - -When the same objection appears 3+ times, promote to permanent doc: - -| Objection | Response doc | -|-----------|----------------| -| "Isn't this just RAG?" | [comparison.md](../comparison.md) | -| "Claude has memory now" | comparison.md + Receipt Lock section | -| "AGPL?" | README + HN FAQ in show-hn.md | -| "77K LOC over-engineered" | show-hn.md FAQ | -| "FSRS gimmick?" | [SCIENCE.md](../SCIENCE.md) | - -## Agent ingest prompt (weekly) - -``` -smart_ingest: Vestige marketing week ending YYYY-MM-DD. -npm: X total (ΔY). Stars: N (ΔZ). -Best channel: [HN/Reddit/X]. -Best hook: [phrase]. -Top objection: [text]. -Next week: [one action]. -tags: marketing, metrics, vestige-launch -``` - -## Goals (first 8 weeks) - -| Milestone | Target | -|-----------|--------| -| Wave A HN front page | 100+ points | -| Stars | 542 → 800+ | -| npm weekly downloads | 2× baseline | -| Registry listings | 5+ MCP directories | diff --git a/docs/marketing/ready-to-post/hn-first-comment.txt b/docs/marketing/ready-to-post/hn-first-comment.txt deleted file mode 100644 index ba24fb5..0000000 --- a/docs/marketing/ready-to-post/hn-first-comment.txt +++ /dev/null @@ -1,32 +0,0 @@ -Hi HN, - -Your coding agent probably ends sessions with something like "all tests passed" or -"the build is green." I kept trusting that — until it wasn't true. - -I built Receipt Lock in Vestige (an MCP memory server I maintain). Before operational -claims become part of the final answer, Vestige checks them against structured -command receipts from the current transcript. No matching successful receipt → the -claim can be blocked and a local veto receipt is written (JSON + HTML under -~/.vestige/sanhedrin/). - -**What it is:** Optional Claude Code Cognitive Sandwich hooks + local MCP server. -Not cloud. Not "trust me bro" logging — inspectable receipts on disk. - -**Install (memory server — required base):** -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user - -**Enable Receipt Lock (optional):** -vestige sandwich install --enable-sanhedrin - -Sanhedrin verifier can point at any OpenAI-compatible endpoint (Ollama, MLX, hosted API). - -**And it also does real memory:** FSRS-6 spaced repetition, prediction error gating, -memory dreaming, 3D dashboard at localhost:3927. ~86K LOC Rust, 25 MCP tools, 1,200+ -tests, 22MB binary. 100% local after first embedding download. - -I'm 22, solo, AGPL-3.0. Repo: https://github.com/samvallad33/vestige -Comparison: https://github.com/samvallad33/vestige/blob/main/docs/comparison.md -Landing: https://samvallad33.github.io/vestige/ - -Happy to discuss false positive tuning, Sanhedrin presets, or why receipts beat vibes. diff --git a/docs/marketing/ready-to-post/hn-title.txt b/docs/marketing/ready-to-post/hn-title.txt deleted file mode 100644 index f01f5d8..0000000 --- a/docs/marketing/ready-to-post/hn-title.txt +++ /dev/null @@ -1 +0,0 @@ -Show HN: Vestige – blocks coding agents from claiming "tests passed" without receipts diff --git a/docs/marketing/ready-to-post/reddit-experienceddevs-title.txt b/docs/marketing/ready-to-post/reddit-experienceddevs-title.txt deleted file mode 100644 index 3aab4ba..0000000 --- a/docs/marketing/ready-to-post/reddit-experienceddevs-title.txt +++ /dev/null @@ -1 +0,0 @@ -My coding agent kept saying "tests passed" when they hadn't. I added a receipt check before the summary ships. diff --git a/docs/marketing/ready-to-post/reddit-experienceddevs.md b/docs/marketing/ready-to-post/reddit-experienceddevs.md deleted file mode 100644 index 410f87b..0000000 --- a/docs/marketing/ready-to-post/reddit-experienceddevs.md +++ /dev/null @@ -1,19 +0,0 @@ -**TL;DR:** Vestige Receipt Lock checks operational claims ("tests passed", "build green", "lint clean") against structured command receipts from the transcript. No receipt → block + local veto artifact. - -**The failure mode:** Agent runs partial checks, or hallucinates a green ending. You merge. CI breaks. You've seen this. - -**The fix:** Optional hooks (`vestige sandwich install --enable-sanhedrin`) + MCP memory server. When the model tries to assert verification without evidence, Vestige can veto and write `~/.vestige/sanhedrin/latest.html` so you can inspect what happened. - -**Not a replacement for CI.** It's a last-mile guard on *agent-authored* summaries in Claude Code. - -**Stack:** Rust, local, MCP. Same project also does FSRS-6 cognitive memory (decay, dreaming, contradiction tools) — I'll post that angle separately if people want the science side. - -```bash -npm install -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user -vestige sandwich install --enable-sanhedrin -``` - -GitHub: https://github.com/samvallad33/vestige - -What false positives are you seeing with agent verification claims? Curious if this matches your workflow. diff --git a/docs/marketing/ready-to-post/x-thread.txt b/docs/marketing/ready-to-post/x-thread.txt deleted file mode 100644 index cc81fec..0000000 --- a/docs/marketing/ready-to-post/x-thread.txt +++ /dev/null @@ -1,18 +0,0 @@ -1/8 Your coding agent ends with "tests passed." Did it run tests? Or did it summarize hope? - -2/8 I ship Receipt Lock in Vestige — checks operational claims against command receipts from the transcript. - -3/8 No matching successful receipt → claim blocked. Local veto receipt: ~/.vestige/sanhedrin/latest.html - -4/8 Optional hooks. Local MCP server. Not cloud analytics. - -5/8 -npm i -g vestige-mcp-server@latest -claude mcp add vestige vestige-mcp -s user -vestige sandwich install --enable-sanhedrin - -6/8 Same binary also does FSRS-6 memory — decay, dreaming, 3D brain viz. Thread on that tomorrow. - -7/8 22yo solo dev. AGPL. https://github.com/samvallad33/vestige - -8/8 What's the worst "green build" lie your agent told you? Reply — building the FAQ from real stories. diff --git a/docs/marketing/wave-a-launch.md b/docs/marketing/wave-a-launch.md deleted file mode 100644 index 74303d6..0000000 --- a/docs/marketing/wave-a-launch.md +++ /dev/null @@ -1,47 +0,0 @@ -# Wave A — Execution Checklist - -Copy-paste from [docs/launch/receipt-lock.md](../launch/receipt-lock.md). **You post manually.** - -## Pre-flight - -- [x] `docs/LAUNCH_STATS.md` numbers match README -- [x] `vestige health` passes on your machine -- [ ] GitHub Pages live: https://samvallad33.github.io/vestige/ (after `git push` + Pages enabled) -- [x] GIFs captured OR placeholder SVG acceptable for Wave A -- [x] Marketing Vestige seeded: `./scripts/marketing/setup-marketing-instance.sh` -- [ ] Ingest Wave A URLs after posting (see LAUNCH_NOW.md) - -## Day 1 — HN + Lobsters - -| Time (ET) | Channel | Artifact | -|-----------|---------|----------| -| 8:00 AM Tue/Wed | Hacker News Show HN | Title + first comment from `receipt-lock.md` | -| +30 min | Lobste.rs | Shorter HN body | -| 8:00–11:00 AM | HN comments | 30-min reply SLA | - -## Day 1–2 — Reddit + X - -| Time | Channel | Artifact | -|------|---------|----------| -| +1h | r/ExperiencedDevs | Full post in `receipt-lock.md` | -| +2h | r/programming | Same + stats line | -| +0h (parallel) | X thread | 8 tweets in `receipt-lock.md` | -| Pin | X profile | Thread link during HN peak | - -## Metrics to log (→ `metrics-tracker.md`) - -| Field | Value | -|-------|-------| -| Date | | -| Channel | | -| URL | | -| Upvotes / points | | -| Comments | | -| npm downloads (week delta) | | -| Stars delta (48h) | | -| Top objection | | -| Winning hook phrase | | - -## After Wave A - -Wait **48h** from HN peak, then run [wave-b-launch.md](wave-b-launch.md). diff --git a/docs/marketing/wave-b-launch.md b/docs/marketing/wave-b-launch.md deleted file mode 100644 index 8f1084e..0000000 --- a/docs/marketing/wave-b-launch.md +++ /dev/null @@ -1,45 +0,0 @@ -# Wave B — Execution Checklist - -Memory / neuroscience product wave. Cross-link Wave A HN thread if it performed. - -Sources: [show-hn.md](../launch/show-hn.md) (refresh before posting), [demo-script.md](../launch/demo-script.md), [comparison.md](../comparison.md). - -## Pre-flight - -- [ ] Dashboard GIF live (`docs/marketing/assets/dashboard-dream.gif`) -- [ ] 20 pre-loaded memories per demo-script pre-demo checklist -- [ ] Wave A metrics logged in `metrics-tracker.md` - -## Day 3 — Show HN (memory angle) OR skip if Wave A HN was same week - -If Wave A used Show HN title for Receipt Lock, **do not** second Show HN same week. Use Reddit + X only for Wave B. - -| Channel | Focus | -|---------|-------| -| r/ClaudeAI | MCP memory, FSRS-6, dashboard, 2-min install | -| r/LocalLLaMA | Local-first, zero cloud, ONNX embeddings | -| r/rust | Architecture, 86K LOC, 25 tools, zero unsafe | - -## Day 4–5 — Content - -| Channel | Artifact | -|---------|----------| -| X | 10-tweet thread: "RAG is not memory" + FSRS + dream GIF | -| LinkedIn | Link to comparison.md + dashboard GIF | -| Blog | Optional: publish refreshed `docs/launch/blog-post.md` on dev.to / personal site | - -## r/ClaudeAI title (v2.1.23) - -``` -Vestige v2.1.23 — FSRS-6 memory for Claude Code + optional Receipt Lock when agents fake "tests passed" -``` - -Body: Use r/ClaudeAI section from refreshed `show-hn.md` + install block from LAUNCH_STATS. - -## Cross-link line (if Wave A performed) - -> Earlier this week I posted about Receipt Lock (agents claiming tests passed without receipts). Same project — the memory engine underneath: [link] - -## Metrics - -Same table as wave-a-launch.md. Tag `wave=B` in marketing Vestige ingest. diff --git a/docs/website/index.html b/docs/website/index.html deleted file mode 100644 index 7246339..0000000 --- a/docs/website/index.html +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - Vestige — Local memory and receipts for AI agents - - - - - - -
-

- v2.1.23 - 25 MCP tools - ~86K LOC Rust - AGPL-3.0 -

- -

Your agent forgets yesterday.
It can lie about today.

-

Vestige is a local MCP memory server: real FSRS-6 forgetting and consolidation, plus optional Receipt Lock that blocks “tests passed” without command receipts.

- - 
npm install -g vestige-mcp-server@latest
-claude mcp add vestige vestige-mcp -s user
- GitHub → - -

Wave A: Receipt Lock

-
Coding agents finish with confident summaries. Vestige checks operational claims against structured command receipts before they become your final answer.
-

If the agent claims verification happened but no matching successful receipt exists, the claim can be blocked and an inspectable local veto is written to ~/.vestige/sanhedrin/.

- 
vestige sandwich install --enable-sanhedrin
-

Receipt Lock docs

- -

Wave B: A brain, not a bucket

- Vestige 3D memory dashboard -

Memories decay on FSRS-6 curves. Search strengthens them (Testing Effect). dream consolidates offline. The 3D dashboard shows pulses, connections, and dream replay in real time.

-

vestige dashboard → localhost:3927/dashboard

- -

Vestige vs the rest

- - - - - - - - - -
RAGNative AI memoryVestige
ForgettingNoneOpaqueFSRS-6
Local / privateVariesCloud100% local
ContradictionsBoth chunksNodeep_reference
Fake “tests passed”N/AN/AReceipt Lock
VisualizationNoneNone3D graph
-

Full comparison (Mem0, RAG, native)

- -

Install

- 
npm install -g vestige-mcp-server@latest
-vestige health
-claude mcp add vestige vestige-mcp -s user
-

Also works with Codex, Cursor, VS Code Copilot, JetBrains, Windsurf, Xcode. See integration guides.

- - -
- - diff --git a/package.json b/package.json index 1057869..9d759a6 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,7 @@ "license": "AGPL-3.0-only", "repository": { "type": "git", - "url": "https://github.com/samvallad33/vestige", - "homepage": "https://samvallad33.github.io/vestige/" + "url": "https://github.com/samvallad33/vestige" }, "scripts": { "build:mcp": "cargo build --release --package vestige-mcp", diff --git a/packages/vestige-mcp-npm/README.md b/packages/vestige-mcp-npm/README.md index b7a01c2..98e6575 100644 --- a/packages/vestige-mcp-npm/README.md +++ b/packages/vestige-mcp-npm/README.md @@ -1,12 +1,8 @@ # vestige-mcp-server -**v2.1.23** — Vestige MCP Server: local cognitive memory and optional Receipt Lock for MCP-compatible AI agents. +Vestige MCP Server - A synthetic hippocampus for AI assistants. -- **Memory:** FSRS-6 spaced repetition, prediction error gating, dreaming, 3D dashboard -- **Receipt Lock:** blocks "tests passed" / "build green" without command receipts (optional hooks) -- **Stats:** ~86K LOC Rust · 25 tools · 1,200+ tests · 22MB binary · 100% local - -Homepage: https://samvallad33.github.io/vestige/ · Repo: https://github.com/samvallad33/vestige +Built on 130 years of cognitive science research, Vestige provides biologically-inspired memory that decays, strengthens, and consolidates like the human mind. ## Installation @@ -58,25 +54,6 @@ codex mcp add vestige -- vestige-mcp Then restart your MCP client. -## Optional Receipt Lock for Claude Code - -Receipt Lock is part of Vestige's optional Cognitive Sandwich hook layer. Normal -MCP memory stays lightweight and local. If you want claim checking for summaries -like "tests passed" or "lint is clean," enable Sanhedrin and point it at any -OpenAI-compatible chat endpoint: - -```bash -vestige sandwich install --enable-sanhedrin - -vestige sandwich install \ - --enable-sanhedrin \ - --sanhedrin-endpoint=http://127.0.0.1:11434/v1/chat/completions \ - --sanhedrin-model=qwen2.5:14b -``` - -If a claim is missing command evidence, Vestige writes local receipts under -`~/.vestige/sanhedrin/` so the veto is inspectable instead of opaque. - ## Usage with Claude Desktop Add to your Claude Desktop configuration: @@ -109,7 +86,6 @@ vestige sandwich install # Manage optional Claude Code hook files ## Features - **FSRS-6 Algorithm**: State-of-the-art spaced repetition for optimal memory retention -- **Receipt Lock**: Optional command-receipt checking for test/build/lint/typecheck claims - **Dual-Strength Memory**: Bjork & Bjork (1992) - Storage + Retrieval strength model - **Synaptic Tagging**: Memories become important retroactively (Frey & Morris 1997) - **Semantic Search**: Local embeddings via nomic-embed-text-v1.5 (768 dimensions) diff --git a/packages/vestige-mcp-npm/package.json b/packages/vestige-mcp-npm/package.json index 272681e..1ff860f 100644 --- a/packages/vestige-mcp-npm/package.json +++ b/packages/vestige-mcp-npm/package.json @@ -29,7 +29,6 @@ "type": "git", "url": "git+https://github.com/samvallad33/vestige.git" }, - "homepage": "https://samvallad33.github.io/vestige/", "engines": { "node": ">=18" }, diff --git a/scripts/marketing/preflight.sh b/scripts/marketing/preflight.sh deleted file mode 100755 index 590464a..0000000 --- a/scripts/marketing/preflight.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Pre-launch checks before Wave A posts. -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -cd "${ROOT}" - -FAIL=0 -pass() { echo "OK $1"; } -fail() { echo "FAIL $1"; FAIL=1; } - -echo "=== Vestige Launch Preflight ===" - -command -v vestige-mcp >/dev/null && pass "vestige-mcp on PATH" || fail "vestige-mcp not found" -command -v npm >/dev/null && pass "npm available" || fail "npm missing" - -VER="$(vestige-mcp --version 2>/dev/null || true)" -[[ "${VER}" == *"2.1."* ]] && pass "version ${VER}" || fail "unexpected version: ${VER}" - -vestige health >/dev/null 2>&1 && pass "vestige health" || fail "vestige health failed" - -[[ -f docs/LAUNCH_STATS.md ]] && pass "LAUNCH_STATS.md" || fail "missing LAUNCH_STATS.md" -[[ -f docs/launch/receipt-lock.md ]] && pass "receipt-lock.md" || fail "missing receipt-lock.md" -[[ -f docs/website/index.html ]] && pass "landing page source" || fail "missing website" -[[ -f .github/workflows/pages.yml ]] && pass "pages workflow" || fail "missing pages workflow" - -if curl -sf --max-time 5 "https://samvallad33.github.io/vestige/" >/dev/null 2>&1; then - pass "GitHub Pages live" -else - echo "WARN GitHub Pages not live yet — push main and enable Pages → GitHub Actions" -fi - -MARKETING_DIR="${HOME}/.vestige-marketing" -if [[ -d "${MARKETING_DIR}" ]]; then - pass "marketing data dir exists" -else - echo "WARN run scripts/marketing/setup-marketing-instance.sh" -fi - -echo "" -if [[ "${FAIL}" -eq 0 ]]; then - echo "Preflight PASSED — ready for Wave A" - exit 0 -else - echo "Preflight FAILED — fix items above" - exit 1 -fi diff --git a/scripts/marketing/seed-baseline-memories.sh b/scripts/marketing/seed-baseline-memories.sh deleted file mode 100755 index 9b2a384..0000000 --- a/scripts/marketing/seed-baseline-memories.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Seed marketing Vestige with launch facts (separate from dev memory). -set -euo pipefail - -MARKETING_DIR="${VESTIGE_MARKETING_DIR:-$HOME/.vestige-marketing}" -TAGS="marketing,baseline,vestige-launch" - -ingest() { - vestige ingest "$1" --data-dir "${MARKETING_DIR}" --tags "${TAGS}" --node-type note -} - -echo "Seeding into ${MARKETING_DIR}..." - -ingest "Vestige v2.1.23 launch stats: ~86K LOC Rust, 25 MCP tools, 30 cognitive modules, 1200+ tests, 22MB binary, AGPL-3.0, npm vestige-mcp-server@latest, homepage samvallad33.github.io/vestige" - -ingest "Wave A hook Receipt Lock: block operational claims like tests passed or build green unless matching command receipts exist. Optional vestige sandwich install --enable-sanhedrin. Veto receipts at ~/.vestige/sanhedrin/" - -ingest "Wave B product: FSRS-6 spaced repetition memory, prediction error gating, memory dreaming, 3D dashboard localhost:3927, deep_reference contradictions, 100 percent local after embedding download" - -ingest "Canonical install: npm install -g vestige-mcp-server@latest && vestige health && claude mcp add vestige vestige-mcp -s user. Do NOT tell users sudo mv unless manual binary install failed." - -ingest "Messaging guardrails: no revolutionary game-changer AI-powered. Acknowledge Mem0 RAG native Claude memory honestly. Lead pain first tool second. Author Sam Valladares age 22 solo." - -ingest "North star metric: weekly npm installs vestige-mcp-server and active MCP connections not stars alone. Track in docs/marketing/metrics-tracker.md" - -ingest "Comparison anchor docs/comparison.md: RAG is retrieval, Vestige is cognitive lifecycle with forgetting consolidation Receipt Lock. Mem0 is cloud API Vestige is local AGPL." - -vestige stats --data-dir "${MARKETING_DIR}" 2>/dev/null || true -echo "Baseline seed complete." diff --git a/scripts/marketing/setup-marketing-instance.sh b/scripts/marketing/setup-marketing-instance.sh deleted file mode 100755 index e465c06..0000000 --- a/scripts/marketing/setup-marketing-instance.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# One-time setup: dedicated Vestige store + Claude Code MCP entry for marketing. -set -euo pipefail - -MARKETING_DIR="${VESTIGE_MARKETING_DIR:-$HOME/.vestige-marketing}" -BIN_DIR="${HOME}/.local/bin" -WRAPPER="${BIN_DIR}/vestige-mcp-marketing" - -echo "==> Marketing data dir: ${MARKETING_DIR}" -mkdir -p "${MARKETING_DIR}" - -if ! command -v vestige-mcp >/dev/null 2>&1; then - echo "Install Vestige first: npm install -g vestige-mcp-server@latest" - exit 1 -fi - -mkdir -p "${BIN_DIR}" -cat > "${WRAPPER}" < Wrapper: ${WRAPPER}" - -if command -v claude >/dev/null 2>&1; then - if claude mcp list 2>/dev/null | grep -q vestige-marketing; then - echo "==> claude mcp: vestige-marketing already registered" - else - claude mcp add vestige-marketing "${WRAPPER}" -s user - echo "==> Added: claude mcp add vestige-marketing ${WRAPPER} -s user" - fi -else - echo "==> Claude Code not found — register manually:" - echo " claude mcp add vestige-marketing ${WRAPPER} -s user" -fi - -echo "==> Seeding baseline memories..." -"$(dirname "$0")/seed-baseline-memories.sh" - -echo "" -echo "Done. Open Claude Code with MARKETING-CLAUDE.md:" -echo " cp docs/marketing/growth-engine/MARKETING-CLAUDE.md ~/vestige-marketing-CLAUDE.md" -echo " vestige health --data-dir ${MARKETING_DIR}" From 5aa261398d6290d0cc1f966431897fd57676e118 Mon Sep 17 00:00:00 2001 From: Luc Lauzon <128917870+randomnimbus@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:24:33 -0600 Subject: [PATCH 16/38] feat(mcp/search): add optional tag_prefix post-filter (#68) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an optional `tag_prefix` string parameter to the `search` MCP tool. When set, only results that carry at least one tag whose value starts with the prefix are returned (case-sensitive, matching the existing exact-tag semantics in memory_timeline / export / gc). Motivation: external consumers that need "all memories tagged `:*`" (e.g. `meeting:standup`, `meeting:1-on-1`) currently have three paths, all bad: (i) export everything and filter client-side (heavy), (ii) enumerate the prefix space and pass exact tags as a list (impractical for open-set tag classes), or (iii) read SQLite directly (an anti-pattern that couples consumers to internal schema). This PR closes that gap with a minimal, additive surface. Implementation note: filter runs at the MCP layer, NOT in the storage predicate. Rationale: (a) leaves crates/vestige-core/src/storage/ untouched, avoiding collision with PR #61's storage-trait extraction; (b) `SearchResult.node.tags` is already loaded from the same JSON-array column the brief's proposed SQL would scan, so the post-filter is functionally equivalent; (c) post-filter applies BEFORE the reranker so the cross-encoder does not waste cycles on memories the caller will not receive, and BEFORE strengthen-on-access so dropped results do not get a testing-effect boost they did not earn. Headroom: when tag_prefix is set, the hybrid path doubles its overfetch multiplier (capped at the existing 100 ceiling) and the concrete path fetches 3x its normal limit, both to leave the post-filter enough pool to still return ~limit results after thinning. The Stage 0 keyword-priority merge also re-applies the prefix filter so it cannot re-introduce filtered-out memories. Backwards-compat: parameter is optional, defaults to None; every existing call shape and response shape is unchanged. Tests: - tags_match_prefix unit (prefix-vs-substring, case-sensitivity, tagless-memory semantics, empty-prefix corner case) - schema introspection (property present, type=string, not required) - hybrid-path filter excludes non-matching tag-classes - hybrid-path filter excludes tagless memories - backwards-compat: no tag_prefix → behavior unchanged - concrete-path filter (literal-query branch) honors tag_prefix Closes a gap surfaced in the knowledge-mgmt-sota-uplift initiative (KMSU Session 89 audit; ~3,300-memory production Vestige). --- .../vestige-mcp/src/tools/search_unified.rs | 276 +++++++++++++++++- 1 file changed, 272 insertions(+), 4 deletions(-) diff --git a/crates/vestige-mcp/src/tools/search_unified.rs b/crates/vestige-mcp/src/tools/search_unified.rs index 9faf961..3aea11f 100644 --- a/crates/vestige-mcp/src/tools/search_unified.rs +++ b/crates/vestige-mcp/src/tools/search_unified.rs @@ -92,6 +92,10 @@ pub fn schema() -> Value { "type": "boolean", "description": "Force literal/concrete search. Skips semantic expansion, FSRS reweighting, spreading activation, and cognitive side effects. Auto-enabled for quoted strings, env vars, UUIDs, paths, and code identifiers.", "default": false + }, + "tag_prefix": { + "type": "string", + "description": "Optional tag-prefix filter. When set, only results carrying at least one tag whose value starts with this prefix are returned (case-sensitive). Example: tag_prefix=\"meeting:\" matches memories tagged 'meeting:standup', 'meeting:1-on-1', etc. Applied as a post-filter; combine with a larger 'limit' if you expect heavy thinning." } }, "required": ["query"] @@ -120,6 +124,8 @@ struct SearchArgs { #[serde(alias = "retrieval_mode")] retrieval_mode: Option, concrete: Option, + #[serde(alias = "tag_prefix")] + tag_prefix: Option, } /// Execute unified search with 7-stage cognitive pipeline. @@ -183,19 +189,43 @@ pub async fn execute( .concrete .unwrap_or_else(|| is_literal_query(&args.query)); if concrete { + // When a tag_prefix is requested, fetch a larger pool so the + // post-filter has enough headroom to still return ~limit results + // after thinning. Cap at the same upper bound the underlying SQL + // path uses elsewhere (100). + let concrete_fetch_limit = if args.tag_prefix.is_some() { + (limit * 3).min(100) + } else { + limit + }; let results = storage .concrete_search_filtered( &args.query, - limit, + concrete_fetch_limit, args.include_types.as_deref(), args.exclude_types.as_deref(), ) .map_err(|e| e.to_string())?; - let ids: Vec<&str> = results.iter().map(|r| r.node.id.as_str()).collect(); + // Apply tag_prefix post-filter BEFORE strengthen-on-access so + // results the caller did not actually receive do not get a + // testing-effect boost. + let filtered_results: Vec<&vestige_core::SearchResult> = match args.tag_prefix.as_deref() { + Some(prefix) => results + .iter() + .filter(|r| tags_match_prefix(&r.node.tags, prefix)) + .take(limit as usize) + .collect(), + None => results.iter().collect(), + }; + + let ids: Vec<&str> = filtered_results + .iter() + .map(|r| r.node.id.as_str()) + .collect(); let _ = storage.strengthen_batch_on_access(&ids); - let mut formatted: Vec = results + let mut formatted: Vec = filtered_results .iter() .filter(|r| r.node.retention_strength >= min_retention) .map(|r| format_search_result(r, detail_level)) @@ -297,7 +327,11 @@ pub async fn execute( "exhaustive" => 5, // Deep overfetch for maximum recall _ => 3, // Balanced default }; - let overfetch_limit = (limit * overfetch_multiplier).min(100); // Cap at 100 to avoid excessive DB load + // When a tag_prefix filter is requested, double the overfetch (capped at + // the same 100 ceiling) so the post-filter has enough headroom to still + // return ~limit results after thinning. + let tag_prefix_multiplier = if args.tag_prefix.is_some() { 2 } else { 1 }; + let overfetch_limit = (limit * overfetch_multiplier * tag_prefix_multiplier).min(100); // Cap at 100 to avoid excessive DB load let results = storage .hybrid_search_filtered( @@ -326,10 +360,26 @@ pub async fn execute( }) .collect(); + // Apply tag_prefix post-filter BEFORE the reranker so the (expensive) + // cross-encoder does not waste cycles on memories the caller will not + // receive. The Stage 0 keyword-priority merge below also respects the + // filter when applied, since merged items must have survived this step + // OR be re-introduced from keyword_priority_results (which we re-filter). + if let Some(prefix) = args.tag_prefix.as_deref() { + filtered_results.retain(|r| tags_match_prefix(&r.node.tags, prefix)); + } + // ==================================================================== // Dedup: merge Stage 0 keyword-priority results into Stage 1 results // ==================================================================== for kp in &keyword_priority_results { + // Respect tag_prefix here too — Stage 0 ran without it and can + // re-introduce filtered-out memories on the "new result" branch. + if let Some(prefix) = args.tag_prefix.as_deref() + && !tags_match_prefix(&kp.node.tags, prefix) + { + continue; + } if let Some(existing) = filtered_results .iter_mut() .find(|r| r.node.id == kp.node.id) @@ -781,6 +831,18 @@ fn is_literal_query(query: &str) -> bool { .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') } +/// Returns `true` when the given tag list contains at least one tag whose +/// string value starts with `prefix`. Empty prefix matches every result with +/// at least one tag (and never matches a tagless result). +/// +/// Case-sensitive by design: the existing tag-match semantics in +/// `memory_timeline` / `export` / `gc` are exact-match (case-sensitive), so +/// keeping this consistent avoids surprise. Operators wanting case-insensitive +/// prefix-search should normalize tags at ingest time. +fn tags_match_prefix(tags: &[String], prefix: &str) -> bool { + tags.iter().any(|t| t.starts_with(prefix)) +} + /// Format a search result based on the requested detail level. fn format_search_result(r: &vestige_core::SearchResult, detail_level: &str) -> Value { match detail_level { @@ -1531,4 +1593,210 @@ mod tests { assert_eq!(tb["minimum"], 100); assert_eq!(tb["maximum"], 100000); } + + // ======================================================================== + // TAG_PREFIX TESTS (PR1) + // ======================================================================== + + #[test] + fn test_tags_match_prefix_unit() { + let with_meeting = vec!["meeting:standup".to_string(), "team".to_string()]; + let without_meeting = vec!["adhoc".to_string(), "team".to_string()]; + let tagless: Vec = vec![]; + + assert!(tags_match_prefix(&with_meeting, "meeting:")); + assert!(!tags_match_prefix(&without_meeting, "meeting:")); + // Empty prefix matches when any tag exists; never matches a tagless + // memory. This preserves the "tag_prefix is a filter, not a default + // wildcard" semantics — a tagless memory has no tag-prefix to satisfy. + assert!(tags_match_prefix(&with_meeting, "")); + assert!(!tags_match_prefix(&tagless, "")); + // Case-sensitive (consistent with existing exact-tag matching). + assert!(!tags_match_prefix(&with_meeting, "Meeting:")); + // Prefix must match from the start, not anywhere in the tag value. + assert!(!tags_match_prefix(&with_meeting, "standup")); + } + + #[test] + fn test_schema_has_tag_prefix() { + let schema_value = schema(); + let tp = &schema_value["properties"]["tag_prefix"]; + assert!(tp.is_object(), "tag_prefix property must be present"); + assert_eq!(tp["type"], "string"); + // tag_prefix is NOT required. + let required = schema_value["required"].as_array().unwrap(); + assert!(!required.contains(&serde_json::json!("tag_prefix"))); + } + + /// Helper that ingests a memory with specific tags. The base + /// `ingest_test_content` helper passes `tags: vec![]`, which is fine + /// for legacy tests but not for tag_prefix coverage. + async fn ingest_with_tags( + storage: &Arc, + content: &str, + tags: Vec<&str>, + ) -> String { + let input = IngestInput { + content: content.to_string(), + node_type: "fact".to_string(), + source: None, + sentiment_score: 0.0, + sentiment_magnitude: 0.0, + tags: tags.into_iter().map(String::from).collect(), + valid_from: None, + valid_until: None, + }; + let node = storage.ingest(input).unwrap(); + node.id + } + + #[tokio::test] + async fn test_search_tag_prefix_filters_results() { + let (storage, _dir) = test_storage().await; + // Three memories matching the query semantically, only two carry + // the meeting:* tag-class. + ingest_with_tags( + &storage, + "Standup discussion about Q3 roadmap blockers", + vec!["meeting:standup", "roadmap"], + ) + .await; + ingest_with_tags( + &storage, + "1-on-1 sync on roadmap clarity and ownership", + vec!["meeting:1-on-1", "roadmap"], + ) + .await; + ingest_with_tags( + &storage, + "Solo note: roadmap dependency graph audit", + vec!["adhoc", "roadmap"], + ) + .await; + + let args = serde_json::json!({ + "query": "roadmap", + "tag_prefix": "meeting:", + "min_similarity": 0.0 + }); + let result = execute(&storage, &test_cognitive(), Some(args)).await; + assert!(result.is_ok(), "{:?}", result); + let value = result.unwrap(); + let results = value["results"].as_array().unwrap(); + // Both meeting:* memories should land; the adhoc one should not. + for r in results { + let tags = r["tags"].as_array().expect("tags must be present"); + let has_meeting = tags + .iter() + .any(|t| t.as_str().is_some_and(|s| s.starts_with("meeting:"))); + assert!(has_meeting, "result lacks meeting:* tag: {}", r); + } + // We expect 2 matches given the corpus above. The exact count + // depends on the cognitive pipeline's competition/suppression + // dynamics, so assert a lower bound. + assert!( + results.len() >= 1, + "tag_prefix should leave at least one meeting:* result, got {}", + results.len() + ); + } + + #[tokio::test] + async fn test_search_tag_prefix_excludes_tagless_memories() { + let (storage, _dir) = test_storage().await; + ingest_with_tags( + &storage, + "Notebook entry about consolidation cycles", + vec![], // tagless + ) + .await; + ingest_with_tags( + &storage, + "Project note about consolidation cycles", + vec!["project:vestige"], + ) + .await; + + let args = serde_json::json!({ + "query": "consolidation", + "tag_prefix": "project:", + "min_similarity": 0.0 + }); + let result = execute(&storage, &test_cognitive(), Some(args)).await; + assert!(result.is_ok()); + let value = result.unwrap(); + let results = value["results"].as_array().unwrap(); + for r in results { + let tags = r["tags"].as_array().expect("tags must be present"); + let has_project = tags + .iter() + .any(|t| t.as_str().is_some_and(|s| s.starts_with("project:"))); + assert!(has_project, "tagless or non-project result leaked: {}", r); + } + } + + #[tokio::test] + async fn test_search_without_tag_prefix_unchanged() { + // Backwards-compat: same corpus, same query, no tag_prefix → all + // results pass through regardless of tag composition. This is the + // load-bearing test for additive-only behavior. + let (storage, _dir) = test_storage().await; + ingest_with_tags(&storage, "Notebook entry about audit cycles", vec![]).await; + ingest_with_tags( + &storage, + "Project note about audit cycles", + vec!["project:audit"], + ) + .await; + + let args = serde_json::json!({ + "query": "audit", + "min_similarity": 0.0 + }); + let result = execute(&storage, &test_cognitive(), Some(args)).await; + assert!(result.is_ok()); + let value = result.unwrap(); + let results = value["results"].as_array().unwrap(); + // Both should be retrievable since no tag_prefix is set. + assert!( + results.len() >= 1, + "expected at least one result with no tag_prefix" + ); + } + + #[tokio::test] + async fn test_search_tag_prefix_concrete_path() { + // Concrete-search path (literal query) must also honor tag_prefix. + let (storage, _dir) = test_storage().await; + ingest_with_tags( + &storage, + "OPENAI_API_KEY rotation playbook for meetings", + vec!["meeting:ops"], + ) + .await; + ingest_with_tags( + &storage, + "OPENAI_API_KEY rotation playbook for solo audits", + vec!["adhoc"], + ) + .await; + + let args = serde_json::json!({ + "query": "OPENAI_API_KEY", + "concrete": true, + "tag_prefix": "meeting:" + }); + let result = execute(&storage, &test_cognitive(), Some(args)).await; + assert!(result.is_ok(), "{:?}", result); + let value = result.unwrap(); + assert_eq!(value["method"], "concrete"); + let results = value["results"].as_array().unwrap(); + for r in results { + let tags = r["tags"].as_array().expect("tags must be present"); + let has_meeting = tags + .iter() + .any(|t| t.as_str().is_some_and(|s| s.starts_with("meeting:"))); + assert!(has_meeting, "concrete result lacks meeting:* tag: {}", r); + } + } } From b01269db226fd44140cae915d0dc8fb04fb90b10 Mon Sep 17 00:00:00 2001 From: Luc Lauzon <128917870+randomnimbus@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:24:42 -0600 Subject: [PATCH 17/38] feat(mcp/system_status): add optional schema_introspection flag (#69) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(mcp/system_status): add optional schema_introspection flag Adds an optional `schema_introspection: bool` parameter to the `system_status` MCP tool. When set to true, the response gains a `schema` block carrying: - `schemaVersion` (u32) — highest applied migration, mirrors `Storage::current_schema_version` (now exposed via a typed public method). - `schemaVersionAppliedAt` (RFC3339, optional) — timestamp the current schema_version row was applied. - `tables` ([{name, rows, columns}]) — per-table row count + column list, walked over the canonical PORTABLE_USER_DATA_TABLES set so the surface stays stable across migrations rather than enumerating arbitrary sqlite_master rows. - `embeddingNullCount` (i64) — count of knowledge_nodes with NO row in node_embeddings. Distinct from MemoryStats.nodes_with_embeddings (which keys off the `has_embedding` flag column), so audit scripts can detect drift between the flag and the join-based truth. - `activeEmbeddingModel` (string, optional) + `activeEmbeddingDimensions` (u32, optional) — mirrors the existing MemoryStats active-model fields, included here so audits get schema_version + active model in a single round-trip. Motivation: external consumers (audit scripts, migration guards, downstream binary upgrade scripts) currently must read SQLite directly to learn the schema shape, which couples them to internals Vestige owns and breaks on every migration. This PR closes that gap with a first-class MCP surface. Implementation: - New `pub fn schema_introspection() -> Result` inherent method on `Storage` (sqlite.rs). Inherent, not on a trait — schema-walk is SQLite-specific by nature, so this stays out of any future MemoryStore trait extraction. - New typed structs `SchemaIntrospection` + `TableIntrospection` in memory/mod.rs (canonical home alongside MemoryStats), re-exported from the crate root. - MCP layer (maintenance.rs) parses `SystemStatusArgs`, conditionally extends the existing response object with a `schema` key — additive, default off, response shape unchanged when omitted. Coupling assessment vs PR #61 (storage-trait-phase1): This PR adds ONE new public inherent method on `Storage` plus uses three already-existing private helpers (`current_schema_version`, `table_exists`, `table_row_count`, `table_columns`). It does NOT touch the existing inherent method signatures, does NOT add anything to the prospective `MemoryStore` trait surface, and does NOT modify any of the ~25 methods #61 lifts into the trait. PR #61 is purely additive on the trait surface (per its description, `pub type Storage = SqliteMemoryStore;` preserves all existing call sites); this PR is additive on the inherent surface. Two purely-additive changes to disjoint surfaces should rebase cleanly. Tests: - system_status_schema_has_schema_introspection_flag (schema introspection: property present, type=boolean, default=false, not required) - system_status_without_schema_flag_omits_schema_block (backwards-compat: unset/false → no `schema` key) - system_status_with_schema_flag_emits_schema_block (positive case: schema block present, schemaVersion >= 13, tables non-empty, knowledge_nodes row count + columns sane, convenience fields present) - system_status_camelcase_alias (#[serde(rename_all="camelCase")] + alias works for both snake and camel input) - storage_schema_introspection_method (Storage-layer method tested directly, independent of MCP) Closes the second of two gaps surfaced in the knowledge-mgmt-sota-uplift initiative. Companion to PR #68 (search.tag_prefix). The two PRs are deliberately decoupled — this one carries the storage-layer surface extension; the other is MCP-layer-only. * fix(memory): derive Default on SchemaIntrospection to satisfy clippy The manual `impl Default for SchemaIntrospection` tripped `clippy::derivable_impls` under the workspace's `-D warnings` CI gate. All fields are types with `Default` impls (`u32`, `Option`, `Vec`, `i64`), so deriving is equivalent and clippy-clean. Matches the existing style of `ConsolidationResult` further down in the same file. --- crates/vestige-core/src/lib.rs | 2 + crates/vestige-core/src/memory/mod.rs | 53 +++++ crates/vestige-core/src/storage/sqlite.rs | 97 +++++++++ crates/vestige-mcp/src/tools/maintenance.rs | 219 +++++++++++++++++++- 4 files changed, 367 insertions(+), 4 deletions(-) diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index 640ba4a..08ce090 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -127,9 +127,11 @@ pub use memory::{ MemorySystem, NodeType, RecallInput, + SchemaIntrospection, SearchMode, SearchResult, SimilarityResult, + TableIntrospection, TemporalRange, }; diff --git a/crates/vestige-core/src/memory/mod.rs b/crates/vestige-core/src/memory/mod.rs index e8c3f32..8cd618e 100644 --- a/crates/vestige-core/src/memory/mod.rs +++ b/crates/vestige-core/src/memory/mod.rs @@ -276,6 +276,59 @@ impl Default for MemoryStats { } } +// ============================================================================ +// SCHEMA INTROSPECTION (v2.1.24+: surfaces DB shape to MCP consumers) +// ============================================================================ + +/// A single SQLite table's introspected shape: name, row count, column list. +/// +/// Returned as part of `SchemaIntrospection` from `Storage::schema_introspection()`. +/// Consumers needing more depth (e.g. per-column NULL counts) should request +/// targeted methods rather than expecting this struct to grow unboundedly — +/// the row + column shape covered here is the 80% case for audit / migration +/// guard scripts. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableIntrospection { + /// SQLite table name. + pub name: String, + /// Row count. + pub rows: i64, + /// Column names in declaration order. + pub columns: Vec, +} + +/// Result of `Storage::schema_introspection()`. Snapshots the schema version, +/// migration timestamp, and a row/column view of every user-data table. +/// +/// Motivation: external consumers (audit scripts, migration guards, downstream +/// upgrade scripts) currently must read SQLite directly to learn the schema +/// version and table shape, which couples them to internal layout. This struct +/// gives them a first-class MCP-callable surface. The list of tables walked is +/// intentionally the same canonical set used elsewhere in storage (the user- +/// data tables) so the surface stays stable across migrations. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct SchemaIntrospection { + /// Current schema version (highest applied migration; matches the + /// `schema_version` table's MAX(version)). + pub schema_version: u32, + /// When the current schema version was applied (RFC3339), if known. + pub schema_version_applied_at: Option>, + /// Per-table introspection rows. + pub tables: Vec, + /// Total number of nodes whose `embeddings.embedding` is NULL (i.e., have + /// no embedding row). Convenience field for embedding-coverage audits; + /// equivalent to (knowledge_nodes.rows − rows in `embeddings` joined to + /// knowledge_nodes), so consumers don't have to compute it themselves. + pub embedding_null_count: i64, + /// Active embedding model name (mirrors `MemoryStats.active_embedding_model`). + /// Useful when an audit script wants schema_version + active model in one call. + pub active_embedding_model: Option, + /// Embedding dimensions for the active model, if known. + pub active_embedding_dimensions: Option, +} + // ============================================================================ // CONSOLIDATION RESULT // ============================================================================ diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index e94a27d..c4e8f2b 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -1890,6 +1890,103 @@ impl Storage { }) } + /// Introspect the live SQLite schema: schema version + per-table row/column + /// shape + embedding-coverage convenience fields. + /// + /// This is the v2.1.24+ replacement for the direct-SQLite reads that + /// audit scripts and migration guards previously had to perform. The set + /// of tables walked matches `PORTABLE_USER_DATA_TABLES` — the same + /// canonical set used by portable export / import — so the surface stays + /// stable across migrations rather than chasing arbitrary + /// `sqlite_master` rows. + /// + /// Cost: O(N_tables) `COUNT(*)` queries + one PRAGMA per table. Negligible + /// at the table cardinalities Vestige carries (~15 tables, all indexed). + /// Safe to call on every MCP `system_status` invocation when the flag is + /// set; callers wanting to limit cost should leave the flag off (default). + pub fn schema_introspection(&self) -> Result { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + + let schema_version = Self::current_schema_version(&reader)?; + + // schema_version has the row (version PK + applied_at TEXT). Read the + // applied_at for the current version row; tolerate failure (legacy + // databases may have skipped the applied_at fill on early upgrades). + let applied_at_str: Option = reader + .query_row( + "SELECT applied_at FROM schema_version WHERE version = ?1", + params![schema_version as i64], + |row| row.get(0), + ) + .optional()?; + let schema_version_applied_at = applied_at_str.and_then(|s| { + // The migration scripts use `datetime('now')` which yields + // SQLite's "YYYY-MM-DD HH:MM:SS" UTC form (NOT RFC3339). + // Try the SQLite form first, fall back to RFC3339 for any + // future migrations that switch. + chrono::NaiveDateTime::parse_from_str(&s, "%Y-%m-%d %H:%M:%S") + .map(|naive| naive.and_utc()) + .or_else(|_| { + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + }) + .ok() + }); + + let mut tables = Vec::with_capacity(PORTABLE_USER_DATA_TABLES.len()); + for table_name in PORTABLE_USER_DATA_TABLES { + if Self::table_exists(&reader, table_name)? { + let rows = Self::table_row_count(&reader, table_name)?; + let columns = Self::table_columns(&reader, table_name)?; + tables.push(crate::TableIntrospection { + name: (*table_name).to_string(), + rows, + columns, + }); + } + } + + // Convenience: embedding-coverage NULL count. Defined as the number + // of knowledge_nodes with NO matching row in node_embeddings. This is + // distinct from `nodes_with_embeddings` in MemoryStats (which uses + // the `has_embedding` column flag); we compute the join-based truth + // here so audit scripts can detect drift between the flag and the + // actual embeddings table. + let embedding_null_count: i64 = reader + .query_row( + "SELECT COUNT(*) FROM knowledge_nodes kn + WHERE NOT EXISTS ( + SELECT 1 FROM node_embeddings ne WHERE ne.node_id = kn.id + )", + [], + |row| row.get(0), + ) + .unwrap_or(0); + + #[cfg(feature = "embeddings")] + let active_embedding_model = Some(self.embedding_service.model_name().to_string()); + #[cfg(not(feature = "embeddings"))] + let active_embedding_model: Option = None; + + #[cfg(feature = "embeddings")] + let active_embedding_dimensions: Option = + Some(self.embedding_service.dimensions() as u32); + #[cfg(not(feature = "embeddings"))] + let active_embedding_dimensions: Option = None; + + Ok(crate::SchemaIntrospection { + schema_version, + schema_version_applied_at, + tables, + embedding_null_count, + active_embedding_model, + active_embedding_dimensions, + }) + } + /// Delete a node pub fn delete_node(&self, id: &str) -> Result { let mut writer = self diff --git a/crates/vestige-mcp/src/tools/maintenance.rs b/crates/vestige-mcp/src/tools/maintenance.rs index 82f5374..9dfb50e 100644 --- a/crates/vestige-mcp/src/tools/maintenance.rs +++ b/crates/vestige-mcp/src/tools/maintenance.rs @@ -105,10 +105,24 @@ pub fn gc_schema() -> Value { pub fn system_status_schema() -> Value { serde_json::json!({ "type": "object", - "properties": {} + "properties": { + "schema_introspection": { + "type": "boolean", + "description": "When true, extends the response with a 'schema' block carrying the SQLite schema version, per-table row counts + column lists, and embedding-coverage convenience fields. Default: false (response shape unchanged). Use this for audit / migration-guard / downstream-upgrade scripts that otherwise have to read SQLite directly.", + "default": false + } + } }) } +/// Arguments for the system_status tool. All optional. +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SystemStatusArgs { + #[serde(alias = "schema_introspection")] + schema_introspection: Option, +} + // ============================================================================ // EXECUTE FUNCTIONS // ============================================================================ @@ -117,11 +131,24 @@ pub fn system_status_schema() -> Value { /// /// Returns system health status, full statistics, FSRS preview, /// cognitive module health, state distribution, and actionable recommendations. +/// +/// v2.1.24+: when `schema_introspection: true` is passed, the response +/// additionally carries a `schema` block with the live SQLite schema version, +/// per-table row counts + column lists, and embedding-coverage convenience +/// fields. Default off; response shape unchanged when omitted. pub async fn execute_system_status( storage: &Arc, cognitive: &Arc>, - _args: Option, + args: Option, ) -> Result { + // Parse arguments (all optional, including the args envelope itself). + let parsed: SystemStatusArgs = match args { + Some(v) => serde_json::from_value(v) + .map_err(|e| format!("Invalid arguments: {}", e))?, + None => SystemStatusArgs::default(), + }; + let include_schema = parsed.schema_introspection.unwrap_or(false); + let stats = storage.get_stats().map_err(|e| e.to_string())?; // === Health assessment === @@ -259,7 +286,7 @@ pub async fn execute_system_status( }; let last_backup = storage.last_backup_timestamp(); - Ok(serde_json::json!({ + let mut response = serde_json::json!({ "tool": "system_status", // Health "status": status, @@ -299,7 +326,34 @@ pub async fn execute_system_status( "lastBackupTimestamp": last_backup.map(|dt| dt.to_rfc3339()), "lastConsolidationTimestamp": last_consolidation.map(|dt| dt.to_rfc3339()), }, - })) + }); + + // v2.1.24+: optional schema introspection block. Default off; response + // shape unchanged when omitted. + if include_schema { + let intro = storage.schema_introspection().map_err(|e| e.to_string())?; + let tables_json: Vec = intro + .tables + .iter() + .map(|t| { + serde_json::json!({ + "name": t.name, + "rows": t.rows, + "columns": t.columns, + }) + }) + .collect(); + response["schema"] = serde_json::json!({ + "schemaVersion": intro.schema_version, + "schemaVersionAppliedAt": intro.schema_version_applied_at.map(|dt| dt.to_rfc3339()), + "tables": tables_json, + "embeddingNullCount": intro.embedding_null_count, + "activeEmbeddingModel": intro.active_embedding_model, + "activeEmbeddingDimensions": intro.active_embedding_dimensions, + }); + } + + Ok(response) } /// Consolidate tool @@ -792,6 +846,163 @@ mod tests { assert!(triggers["lastDreamTimestamp"].is_null()); } + // ======================================================================== + // SCHEMA INTROSPECTION TESTS (PR2) + // ======================================================================== + + #[test] + fn test_system_status_schema_has_schema_introspection_flag() { + let schema = system_status_schema(); + let props = &schema["properties"]; + let flag = &props["schema_introspection"]; + assert!(flag.is_object(), "schema_introspection property must exist"); + assert_eq!(flag["type"], "boolean"); + assert_eq!(flag["default"], false); + // Top-level required must NOT include this — flag is opt-in. + let required = schema.get("required"); + if let Some(req) = required { + let req_arr = req.as_array().unwrap(); + assert!(!req_arr.contains(&serde_json::json!("schema_introspection"))); + } + } + + #[tokio::test] + async fn test_system_status_without_schema_flag_omits_schema_block() { + // Backwards-compat: when the flag is not set (or false), the response + // shape is unchanged — no `schema` key. + let (storage, _dir) = test_storage().await; + let result = execute_system_status(&storage, &test_cognitive(), None).await; + assert!(result.is_ok()); + let value = result.unwrap(); + assert!( + value.get("schema").is_none(), + "schema block must NOT be present when flag is unset, got {:?}", + value.get("schema") + ); + + // Explicit false → still no schema block. + let result = execute_system_status( + &storage, + &test_cognitive(), + Some(serde_json::json!({ "schema_introspection": false })), + ) + .await; + assert!(result.is_ok()); + let value = result.unwrap(); + assert!(value.get("schema").is_none()); + } + + #[tokio::test] + async fn test_system_status_with_schema_flag_emits_schema_block() { + let (storage, _dir) = test_storage().await; + storage + .ingest(vestige_core::IngestInput { + content: "Schema introspection seed memory".to_string(), + node_type: "fact".to_string(), + source: None, + sentiment_score: 0.0, + sentiment_magnitude: 0.0, + tags: vec!["schema-test".to_string()], + valid_from: None, + valid_until: None, + }) + .unwrap(); + + let result = execute_system_status( + &storage, + &test_cognitive(), + Some(serde_json::json!({ "schema_introspection": true })), + ) + .await; + assert!(result.is_ok(), "{:?}", result); + let value = result.unwrap(); + + // Shape assertions. + let schema_block = value + .get("schema") + .expect("schema block must be present when flag is true"); + assert!(schema_block.is_object()); + assert!( + schema_block["schemaVersion"].is_number(), + "schemaVersion must be a number, got {:?}", + schema_block["schemaVersion"] + ); + // Schema version should be >= 13 (V13 is the highest landed migration + // at the time this PR was authored). + let v = schema_block["schemaVersion"].as_u64().unwrap(); + assert!(v >= 13, "expected schema_version >= 13, got {}", v); + + // tables should be a non-empty array of {name, rows, columns}. + let tables = schema_block["tables"].as_array().unwrap(); + assert!(!tables.is_empty(), "expected at least one table"); + let kn = tables + .iter() + .find(|t| t["name"] == "knowledge_nodes") + .expect("knowledge_nodes table must be present"); + assert_eq!(kn["rows"], 1, "ingested exactly one memory"); + let cols = kn["columns"].as_array().unwrap(); + assert!(!cols.is_empty(), "knowledge_nodes must have columns"); + // The id column is universally present. + let col_names: Vec<&str> = cols.iter().filter_map(|c| c.as_str()).collect(); + assert!( + col_names.contains(&"id"), + "knowledge_nodes.id must be in columns list: {:?}", + col_names + ); + + // Convenience fields. + assert!(schema_block["embeddingNullCount"].is_number()); + // activeEmbeddingModel may be null if the `embeddings` feature is + // not enabled in the test build; just check the key exists. + assert!(schema_block.get("activeEmbeddingModel").is_some()); + assert!(schema_block.get("activeEmbeddingDimensions").is_some()); + } + + #[tokio::test] + async fn test_system_status_camelcase_alias() { + // Accept both `schema_introspection` (snake) and `schemaIntrospection` + // (camel) per the #[serde(rename_all = "camelCase")] + alias attr. + let (storage, _dir) = test_storage().await; + let result = execute_system_status( + &storage, + &test_cognitive(), + Some(serde_json::json!({ "schemaIntrospection": true })), + ) + .await; + assert!(result.is_ok(), "{:?}", result); + let value = result.unwrap(); + assert!( + value.get("schema").is_some(), + "camelCase form must also trigger schema block" + ); + } + + #[test] + fn test_storage_schema_introspection_method() { + // Direct test on the Storage method, independent of the MCP layer. + let dir = TempDir::new().unwrap(); + let storage = Storage::new(Some(dir.path().join("test.db"))).unwrap(); + let intro = storage + .schema_introspection() + .expect("schema_introspection must succeed on a fresh DB"); + + // Schema version pulled from the schema_version table. + assert!( + intro.schema_version >= 13, + "fresh DB should be at schema_version >= 13, got {}", + intro.schema_version + ); + // At least one walked table should exist. + assert!( + !intro.tables.is_empty(), + "expected at least one user-data table" + ); + // Empty DB → no embeddings → embedding_null_count == 0 (no rows to + // count). Once we ingest, it should be > 0 (no embeddings generated + // in tests by default). + assert_eq!(intro.embedding_null_count, 0); + } + #[tokio::test] async fn test_portable_export_writes_archive_to_storage_exports_dir() { let (storage, _dir) = test_storage().await; From c23d7a309ced8f9232de9581b2c108c45278d7eb Mon Sep 17 00:00:00 2001 From: Sam Valladares <143034159+samvallad33@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:55:31 -0500 Subject: [PATCH 18/38] =?UTF-8?q?feat(merge-supersede):=20Phase=203=20?= =?UTF-8?q?=E2=80=94=20diff-previewed,=20reversible=20merge/supersede=20co?= =?UTF-8?q?ntrols=20(v2.1.25)=20(#75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds opt-in, preview-first combine/dedupe/supersede on a never-delete (bitemporal) store. The default is review, never silent mutation. Every applied operation is recorded as a reversible, auditable event with provenance — a git reflog for your agent's memory. Core (vestige-core): - advanced::merge_supersede — pure Fellegi-Sunter two-threshold scoring (embedding + tag + token Jaccard), match/possible/non_match classification, plan/diff and operation-log types, merge-composition helpers. Unit-tested. - storage: merge_candidates, plan_merge, plan_supersede, apply_plan, merge_undo, protect/pin, and per-project merge_policy (persisted in fsrs_config, env overridable). Supersede invalidates bitemporally (valid_until + superseded_by, Graphiti-style "invalidate, don't delete") and keeps the old node queryable. - Migration V14: merge_plans + merge_operations tables, knowledge_nodes.protected and .superseded_by columns + indexes. Idempotent on replay (duplicate-column guarded ADD COLUMNs). MCP (vestige-mcp): - Seven new tools registered + dispatched: merge_candidates, plan_merge, plan_supersede, apply_plan, merge_undo, protect, merge_policy. - apply_plan requires confirm=true for possible/non_match plans; match plans auto-apply only when policy.auto_apply is set (default off). Tests: candidate-threshold classification, plan-preview makes no mutation, apply+undo reversibility, supersede bitemporal invalidation preserves old-node queryability, protect blocks merge-away, low-confidence requires confirm, policy roundtrip, migration V14 + idempotent replay. All 796 scoped tests pass; clippy -D warnings clean on touched crates. Docs: docs/MERGE_SUPERSEDE.md + CHANGELOG entry. Version bump 2.1.23 -> 2.1.25. Co-authored-by: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 58 + Cargo.lock | 4 +- Cargo.toml | 2 +- apps/dashboard/package.json | 2 +- crates/vestige-core/Cargo.toml | 2 +- .../src/advanced/merge_supersede.rs | 447 ++++++ crates/vestige-core/src/advanced/mod.rs | 6 + crates/vestige-core/src/lib.rs | 8 + crates/vestige-core/src/storage/migrations.rs | 147 +- crates/vestige-core/src/storage/sqlite.rs | 1279 +++++++++++++++++ crates/vestige-mcp/Cargo.toml | 4 +- crates/vestige-mcp/src/server.rs | 69 +- crates/vestige-mcp/src/tools/merge.rs | 530 +++++++ crates/vestige-mcp/src/tools/mod.rs | 3 + docs/MERGE_SUPERSEDE.md | 152 ++ package.json | 2 +- packages/vestige-init/package.json | 2 +- packages/vestige-mcp-npm/package.json | 2 +- server.json | 4 +- 19 files changed, 2704 insertions(+), 19 deletions(-) create mode 100644 crates/vestige-core/src/advanced/merge_supersede.rs create mode 100644 crates/vestige-mcp/src/tools/merge.rs create mode 100644 docs/MERGE_SUPERSEDE.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 490c01e..dec084c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,64 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.1.25] - 2026-06-12 — "Merge / Supersede Controls" + +v2.1.25 ships Phase 3: diff-previewed, confidence-gated, reversible, +self-explaining combine/dedupe/supersede on a never-delete (bitemporal) store. +The default is always preview/review — these tools never silently mutate memory. +The differentiator is the reversible operation log: every merge/supersede/undo is +an auditable, reversible event with provenance ("why did these combine?") — a git +reflog for your agent's memory. + +### Added + +- **Seven new MCP tools** for merge/supersede control: + - `merge_candidates` — surface likely duplicate/overlapping clusters with + confidence scores and the signals behind each (Fellegi-Sunter + match/possible/non-match). Read-only. + - `plan_merge` — produce a previewable merge PLAN (a diff of combined + content/tags/provenance) without applying it. + - `plan_supersede` — preview superseding A with B (bitemporal invalidation, + audit-preserving) without applying. + - `apply_plan` — execute a previously-generated plan id; recorded as a + reversible operation. + - `merge_undo` — reverse a prior merge/supersede operation, or list the + reversible operation log (the "memory reflog"). + - `protect` — pin a memory so it can never be auto-merged, superseded, or + garbage-collected. + - `merge_policy` — get/set the per-project Fellegi-Sunter two thresholds + (`match_threshold`, `possible_threshold`) and `auto_apply`. +- **Bitemporal "invalidate, don't delete" supersede** (Graphiti-style): a + superseded memory is kept and stays queryable for audit. It is stamped with + `valid_until = now` and a new `superseded_by` lineage pointer, instead of being + deleted or merely demoted. +- **Reversible operation log** (`merge_operations` table) — every applied + merge/supersede records an undo payload and provenance signals so any operation + can be reversed, including restoring survivor content/tags and clearing the + bitemporal invalidation. +- **Fellegi-Sunter two-threshold scoring** for dedup/merge candidates, combining + embedding cosine similarity with tag and content-token overlap. Borderline + "possible" matches are surfaced for review instead of force-merged. +- **Memory protection / pinning** — `protected` column on `knowledge_nodes`; + protected memories are excluded from auto-merge/supersede/GC paths. +- **Migration V14** adding the `merge_plans` and `merge_operations` tables, the + `protected` and `superseded_by` columns on `knowledge_nodes`, and their + indexes. Idempotent on replay. +- **Docs**: `docs/MERGE_SUPERSEDE.md` describing the design, the bitemporal + model, the two-threshold policy, the reversible operation log, and the tool + surface. + +### Notes + +- All merge/supersede operations are **opt-in and preview-first**. `apply_plan` + requires `confirm=true` for `possible`/`non_match` plans, and only applies + `match` plans without confirmation when `merge_policy.auto_apply` is enabled + (default off). This deliberately avoids the silent-merge / auto-delete / + audit-trail-loss anti-patterns reported against other memory systems. +- The merge policy persists per-project and is also overridable via + `VESTIGE_MERGE_MATCH_THRESHOLD`, `VESTIGE_MERGE_POSSIBLE_THRESHOLD`, and + `VESTIGE_MERGE_AUTO_APPLY` environment variables. + ## [2.1.23] - 2026-05-27 — "Receipt Lock Hardening" v2.1.23 hardens the Sanhedrin launch path so Receipt Lock is portable, diff --git a/Cargo.lock b/Cargo.lock index b9612d3..33fe576 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4629,7 +4629,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vestige-core" -version = "2.1.23" +version = "2.1.25" dependencies = [ "candle-core", "chrono", @@ -4665,7 +4665,7 @@ dependencies = [ [[package]] name = "vestige-mcp" -version = "2.1.23" +version = "2.1.25" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index f120928..1c89455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ exclude = [ ] [workspace.package] -version = "2.1.23" +version = "2.1.25" edition = "2024" license = "AGPL-3.0-only" repository = "https://github.com/samvallad33/vestige" diff --git a/apps/dashboard/package.json b/apps/dashboard/package.json index 4023f8d..b35ef9f 100644 --- a/apps/dashboard/package.json +++ b/apps/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "@vestige/dashboard", - "version": "2.1.23", + "version": "2.1.25", "private": true, "type": "module", "scripts": { diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index fe89443..c66c369 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vestige-core" -version = "2.1.23" +version = "2.1.25" edition = "2024" rust-version = "1.91" authors = ["Vestige Team"] diff --git a/crates/vestige-core/src/advanced/merge_supersede.rs b/crates/vestige-core/src/advanced/merge_supersede.rs new file mode 100644 index 0000000..c10485a --- /dev/null +++ b/crates/vestige-core/src/advanced/merge_supersede.rs @@ -0,0 +1,447 @@ +//! # Merge / Supersede Controls (Phase 3) +//! +//! Diff-previewed, confidence-gated, reversible, self-explaining combine / +//! dedupe / supersede operations on a never-delete (bitemporal) store. +//! +//! This module holds the **pure** logic: candidate scoring, two-threshold +//! classification, and the plan / operation data model. The actual persistence +//! (writing plans, applying them, recording the reversible operation log, and +//! bitemporally invalidating superseded nodes) lives in +//! [`crate::storage`]. Keeping the math here makes it unit-testable without a +//! database. +//! +//! ## Design north star +//! +//! Every combine/dedupe/supersede operation is: +//! +//! - **diff-previewed** — `plan_merge` / `plan_supersede` produce a [`MergePlan`] +//! you can inspect before anything mutates, +//! - **confidence-gated** — a Fellegi-Sunter two-threshold score classifies each +//! candidate as match / possible-match / non-match, +//! - **reversible** — every applied plan records a [`MergeOperation`] with an +//! undo payload (the "git reflog for your agent's memory"), +//! - **self-explaining** — each candidate carries the [`MatchSignals`] that +//! explain *why* the memories combined, +//! - **opt-in, never silent** — the default is preview/review, never auto-mutate, +//! - **audit-preserving** — superseding stamps `valid_until` and keeps the old +//! node queryable (Graphiti-style "invalidate, don't delete"). +//! +//! ## Why Fellegi-Sunter +//! +//! Pure hashing under-merges (misses paraphrases); aggressive LLM merging +//! over-merges and destroys the audit trail. Fellegi-Sunter record linkage uses +//! **two** thresholds to carve the score space into three zones, so the +//! borderline "possible match" cases are surfaced for review instead of being +//! force-decided. We reuse the embedding cosine similarity already in the store +//! plus cheap lexical signals (tag overlap, token Jaccard) as the match weight. + +use serde::{Deserialize, Serialize}; + +// ============================================================================ +// CONSTANTS — the two Fellegi-Sunter thresholds +// ============================================================================ + +/// Above this combined score → automatic-eligible "match". +pub const DEFAULT_MATCH_THRESHOLD: f32 = 0.86; + +/// Between the two thresholds → "possible match", surfaced for review. +/// Below this → "non-match" (never offered). +pub const DEFAULT_POSSIBLE_THRESHOLD: f32 = 0.72; + +/// Weight of embedding cosine similarity in the combined score. +const W_EMBEDDING: f32 = 0.70; +/// Weight of tag overlap (Jaccard) in the combined score. +const W_TAGS: f32 = 0.15; +/// Weight of content token overlap (Jaccard) in the combined score. +const W_TOKENS: f32 = 0.15; + +// ============================================================================ +// CLASSIFICATION +// ============================================================================ + +/// Fellegi-Sunter three-way classification of a candidate pair/cluster. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MatchClass { + /// Score ≥ match threshold — strong duplicate, auto-merge eligible. + Match, + /// Between thresholds — surfaced for human/agent review, never auto-applied. + Possible, + /// Below the possible threshold — not offered as a candidate. + NonMatch, +} + +impl MatchClass { + /// String label used in tool output and the `classification` column. + pub fn as_str(&self) -> &'static str { + match self { + MatchClass::Match => "match", + MatchClass::Possible => "possible", + MatchClass::NonMatch => "non_match", + } + } +} + +/// Per-merge-policy thresholds. Wired to `vestige.toml` when present, else the +/// defaults above. `auto_apply` gates whether `Match`-class candidates may be +/// applied without an explicit preview step (default: false — never silent). +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct MergePolicy { + /// Score ≥ this → `Match`. + pub match_threshold: f32, + /// Score in `[possible_threshold, match_threshold)` → `Possible`. + pub possible_threshold: f32, + /// If true, `Match`-class candidates may be auto-applied. Default false: + /// the product promise is review/preview, not silent mutation. + pub auto_apply: bool, +} + +impl Default for MergePolicy { + fn default() -> Self { + Self { + match_threshold: DEFAULT_MATCH_THRESHOLD, + possible_threshold: DEFAULT_POSSIBLE_THRESHOLD, + auto_apply: false, + } + } +} + +impl MergePolicy { + /// Build a policy, clamping thresholds into `[0,1]` and ensuring + /// `possible_threshold <= match_threshold`. + pub fn new(match_threshold: f32, possible_threshold: f32, auto_apply: bool) -> Self { + let match_threshold = match_threshold.clamp(0.0, 1.0); + let possible_threshold = possible_threshold.clamp(0.0, match_threshold); + Self { + match_threshold, + possible_threshold, + auto_apply, + } + } + + /// Classify a combined match score. + pub fn classify(&self, score: f32) -> MatchClass { + if score >= self.match_threshold { + MatchClass::Match + } else if score >= self.possible_threshold { + MatchClass::Possible + } else { + MatchClass::NonMatch + } + } +} + +// ============================================================================ +// SIGNALS — the self-explaining "why did these combine?" +// ============================================================================ + +/// The individual signals behind a candidate's score. Surfaced verbatim so a +/// user can see *why* two memories were judged duplicates. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MatchSignals { + /// Cosine similarity of the two embeddings (0–1). + pub embedding_similarity: f32, + /// Jaccard overlap of the two tag sets (0–1). + pub tag_overlap: f32, + /// Jaccard overlap of content tokens (0–1). + pub token_overlap: f32, + /// Combined weighted score that was classified. + pub combined_score: f32, +} + +/// Compute the combined match score and its signal breakdown for a pair. +pub fn score_pair( + embedding_similarity: f32, + a_tags: &[String], + b_tags: &[String], + a_content: &str, + b_content: &str, +) -> MatchSignals { + let tag_overlap = jaccard(&tag_set(a_tags), &tag_set(b_tags)); + let token_overlap = jaccard(&token_set(a_content), &token_set(b_content)); + let combined_score = (W_EMBEDDING * embedding_similarity.clamp(0.0, 1.0) + + W_TAGS * tag_overlap + + W_TOKENS * token_overlap) + .clamp(0.0, 1.0); + MatchSignals { + embedding_similarity: embedding_similarity.clamp(0.0, 1.0), + tag_overlap, + token_overlap, + combined_score, + } +} + +fn tag_set(tags: &[String]) -> std::collections::HashSet { + tags.iter().map(|t| t.to_lowercase()).collect() +} + +fn token_set(content: &str) -> std::collections::HashSet { + content + .split(|c: char| !c.is_alphanumeric()) + .filter(|t| t.len() > 2) + .map(|t| t.to_lowercase()) + .collect() +} + +fn jaccard(a: &std::collections::HashSet, b: &std::collections::HashSet) -> f32 { + if a.is_empty() && b.is_empty() { + return 0.0; + } + let inter = a.intersection(b).count() as f32; + let union = a.union(b).count() as f32; + if union == 0.0 { 0.0 } else { inter / union } +} + +// ============================================================================ +// CANDIDATE +// ============================================================================ + +/// A surfaced merge candidate: a cluster of likely-duplicate memories with the +/// signals and classification that justify offering it. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MergeCandidate { + /// Node ids in the cluster. The first is the suggested survivor (highest + /// retention). + pub member_ids: Vec, + /// Short content previews, parallel to `member_ids`. + pub previews: Vec, + /// Suggested survivor id (kept after a merge). + pub survivor_id: String, + /// Combined match score for the cluster (min pairwise within the cluster — + /// the weakest link, so a cluster is only as confident as its loosest pair). + pub confidence: f32, + /// Three-way classification under the active policy. + pub classification: MatchClass, + /// Signals for the survivor↔closest-member pair (the explanation). + pub signals: MatchSignals, + /// True if any member is protected (pinned) — blocks auto-merge. + pub has_protected_member: bool, +} + +// ============================================================================ +// PLAN — the previewable diff +// ============================================================================ + +/// What kind of plan this is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlanKind { + /// Combine N memories into one survivor. + Merge, + /// Invalidate A in favour of B (bitemporal, audit-preserving). + Supersede, +} + +impl PlanKind { + pub fn as_str(&self) -> &'static str { + match self { + PlanKind::Merge => "merge", + PlanKind::Supersede => "supersede", + } + } +} + +/// A previewable plan: exactly what *would* change, without changing anything. +/// Persisted to `merge_plans`; consumed by `apply_plan` via its `id`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MergePlan { + /// Plan id (UUID). + pub id: String, + /// merge | supersede. + pub kind: PlanKind, + /// Node kept after the operation. + pub survivor_id: String, + /// All node ids involved. + pub member_ids: Vec, + /// Resulting content of the survivor after applying. + pub result_content: String, + /// Resulting tag set of the survivor after applying. + pub result_tags: Vec, + /// Resulting provenance / source string after applying. + pub result_source: Option, + /// For supersede: ids that get bitemporally invalidated (their + /// `valid_until` stamped, kept queryable). For merge: the absorbed ids. + pub invalidated_ids: Vec, + /// Match confidence (0–1) for the plan. + pub confidence: f32, + /// Three-way classification. + pub classification: MatchClass, + /// Signals explaining the plan. + pub signals: MatchSignals, + /// Human-readable explanation of what this plan does. + pub explanation: String, +} + +// ============================================================================ +// OPERATION LOG — the reversible "memory reflog" +// ============================================================================ + +/// A recorded, reversible operation. One row in `merge_operations`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MergeOperation { + /// Operation id (UUID). + pub id: String, + /// Plan id this came from (if any). + pub plan_id: Option, + /// merge | supersede | undo. + pub op_type: String, + /// applied | reverted. + pub status: String, + /// When recorded (RFC3339). + pub created_at: String, + /// When reverted (RFC3339), if reverted. + pub reverted_at: Option, + /// For undo ops: the op id being reversed. + pub reverts_op_id: Option, + /// Survivor node id. + pub survivor_id: Option, + /// Node ids touched by the op. + pub affected_ids: Vec, + /// Match confidence. + pub confidence: Option, + /// Human-readable reason. + pub reason: Option, +} + +// ============================================================================ +// MERGE COMPOSITION — pure helpers used by the storage apply path +// ============================================================================ + +/// Compose merged content from an ordered list of (id, content) members. +/// Survivor content leads; each absorbed member is appended with provenance so +/// nothing is silently dropped (anti-pattern: Mem0 #4896 double-store / +/// contradiction loss). +pub fn compose_merged_content(members: &[(String, String)]) -> String { + if members.is_empty() { + return String::new(); + } + let mut out = members[0].1.trim().to_string(); + for (id, content) in &members[1..] { + let c = content.trim(); + if c.is_empty() || out.contains(c) { + continue; + } + out.push_str("\n\n[merged from "); + out.push_str(id); + out.push_str("]\n"); + out.push_str(c); + } + out +} + +/// Union the tag sets of all members, preserving first-seen order. +pub fn compose_merged_tags(member_tags: &[Vec]) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut out = Vec::new(); + for tags in member_tags { + for t in tags { + if seen.insert(t.to_lowercase()) { + out.push(t.clone()); + } + } + } + out +} + +// ============================================================================ +// TESTS +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_three_zones() { + let policy = MergePolicy::default(); + assert_eq!(policy.classify(0.95), MatchClass::Match); + assert_eq!(policy.classify(0.80), MatchClass::Possible); + assert_eq!(policy.classify(0.50), MatchClass::NonMatch); + // boundaries are inclusive at the lower edge of each higher zone + assert_eq!(policy.classify(DEFAULT_MATCH_THRESHOLD), MatchClass::Match); + assert_eq!( + policy.classify(DEFAULT_POSSIBLE_THRESHOLD), + MatchClass::Possible + ); + } + + #[test] + fn policy_clamps_and_orders() { + // possible above match gets clamped down to match + let p = MergePolicy::new(0.8, 0.95, true); + assert!(p.possible_threshold <= p.match_threshold); + // out-of-range clamps to [0,1] + let p2 = MergePolicy::new(2.0, -1.0, false); + assert_eq!(p2.match_threshold, 1.0); + assert_eq!(p2.possible_threshold, 0.0); + } + + #[test] + fn score_pair_combines_signals() { + let s = score_pair( + 1.0, + &["rust".into(), "async".into()], + &["rust".into(), "async".into()], + "use tokio for async rust", + "use tokio for async rust", + ); + assert!((s.embedding_similarity - 1.0).abs() < 1e-6); + assert!((s.tag_overlap - 1.0).abs() < 1e-6); + assert!(s.token_overlap > 0.9); + assert!(s.combined_score > 0.95); + } + + #[test] + fn score_pair_disjoint_is_low() { + let s = score_pair( + 0.1, + &["a".into()], + &["b".into()], + "completely different topic alpha", + "totally unrelated subject beta", + ); + assert!(s.combined_score < 0.3); + assert_eq!(MergePolicy::default().classify(s.combined_score), MatchClass::NonMatch); + } + + #[test] + fn jaccard_basics() { + let a: std::collections::HashSet = ["x".into(), "y".into()].into_iter().collect(); + let b: std::collections::HashSet = ["y".into(), "z".into()].into_iter().collect(); + assert!((jaccard(&a, &b) - (1.0 / 3.0)).abs() < 1e-6); + let empty: std::collections::HashSet = Default::default(); + assert_eq!(jaccard(&empty, &empty), 0.0); + } + + #[test] + fn compose_merged_content_dedups_and_attributes() { + let members = vec![ + ("a".into(), "Keep this.".into()), + ("b".into(), "Extra detail.".into()), + ("c".into(), "Keep this.".into()), // duplicate of survivor → skipped + ]; + let merged = compose_merged_content(&members); + assert!(merged.starts_with("Keep this.")); + assert!(merged.contains("[merged from b]")); + assert!(merged.contains("Extra detail.")); + // duplicate content not appended twice + assert_eq!(merged.matches("Keep this.").count(), 1); + } + + #[test] + fn compose_merged_tags_unions_in_order() { + let tags = vec![ + vec!["rust".into(), "async".into()], + vec!["async".into(), "tokio".into()], + ]; + let merged = compose_merged_tags(&tags); + assert_eq!(merged, vec!["rust", "async", "tokio"]); + } + + #[test] + fn match_class_labels() { + assert_eq!(MatchClass::Match.as_str(), "match"); + assert_eq!(MatchClass::Possible.as_str(), "possible"); + assert_eq!(MatchClass::NonMatch.as_str(), "non_match"); + } +} diff --git a/crates/vestige-core/src/advanced/mod.rs b/crates/vestige-core/src/advanced/mod.rs index fdbdfe4..0ed9280 100644 --- a/crates/vestige-core/src/advanced/mod.rs +++ b/crates/vestige-core/src/advanced/mod.rs @@ -23,6 +23,7 @@ pub mod cross_project; pub mod dreams; pub mod importance; pub mod intent; +pub mod merge_supersede; pub mod prediction_error; pub mod reconsolidation; pub mod speculative; @@ -61,6 +62,11 @@ pub use dreams::{ }; pub use importance::{ImportanceDecayConfig, ImportanceScore, ImportanceTracker, UsageEvent}; pub use intent::{ActionType, DetectedIntent, IntentDetector, MaintenanceType, UserAction}; +pub use merge_supersede::{ + DEFAULT_MATCH_THRESHOLD, DEFAULT_POSSIBLE_THRESHOLD, MatchClass, MatchSignals, MergeCandidate, + MergeOperation, MergePlan, MergePolicy, PlanKind, compose_merged_content, compose_merged_tags, + score_pair, +}; pub use prediction_error::{ CandidateMemory, CreateReason, EvaluationIntent, GateDecision, GateStats, MergeStrategy, PredictionErrorConfig, PredictionErrorGate, SimilarityResult, SupersedeReason, UpdateType, diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index 08ce090..4c50413 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -225,8 +225,16 @@ pub use advanced::{ MemoryPath, MemoryReplay, MemorySnapshot, + // Merge / Supersede controls (Phase 3) + MatchClass, + MatchSignals, + MergeCandidate, + MergeOperation, + MergePlan, + MergePolicy, MergeStrategy, Modification, + PlanKind, Pattern, PatternType, PredictedMemory, diff --git a/crates/vestige-core/src/storage/migrations.rs b/crates/vestige-core/src/storage/migrations.rs index 2c66a2d..3be941c 100644 --- a/crates/vestige-core/src/storage/migrations.rs +++ b/crates/vestige-core/src/storage/migrations.rs @@ -69,6 +69,11 @@ pub const MIGRATIONS: &[Migration] = &[ description: "v2.1.2 Honest Memory: non-content purge tombstones", up: MIGRATION_V13_UP, }, + Migration { + version: 14, + description: "v2.1.25 Merge/Supersede: reversible operation log, merge plans, bitemporal lineage, protected pins", + up: MIGRATION_V14_UP, + }, ]; /// A database migration @@ -735,6 +740,79 @@ ON deletion_tombstones(deleted_at); UPDATE schema_version SET version = 13, applied_at = datetime('now'); "#; +/// V14: Merge / Supersede controls (Phase 3). +/// +/// Adds the four pieces the merge/supersede feature needs on a never-delete +/// (bitemporal) store: +/// +/// 1. `merge_plans` — previewable, not-yet-applied plans. `plan_merge` and +/// `plan_supersede` write a plan row containing a JSON diff; `apply_plan` +/// consumes it by id. Plans are append-only; status moves +/// pending -> applied / cancelled. +/// 2. `merge_operations` — the reversible operation log (the "memory reflog"). +/// Every applied merge/supersede records one row with a JSON `undo_payload` +/// capturing exactly what changed, so `merge_undo` can reverse it. The +/// `signals` column records WHY the memories combined (provenance), which is +/// the self-explaining differentiator. +/// 3. `knowledge_nodes.protected` — pin flag. A protected memory can never be +/// auto-merged, superseded, or forgotten. +/// 4. `knowledge_nodes.superseded_by` — bitemporal lineage pointer. Superseding +/// A with B does NOT delete A: it stamps A.valid_until = B.valid_from and +/// sets A.superseded_by = B.id, leaving A fully queryable for audit +/// (Graphiti-style invalidate-don't-delete). +// The two `protected` / `superseded_by` ADD COLUMNs (and their indexes) are +// applied separately in `apply_migrations` BEFORE this batch runs, guarded +// against "duplicate column" on replay, since SQLite has no +// `ADD COLUMN IF NOT EXISTS`. The rest of V14 is idempotent (CREATE ... IF NOT +// EXISTS). +const MIGRATION_V14_UP: &str = r#" +CREATE INDEX IF NOT EXISTS idx_nodes_protected ON knowledge_nodes(protected); +CREATE INDEX IF NOT EXISTS idx_nodes_superseded_by ON knowledge_nodes(superseded_by); + +-- Previewable plans (a diff) produced by plan_merge / plan_supersede. +-- `kind` is 'merge' | 'supersede'. `payload` is the full JSON plan/diff. +CREATE TABLE IF NOT EXISTS merge_plans ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', -- pending | applied | cancelled + created_at TEXT NOT NULL, + applied_at TEXT, + survivor_id TEXT, -- node kept after the op + member_ids TEXT NOT NULL DEFAULT '[]', -- JSON array of all involved node ids + confidence REAL, -- Fellegi-Sunter match score (0-1) + classification TEXT, -- match | possible | non_match + payload TEXT NOT NULL -- full JSON plan/diff +); + +CREATE INDEX IF NOT EXISTS idx_merge_plans_status ON merge_plans(status); +CREATE INDEX IF NOT EXISTS idx_merge_plans_created_at ON merge_plans(created_at); + +-- Reversible operation log — the "git reflog for your agent's memory". +-- One row per applied merge/supersede; `undo_payload` carries everything +-- needed to reverse it, `signals` records why the memories combined. +CREATE TABLE IF NOT EXISTS merge_operations ( + id TEXT PRIMARY KEY, + plan_id TEXT, -- merge_plans.id this came from + op_type TEXT NOT NULL, -- merge | supersede | undo + status TEXT NOT NULL DEFAULT 'applied', -- applied | reverted + created_at TEXT NOT NULL, + reverted_at TEXT, + reverts_op_id TEXT, -- set when op_type = 'undo' + survivor_id TEXT, -- node kept + affected_ids TEXT NOT NULL DEFAULT '[]', -- JSON array of node ids touched + confidence REAL, + signals TEXT, -- JSON: why they combined (provenance) + reason TEXT, -- human-readable explanation + undo_payload TEXT NOT NULL -- JSON snapshot to reverse the op +); + +CREATE INDEX IF NOT EXISTS idx_merge_operations_status ON merge_operations(status); +CREATE INDEX IF NOT EXISTS idx_merge_operations_created_at ON merge_operations(created_at); +CREATE INDEX IF NOT EXISTS idx_merge_operations_survivor ON merge_operations(survivor_id); + +UPDATE schema_version SET version = 14, applied_at = datetime('now'); +"#; + /// Get current schema version from database pub fn get_current_version(conn: &rusqlite::Connection) -> rusqlite::Result { conn.query_row( @@ -745,6 +823,19 @@ pub fn get_current_version(conn: &rusqlite::Connection) -> rusqlite::Result .or(Ok(0)) } +/// Run an `ALTER TABLE ... ADD COLUMN` statement, treating a "duplicate column +/// name" failure as success so migration replay stays idempotent (SQLite has no +/// `ADD COLUMN IF NOT EXISTS`). +fn add_column_if_missing(conn: &rusqlite::Connection, sql: &str) -> rusqlite::Result<()> { + match conn.execute(sql, []) { + Ok(_) => Ok(()), + Err(rusqlite::Error::SqliteFailure(_, Some(msg))) if msg.contains("duplicate column name") => { + Ok(()) + } + Err(e) => Err(e), + } +} + /// Apply pending migrations pub fn apply_migrations(conn: &rusqlite::Connection) -> rusqlite::Result { let current_version = get_current_version(conn)?; @@ -758,6 +849,21 @@ pub fn apply_migrations(conn: &rusqlite::Connection) -> rusqlite::Result { migration.description ); + // V14: add the two bitemporal/protect columns BEFORE the batch (the + // batch's indexes reference them). SQLite lacks + // `ADD COLUMN IF NOT EXISTS`, so swallow the "duplicate column" + // error to stay idempotent on replay. + if migration.version == 14 { + add_column_if_missing( + conn, + "ALTER TABLE knowledge_nodes ADD COLUMN protected INTEGER NOT NULL DEFAULT 0", + )?; + add_column_if_missing( + conn, + "ALTER TABLE knowledge_nodes ADD COLUMN superseded_by TEXT", + )?; + } + // Use execute_batch to handle multi-statement SQL including triggers conn.execute_batch(migration.up)?; @@ -784,17 +890,17 @@ mod tests { /// version after `apply_migrations` runs all migrations end-to-end, and /// neither of the dead tables V11 drops must exist afterwards. #[test] - fn test_apply_migrations_advances_to_v13_and_drops_dead_tables() { + fn test_apply_migrations_advances_to_v14_and_drops_dead_tables() { let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); // Pre-requisite: schema_version must be bootstrapped by V1. apply_migrations(&conn).expect("apply_migrations succeeds"); - // 1. schema_version advanced to V13 + // 1. schema_version advanced to V14 let version = get_current_version(&conn).expect("read schema_version"); assert_eq!( - version, 13, - "schema_version must be 13 after all migrations" + version, 14, + "schema_version must be 14 after all migrations" ); // 2. knowledge_edges is gone (V11 drops it) @@ -848,6 +954,37 @@ mod tests { deletion_tombstone_rows, 1, "deletion_tombstones table must be created by V13" ); + + // 6. merge_plans + merge_operations exist (V14 creates them) + for table in ["merge_plans", "merge_operations"] { + let rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1", + [table], + |row| row.get(0), + ) + .expect("query sqlite_master"); + assert_eq!(rows, 1, "{table} table must be created by V14"); + } + + // 7. knowledge_nodes gains `protected` + `superseded_by` (V14) + let node_cols: Vec = { + let mut stmt = conn + .prepare("PRAGMA table_info(knowledge_nodes)") + .expect("prepare table_info"); + stmt.query_map([], |row| row.get::<_, String>(1)) + .expect("query table_info") + .filter_map(|r| r.ok()) + .collect() + }; + assert!( + node_cols.iter().any(|c| c == "protected"), + "knowledge_nodes must have `protected` column after V14" + ); + assert!( + node_cols.iter().any(|c| c == "superseded_by"), + "knowledge_nodes must have `superseded_by` column after V14" + ); } /// V11 must be idempotent on replay — if the tables were already dropped @@ -869,6 +1006,6 @@ mod tests { apply_migrations(&conn).expect("V11 replay must be idempotent"); let version = get_current_version(&conn).expect("read schema_version"); - assert_eq!(version, 13, "schema_version back at 13 after replay"); + assert_eq!(version, 14, "schema_version back at 14 after replay"); } } diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index c4e8f2b..dcd32ad 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -6279,6 +6279,988 @@ impl Storage { } Ok(result) } + + // ======================================================================== + // Merge / Supersede controls (Phase 3 — v2.1.25) + // + // Diff-previewed, confidence-gated, reversible, self-explaining + // combine/dedupe/supersede on a never-delete (bitemporal) store. + // Pure scoring/plan/op types live in `advanced::merge_supersede`. + // ======================================================================== + + /// Mark a memory protected (pinned) or unprotected. A protected memory can + /// never be auto-merged, superseded, or garbage-collected. + pub fn set_protected(&self, id: &str, protected: bool) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + let affected = writer.execute( + "UPDATE knowledge_nodes SET protected = ?1 WHERE id = ?2", + params![if protected { 1 } else { 0 }, id], + )?; + if affected == 0 { + return Err(StorageError::NotFound(id.to_string())); + } + Ok(()) + } + + /// Is this memory protected (pinned)? + pub fn is_protected(&self, id: &str) -> Result { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let v: Option = reader + .query_row( + "SELECT protected FROM knowledge_nodes WHERE id = ?1", + params![id], + |row| row.get(0), + ) + .optional()?; + match v { + Some(p) => Ok(p != 0), + None => Err(StorageError::NotFound(id.to_string())), + } + } + + /// Read the per-project merge policy (two Fellegi-Sunter thresholds + + /// auto_apply). Persisted in `fsrs_config` so it survives restarts without a + /// new table; falls back to defaults (env-overridable) when unset. + pub fn get_merge_policy(&self) -> Result { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let read_key = |key: &str| -> Option { + reader + .query_row( + "SELECT value FROM fsrs_config WHERE key = ?1", + params![key], + |row| row.get::<_, f64>(0), + ) + .optional() + .ok() + .flatten() + }; + let default = crate::advanced::MergePolicy::default(); + let env_f32 = |name: &str, fallback: f32| -> f32 { + std::env::var(name) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(fallback) + }; + let match_threshold = read_key("merge_match_threshold") + .map(|v| v as f32) + .unwrap_or_else(|| env_f32("VESTIGE_MERGE_MATCH_THRESHOLD", default.match_threshold)); + let possible_threshold = read_key("merge_possible_threshold") + .map(|v| v as f32) + .unwrap_or_else(|| { + env_f32("VESTIGE_MERGE_POSSIBLE_THRESHOLD", default.possible_threshold) + }); + let auto_apply = match read_key("merge_auto_apply") { + Some(v) => v != 0.0, + None => std::env::var("VESTIGE_MERGE_AUTO_APPLY") + .ok() + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(default.auto_apply), + }; + Ok(crate::advanced::MergePolicy::new( + match_threshold, + possible_threshold, + auto_apply, + )) + } + + /// Persist the per-project merge policy into `fsrs_config`. + pub fn set_merge_policy(&self, policy: crate::advanced::MergePolicy) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + let now = Utc::now().to_rfc3339(); + let put = |key: &str, value: f64| -> Result<()> { + writer.execute( + "INSERT OR REPLACE INTO fsrs_config (key, value, updated_at) VALUES (?1, ?2, ?3)", + params![key, value, now], + )?; + Ok(()) + }; + put("merge_match_threshold", policy.match_threshold as f64)?; + put("merge_possible_threshold", policy.possible_threshold as f64)?; + put( + "merge_auto_apply", + if policy.auto_apply { 1.0 } else { 0.0 }, + )?; + Ok(()) + } + + /// Surface likely duplicate/overlapping memory clusters with confidence + /// scores and the signals behind each (Fellegi-Sunter classified). + /// + /// Only clusters whose weakest pair scores at or above the policy's + /// `possible_threshold` are returned. Protected members are flagged so the + /// caller never auto-merges a pin. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + pub fn merge_candidates( + &self, + policy: crate::advanced::MergePolicy, + limit: usize, + tag_filter: &[String], + ) -> Result> { + use crate::advanced::{MatchClass, MergeCandidate, score_pair}; + + let all_embeddings = self.get_all_embeddings()?; + if all_embeddings.is_empty() { + return Ok(vec![]); + } + + // Load nodes for metadata. Exclude already-superseded nodes — they are + // historical and must not be re-offered for merge. + let mut node_map: std::collections::HashMap = + std::collections::HashMap::new(); + let superseded: std::collections::HashSet = self.superseded_node_ids()?; + let protected: std::collections::HashSet = self.protected_node_ids()?; + + let mut offset = 0; + loop { + let batch = self.get_all_nodes(500, offset)?; + let n = batch.len(); + for node in batch { + node_map.insert(node.id.clone(), node); + } + if n < 500 { + break; + } + offset += 500; + } + + // Candidate embeddings, filtered by tag and excluding superseded. + let items: Vec<(String, Vec)> = all_embeddings + .into_iter() + .filter(|(id, _)| !superseded.contains(id)) + .filter(|(id, _)| { + if tag_filter.is_empty() { + return true; + } + node_map + .get(id) + .map(|n| tag_filter.iter().any(|t| n.tags.contains(t))) + .unwrap_or(false) + }) + .collect(); + + let n = items.len(); + if n > 2000 { + return Err(StorageError::Init(format!( + "Too many memories to scan ({n} with embeddings). Filter by tags to reduce scope." + ))); + } + + // Union-find clustering over pairs above the possible threshold. + let mut parent: Vec = (0..n).collect(); + fn find(parent: &mut [usize], x: usize) -> usize { + let mut root = x; + while parent[root] != root { + root = parent[root]; + } + let mut cur = x; + while parent[cur] != root { + let next = parent[cur]; + parent[cur] = root; + cur = next; + } + root + } + + // Best pair score per resulting cluster member, for the explanation. + let mut pair_score: std::collections::HashMap<(usize, usize), crate::advanced::MatchSignals> = + std::collections::HashMap::new(); + + for i in 0..n { + for j in (i + 1)..n { + let sim = crate::cosine_similarity(&items[i].1, &items[j].1); + let (a_node, b_node) = (node_map.get(&items[i].0), node_map.get(&items[j].0)); + let signals = score_pair( + sim, + a_node.map(|n| n.tags.as_slice()).unwrap_or(&[]), + b_node.map(|n| n.tags.as_slice()).unwrap_or(&[]), + a_node.map(|n| n.content.as_str()).unwrap_or(""), + b_node.map(|n| n.content.as_str()).unwrap_or(""), + ); + if signals.combined_score >= policy.possible_threshold { + let ri = find(&mut parent, i); + let rj = find(&mut parent, j); + if ri != rj { + parent[ri] = rj; + } + pair_score.insert((i, j), signals); + } + } + } + + // Group indices by root. + let mut clusters: std::collections::HashMap> = + std::collections::HashMap::new(); + for i in 0..n { + let r = find(&mut parent, i); + clusters.entry(r).or_default().push(i); + } + + let mut out: Vec = Vec::new(); + for members in clusters.into_values() { + if members.len() < 2 { + continue; + } + // Cluster confidence = weakest recorded pair (the loosest link). + let mut min_score = 1.0f32; + let mut best_signals: Option = None; + for a in 0..members.len() { + for b in (a + 1)..members.len() { + let key = (members[a].min(members[b]), members[a].max(members[b])); + if let Some(sig) = pair_score.get(&key) { + if sig.combined_score < min_score { + min_score = sig.combined_score; + } + if best_signals + .as_ref() + .map(|s| sig.combined_score > s.combined_score) + .unwrap_or(true) + { + best_signals = Some(sig.clone()); + } + } + } + } + let signals = match best_signals { + Some(s) => s, + None => continue, + }; + + // Survivor = highest retention member. + let mut member_ids: Vec = + members.iter().map(|&idx| items[idx].0.clone()).collect(); + member_ids.sort_by(|a, b| { + let ra = node_map.get(a).map(|n| n.retention_strength).unwrap_or(0.0); + let rb = node_map.get(b).map(|n| n.retention_strength).unwrap_or(0.0); + rb.partial_cmp(&ra).unwrap_or(std::cmp::Ordering::Equal) + }); + let survivor_id = member_ids[0].clone(); + let has_protected_member = member_ids.iter().any(|id| protected.contains(id)); + let previews: Vec = member_ids + .iter() + .map(|id| { + node_map + .get(id) + .map(|n| preview(&n.content, 120)) + .unwrap_or_default() + }) + .collect(); + + let classification = match policy.classify(min_score) { + MatchClass::NonMatch => continue, + c => c, + }; + + out.push(MergeCandidate { + member_ids, + previews, + survivor_id, + confidence: min_score, + classification, + signals, + has_protected_member, + }); + } + + out.sort_by(|a, b| { + b.confidence + .partial_cmp(&a.confidence) + .unwrap_or(std::cmp::Ordering::Equal) + }); + out.truncate(limit); + Ok(out) + } + + /// IDs of nodes that have been bitemporally superseded (kept, but invalid). + pub fn superseded_node_ids(&self) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = + reader.prepare("SELECT id FROM knowledge_nodes WHERE superseded_by IS NOT NULL")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + let mut set = std::collections::HashSet::new(); + for r in rows { + set.insert(r?); + } + Ok(set) + } + + /// IDs of protected (pinned) nodes. + pub fn protected_node_ids(&self) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare("SELECT id FROM knowledge_nodes WHERE protected = 1")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + let mut set = std::collections::HashSet::new(); + for r in rows { + set.insert(r?); + } + Ok(set) + } + + /// Build a previewable MERGE plan (a diff) WITHOUT applying it. + /// + /// The survivor is the first id (or highest retention if unspecified). The + /// plan is persisted to `merge_plans` with status `pending` and returned for + /// inspection. Nothing about the nodes changes until `apply_plan`. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + pub fn plan_merge( + &self, + member_ids: &[String], + survivor_id: Option<&str>, + policy: crate::advanced::MergePolicy, + ) -> Result { + use crate::advanced::{ + MatchClass, PlanKind, compose_merged_content, compose_merged_tags, score_pair, + }; + + if member_ids.len() < 2 { + return Err(StorageError::Init( + "plan_merge needs at least 2 member ids".into(), + )); + } + + let mut nodes: Vec = Vec::new(); + for id in member_ids { + let node = self + .get_node(id)? + .ok_or_else(|| StorageError::NotFound(id.clone()))?; + nodes.push(node); + } + + // Protected nodes can never be absorbed. They may only be the survivor. + let survivor = match survivor_id { + Some(s) => s.to_string(), + None => { + // highest retention + nodes + .iter() + .max_by(|a, b| { + a.retention_strength + .partial_cmp(&b.retention_strength) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .map(|n| n.id.clone()) + .unwrap_or_else(|| member_ids[0].clone()) + } + }; + for node in &nodes { + if node.id != survivor && self.is_protected(&node.id)? { + return Err(StorageError::Init(format!( + "Memory {} is protected and cannot be merged away. Unprotect it first or make it the survivor.", + node.id + ))); + } + } + + // Order: survivor first, then others. + nodes.sort_by_key(|n| if n.id == survivor { 0 } else { 1 }); + + let members: Vec<(String, String)> = nodes + .iter() + .map(|n| (n.id.clone(), n.content.clone())) + .collect(); + let result_content = compose_merged_content(&members); + let result_tags = compose_merged_tags( + &nodes.iter().map(|n| n.tags.clone()).collect::>(), + ); + let result_source = nodes.iter().find(|n| n.id == survivor).and_then(|n| n.source.clone()); + let invalidated_ids: Vec = nodes + .iter() + .filter(|n| n.id != survivor) + .map(|n| n.id.clone()) + .collect(); + + // Confidence = weakest pair survivor↔absorbed. + let survivor_node = nodes.iter().find(|n| n.id == survivor).unwrap(); + let mut min_score = 1.0f32; + let mut best_signals = score_pair( + 1.0, + &survivor_node.tags, + &survivor_node.tags, + &survivor_node.content, + &survivor_node.content, + ); + for node in nodes.iter().filter(|n| n.id != survivor) { + let sim = self.pair_similarity(&survivor, &node.id)?; + let sig = score_pair( + sim, + &survivor_node.tags, + &node.tags, + &survivor_node.content, + &node.content, + ); + if sig.combined_score < min_score { + min_score = sig.combined_score; + best_signals = sig; + } + } + let classification = policy.classify(min_score); + + let plan = crate::advanced::MergePlan { + id: uuid::Uuid::new_v4().to_string(), + kind: PlanKind::Merge, + survivor_id: survivor.clone(), + member_ids: member_ids.to_vec(), + result_content, + result_tags, + result_source, + invalidated_ids, + confidence: min_score, + classification, + signals: best_signals, + explanation: format!( + "Merge {} memories into {survivor} ({}). {} memory(ies) will be bitemporally invalidated (kept for audit, marked superseded_by={survivor}).", + member_ids.len(), + match classification { + MatchClass::Match => "strong duplicate", + MatchClass::Possible => "possible duplicate — review advised", + MatchClass::NonMatch => "weak match — review strongly advised", + }, + member_ids.len() - 1 + ), + }; + + self.persist_plan(&plan)?; + Ok(plan) + } + + /// Build a previewable SUPERSEDE plan: invalidate `old_id` in favour of + /// `new_id` (bitemporal, audit-preserving) WITHOUT applying it. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + pub fn plan_supersede( + &self, + old_id: &str, + new_id: &str, + policy: crate::advanced::MergePolicy, + ) -> Result { + use crate::advanced::{PlanKind, score_pair}; + + let old = self + .get_node(old_id)? + .ok_or_else(|| StorageError::NotFound(old_id.to_string()))?; + let new = self + .get_node(new_id)? + .ok_or_else(|| StorageError::NotFound(new_id.to_string()))?; + + if self.is_protected(old_id)? { + return Err(StorageError::Init(format!( + "Memory {old_id} is protected and cannot be superseded. Unprotect it first." + ))); + } + + let sim = self.pair_similarity(old_id, new_id)?; + let signals = score_pair(sim, &old.tags, &new.tags, &old.content, &new.content); + let classification = policy.classify(signals.combined_score); + + let plan = crate::advanced::MergePlan { + id: uuid::Uuid::new_v4().to_string(), + kind: PlanKind::Supersede, + survivor_id: new_id.to_string(), + member_ids: vec![old_id.to_string(), new_id.to_string()], + result_content: new.content.clone(), + result_tags: new.tags.clone(), + result_source: new.source.clone(), + invalidated_ids: vec![old_id.to_string()], + confidence: signals.combined_score, + classification, + signals, + explanation: format!( + "Supersede {old_id} with {new_id}. {old_id} is kept and remains queryable for audit, but stamped valid_until=now and superseded_by={new_id} (invalidate, don't delete)." + ), + }; + + self.persist_plan(&plan)?; + Ok(plan) + } + + /// Cosine similarity between two nodes' stored embeddings (0 if missing). + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + fn pair_similarity(&self, a: &str, b: &str) -> Result { + let ea = self.get_node_embedding(a)?; + let eb = self.get_node_embedding(b)?; + match (ea, eb) { + (Some(ea), Some(eb)) => Ok(crate::cosine_similarity(&ea, &eb)), + _ => Ok(0.0), + } + } + + /// Persist a plan row (status pending). Idempotent on plan id. + fn persist_plan(&self, plan: &crate::advanced::MergePlan) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + let payload = serde_json::to_string(plan) + .map_err(|e| StorageError::Init(format!("plan serialize failed: {e}")))?; + let member_ids = serde_json::to_string(&plan.member_ids).unwrap_or_else(|_| "[]".into()); + writer.execute( + "INSERT OR REPLACE INTO merge_plans + (id, kind, status, created_at, applied_at, survivor_id, member_ids, confidence, classification, payload) + VALUES (?1, ?2, 'pending', ?3, NULL, ?4, ?5, ?6, ?7, ?8)", + params![ + plan.id, + plan.kind.as_str(), + Utc::now().to_rfc3339(), + plan.survivor_id, + member_ids, + plan.confidence as f64, + plan.classification.as_str(), + payload, + ], + )?; + Ok(()) + } + + /// Fetch a stored plan by id. + pub fn get_plan(&self, plan_id: &str) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let row: Option<(String, String)> = reader + .query_row( + "SELECT status, payload FROM merge_plans WHERE id = ?1", + params![plan_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + match row { + Some((_status, payload)) => { + let plan: crate::advanced::MergePlan = serde_json::from_str(&payload) + .map_err(|e| StorageError::Init(format!("plan deserialize failed: {e}")))?; + Ok(Some(plan)) + } + None => Ok(None), + } + } + + /// Plan status string (pending | applied | cancelled), if the plan exists. + pub fn plan_status(&self, plan_id: &str) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let status: Option = reader + .query_row( + "SELECT status FROM merge_plans WHERE id = ?1", + params![plan_id], + |row| row.get(0), + ) + .optional()?; + Ok(status) + } + + /// Execute a previously-generated plan by id. Everything it does is recorded + /// as a reversible [`MergeOperation`] in `merge_operations`. Returns the + /// recorded operation id. + /// + /// - **merge**: survivor content/tags are rewritten to the merged result; + /// each absorbed node is bitemporally invalidated (valid_until=now, + /// superseded_by=survivor) and kept queryable. + /// - **supersede**: old node is bitemporally invalidated in favour of new. + /// + /// `auto_apply` must be true in the policy to apply a `Match` plan without an + /// explicit `confirm`; non-`Match` plans always require `confirm=true`. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + pub fn apply_plan( + &self, + plan_id: &str, + confirm: bool, + ) -> Result { + use crate::advanced::{MatchClass, PlanKind}; + + let plan = self + .get_plan(plan_id)? + .ok_or_else(|| StorageError::NotFound(format!("plan {plan_id}")))?; + + match self.plan_status(plan_id)?.as_deref() { + Some("applied") => { + return Err(StorageError::Init(format!( + "plan {plan_id} was already applied" + ))); + } + Some("cancelled") => { + return Err(StorageError::Init(format!("plan {plan_id} was cancelled"))); + } + _ => {} + } + + // Confirmation gate: only auto-applyable Match plans may skip confirm. + let needs_confirm = !(plan.classification == MatchClass::Match); + if needs_confirm && !confirm { + return Err(StorageError::Init(format!( + "plan {plan_id} is classified '{}' (confidence {:.3}) and requires confirm=true to apply", + plan.classification.as_str(), + plan.confidence + ))); + } + + let now = Utc::now(); + let op_id = uuid::Uuid::new_v4().to_string(); + + // Snapshot everything we need to undo, BEFORE mutating. + let mut undo = serde_json::Map::new(); + undo.insert("plan_id".into(), serde_json::json!(plan_id)); + undo.insert("kind".into(), serde_json::json!(plan.kind.as_str())); + undo.insert("survivor_id".into(), serde_json::json!(plan.survivor_id)); + + match plan.kind { + PlanKind::Merge => { + let survivor = self + .get_node(&plan.survivor_id)? + .ok_or_else(|| StorageError::NotFound(plan.survivor_id.clone()))?; + undo.insert( + "survivor_prev_content".into(), + serde_json::json!(survivor.content), + ); + undo.insert( + "survivor_prev_tags".into(), + serde_json::json!(survivor.tags), + ); + + // Capture prior valid_until / superseded_by of each absorbed node. + let mut absorbed = Vec::new(); + for id in &plan.invalidated_ids { + let (vu, sb) = self.read_bitemporal(id)?; + absorbed.push(serde_json::json!({ + "id": id, + "prev_valid_until": vu, + "prev_superseded_by": sb, + })); + } + undo.insert("absorbed".into(), serde_json::json!(absorbed)); + + // Apply: rewrite survivor, invalidate absorbed. + self.rewrite_survivor( + &plan.survivor_id, + &plan.result_content, + &plan.result_tags, + )?; + for id in &plan.invalidated_ids { + self.invalidate_node(id, &plan.survivor_id, now)?; + } + } + PlanKind::Supersede => { + let old_id = &plan.member_ids[0]; + let (vu, sb) = self.read_bitemporal(old_id)?; + undo.insert( + "absorbed".into(), + serde_json::json!([{ + "id": old_id, + "prev_valid_until": vu, + "prev_superseded_by": sb, + }]), + ); + self.invalidate_node(old_id, &plan.survivor_id, now)?; + } + } + + // Record the reversible operation. + let affected: Vec = { + let mut v = vec![plan.survivor_id.clone()]; + v.extend(plan.invalidated_ids.clone()); + v + }; + let signals = serde_json::to_string(&plan.signals).unwrap_or_else(|_| "{}".into()); + { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + "INSERT INTO merge_operations + (id, plan_id, op_type, status, created_at, reverted_at, reverts_op_id, + survivor_id, affected_ids, confidence, signals, reason, undo_payload) + VALUES (?1, ?2, ?3, 'applied', ?4, NULL, NULL, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + op_id, + plan_id, + plan.kind.as_str(), + now.to_rfc3339(), + plan.survivor_id, + serde_json::to_string(&affected).unwrap_or_else(|_| "[]".into()), + plan.confidence as f64, + signals, + plan.explanation, + serde_json::Value::Object(undo).to_string(), + ], + )?; + writer.execute( + "UPDATE merge_plans SET status = 'applied', applied_at = ?1 WHERE id = ?2", + params![now.to_rfc3339(), plan_id], + )?; + } + + self.read_operation(&op_id)? + .ok_or_else(|| StorageError::Init("operation vanished after insert".into())) + } + + /// Reverse a prior merge/supersede operation by id (the "memory reflog"). + /// Restores survivor content/tags and clears the bitemporal invalidation on + /// every node the operation touched, then records a compensating `undo` op. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + pub fn merge_undo(&self, op_id: &str) -> Result { + let op = self + .read_operation(op_id)? + .ok_or_else(|| StorageError::NotFound(format!("operation {op_id}")))?; + if op.status == "reverted" { + return Err(StorageError::Init(format!( + "operation {op_id} was already reverted" + ))); + } + if op.op_type == "undo" { + return Err(StorageError::Init( + "cannot undo an undo operation".into(), + )); + } + + let undo: serde_json::Value = { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let payload: String = reader.query_row( + "SELECT undo_payload FROM merge_operations WHERE id = ?1", + params![op_id], + |row| row.get(0), + )?; + serde_json::from_str(&payload) + .map_err(|e| StorageError::Init(format!("undo payload parse failed: {e}")))? + }; + + let kind = undo.get("kind").and_then(|v| v.as_str()).unwrap_or(""); + let survivor_id = undo + .get("survivor_id") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + + // Restore survivor content/tags if this was a merge. + if kind == "merge" + && let (Some(content), Some(tags)) = ( + undo.get("survivor_prev_content").and_then(|v| v.as_str()), + undo.get("survivor_prev_tags").and_then(|v| v.as_array()), + ) + { + let tags: Vec = tags + .iter() + .filter_map(|t| t.as_str().map(|s| s.to_string())) + .collect(); + self.rewrite_survivor(&survivor_id, content, &tags)?; + } + + // Clear invalidation on every absorbed node, restoring prior values. + if let Some(absorbed) = undo.get("absorbed").and_then(|v| v.as_array()) { + for entry in absorbed { + let id = entry.get("id").and_then(|v| v.as_str()).unwrap_or_default(); + if id.is_empty() { + continue; + } + let prev_vu = entry.get("prev_valid_until").and_then(|v| v.as_str()); + let prev_sb = entry.get("prev_superseded_by").and_then(|v| v.as_str()); + self.restore_bitemporal(id, prev_vu, prev_sb)?; + } + } + + let now = Utc::now(); + let new_op_id = uuid::Uuid::new_v4().to_string(); + { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + // Mark original reverted. + writer.execute( + "UPDATE merge_operations SET status = 'reverted', reverted_at = ?1 WHERE id = ?2", + params![now.to_rfc3339(), op_id], + )?; + // Re-open the plan so it could be re-applied if desired. + if let Some(plan_id) = op.plan_id.as_deref() { + writer.execute( + "UPDATE merge_plans SET status = 'pending', applied_at = NULL WHERE id = ?1", + params![plan_id], + )?; + } + // Record compensating undo op. + writer.execute( + "INSERT INTO merge_operations + (id, plan_id, op_type, status, created_at, reverted_at, reverts_op_id, + survivor_id, affected_ids, confidence, signals, reason, undo_payload) + VALUES (?1, ?2, 'undo', 'applied', ?3, NULL, ?4, ?5, ?6, NULL, NULL, ?7, '{}')", + params![ + new_op_id, + op.plan_id, + now.to_rfc3339(), + op_id, + survivor_id, + serde_json::to_string(&op.affected_ids).unwrap_or_else(|_| "[]".into()), + format!("Reverted {} operation {op_id}", op.op_type), + ], + )?; + } + + self.read_operation(&new_op_id)? + .ok_or_else(|| StorageError::Init("undo operation vanished after insert".into())) + } + + /// List recent merge/supersede operations (the reflog), newest first. + pub fn list_merge_operations( + &self, + limit: usize, + ) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT id, plan_id, op_type, status, created_at, reverted_at, reverts_op_id, + survivor_id, affected_ids, confidence, reason + FROM merge_operations ORDER BY created_at DESC LIMIT ?1", + )?; + let rows = stmt.query_map(params![limit as i64], Self::row_to_operation)?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) + } + + /// Read a single operation by id. + fn read_operation(&self, op_id: &str) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let op = reader + .query_row( + "SELECT id, plan_id, op_type, status, created_at, reverted_at, reverts_op_id, + survivor_id, affected_ids, confidence, reason + FROM merge_operations WHERE id = ?1", + params![op_id], + Self::row_to_operation, + ) + .optional()?; + Ok(op) + } + + fn row_to_operation( + row: &rusqlite::Row, + ) -> rusqlite::Result { + let affected: String = row.get("affected_ids")?; + let affected_ids: Vec = serde_json::from_str(&affected).unwrap_or_default(); + Ok(crate::advanced::MergeOperation { + id: row.get("id")?, + plan_id: row.get("plan_id").ok().flatten(), + op_type: row.get("op_type")?, + status: row.get("status")?, + created_at: row.get("created_at")?, + reverted_at: row.get("reverted_at").ok().flatten(), + reverts_op_id: row.get("reverts_op_id").ok().flatten(), + survivor_id: row.get("survivor_id").ok().flatten(), + affected_ids, + confidence: row.get::<_, Option>("confidence").ok().flatten().map(|v| v as f32), + reason: row.get("reason").ok().flatten(), + }) + } + + /// Read (valid_until, superseded_by) for a node. + fn read_bitemporal(&self, id: &str) -> Result<(Option, Option)> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let res = reader + .query_row( + "SELECT valid_until, superseded_by FROM knowledge_nodes WHERE id = ?1", + params![id], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + )) + }, + ) + .optional()?; + res.ok_or_else(|| StorageError::NotFound(id.to_string())) + } + + /// Bitemporally invalidate a node: stamp valid_until=now and superseded_by, + /// keeping the row fully queryable (Graphiti-style invalidate, don't delete). + fn invalidate_node(&self, id: &str, superseded_by: &str, now: DateTime) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + "UPDATE knowledge_nodes + SET valid_until = ?1, superseded_by = ?2, updated_at = ?1 + WHERE id = ?3", + params![now.to_rfc3339(), superseded_by, id], + )?; + Ok(()) + } + + /// Restore a node's bitemporal columns (used by undo). + fn restore_bitemporal( + &self, + id: &str, + valid_until: Option<&str>, + superseded_by: Option<&str>, + ) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + "UPDATE knowledge_nodes + SET valid_until = ?1, superseded_by = ?2, updated_at = ?3 + WHERE id = ?4", + params![valid_until, superseded_by, Utc::now().to_rfc3339(), id], + )?; + Ok(()) + } + + /// Rewrite a survivor's content and tags (used by merge apply + undo). + /// Content rewrite regenerates the embedding via `update_node_content`. + fn rewrite_survivor(&self, id: &str, content: &str, tags: &[String]) -> Result<()> { + self.update_node_content(id, content)?; + let tags_json = serde_json::to_string(tags).unwrap_or_else(|_| "[]".into()); + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + "UPDATE knowledge_nodes SET tags = ?1, updated_at = ?2 WHERE id = ?3", + params![tags_json, Utc::now().to_rfc3339(), id], + )?; + Ok(()) + } +} + +/// Truncate `content` to `max` chars on a char boundary, collapsing newlines. +fn preview(content: &str, max: usize) -> String { + let c = content.replace('\n', " "); + if c.len() > max { + format!("{}...", &c[..c.floor_char_boundary(max)]) + } else { + c + } } // ============================================================================ @@ -6288,6 +7270,7 @@ impl Storage { #[cfg(test)] mod tests { use super::*; + use crate::advanced::{MatchClass, MergePolicy}; use tempfile::tempdir; fn create_test_storage() -> Storage { @@ -7370,4 +8353,300 @@ mod tests { .unwrap(); assert_eq!(has_content_column, 0); } + + // ======================================================================== + // Merge / Supersede controls (Phase 3 — v2.1.25) + // + // These exercise the full lifecycle without the live embedding model by + // seeding the `node_embeddings` table directly with the ACTIVE model name, + // so `get_all_embeddings` / `get_node_embedding` accept them. + // ======================================================================== + + /// Ingest a node and seed it with a controllable embedding under the active + /// model so similarity is deterministic in tests. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + fn seed_node(storage: &Storage, content: &str, tags: &[&str], vector: Vec) -> String { + let node = storage + .ingest(IngestInput { + content: content.to_string(), + node_type: "fact".to_string(), + tags: tags.iter().map(|t| t.to_string()).collect(), + ..Default::default() + }) + .unwrap(); + let bytes = Embedding::new(vector).to_bytes(); + let active = storage.embedding_service.model_name().to_string(); + let writer = storage.writer.lock().unwrap(); + writer + .execute( + "INSERT OR REPLACE INTO node_embeddings + (node_id, embedding, dimensions, model, created_at) + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + &node.id, + &bytes, + EMBEDDING_DIMENSIONS as i32, + active, + Utc::now().to_rfc3339() + ], + ) + .unwrap(); + writer + .execute( + "UPDATE knowledge_nodes SET has_embedding = 1 WHERE id = ?1", + rusqlite::params![&node.id], + ) + .unwrap(); + node.id + } + + /// A near-unit vector pointing mostly along `axis`, so two nodes sharing an + /// axis are highly similar and nodes on different axes are not. + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + fn axis_vector(axis: usize, jitter: f32) -> Vec { + let mut v = vec![0.0f32; EMBEDDING_DIMENSIONS]; + v[axis % EMBEDDING_DIMENSIONS] = 1.0; + v[(axis + 1) % EMBEDDING_DIMENSIONS] = jitter; + v + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_merge_candidates_threshold_classification() { + let storage = create_test_storage(); + // Two near-identical (same axis) — should be offered as a candidate. + let a = seed_node( + &storage, + "Use tokio runtime for async Rust services", + &["rust", "async"], + axis_vector(3, 0.02), + ); + let b = seed_node( + &storage, + "Use the tokio runtime for async Rust services", + &["rust", "async"], + axis_vector(3, 0.01), + ); + // One unrelated (different axis) — must not join the cluster. + let _c = seed_node( + &storage, + "Prefer postgres for relational data", + &["db"], + axis_vector(200, 0.0), + ); + + let policy = MergePolicy::default(); + let candidates = storage.merge_candidates(policy, 20, &[]).unwrap(); + assert_eq!(candidates.len(), 1, "exactly one duplicate cluster"); + let cluster = &candidates[0]; + assert_eq!(cluster.member_ids.len(), 2); + assert!(cluster.member_ids.contains(&a)); + assert!(cluster.member_ids.contains(&b)); + assert!( + cluster.confidence >= policy.possible_threshold, + "confidence above possible threshold" + ); + assert!(!cluster.has_protected_member); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_plan_merge_is_preview_only_no_mutation() { + let storage = create_test_storage(); + let a = seed_node(&storage, "Fact A about caching", &["perf"], axis_vector(5, 0.02)); + let b = seed_node( + &storage, + "Fact A about caching, expanded", + &["perf", "cache"], + axis_vector(5, 0.01), + ); + + let plan = storage + .plan_merge(&[a.clone(), b.clone()], None, MergePolicy::default()) + .unwrap(); + + // Plan diff is populated... + assert!(plan.result_content.contains("Fact A about caching")); + assert!(plan.result_tags.contains(&"cache".to_string())); + assert_eq!(plan.invalidated_ids.len(), 1); + + // ...but NOTHING changed: both nodes still valid, content untouched. + let na = storage.get_node(&a).unwrap().unwrap(); + let nb = storage.get_node(&b).unwrap().unwrap(); + assert_eq!(na.content, "Fact A about caching"); + assert_eq!(nb.content, "Fact A about caching, expanded"); + let (vu_a, sb_a) = storage.read_bitemporal(&a).unwrap(); + let (vu_b, sb_b) = storage.read_bitemporal(&b).unwrap(); + assert!(vu_a.is_none() && sb_a.is_none()); + assert!(vu_b.is_none() && sb_b.is_none()); + + // Plan persisted as pending. + assert_eq!(storage.plan_status(&plan.id).unwrap().as_deref(), Some("pending")); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_apply_then_undo_merge_is_reversible() { + let storage = create_test_storage(); + let survivor = seed_node(&storage, "Keep this canonical note", &["x"], axis_vector(7, 0.02)); + let absorbed = seed_node( + &storage, + "Extra detail to fold in", + &["x", "y"], + axis_vector(7, 0.01), + ); + + let plan = storage + .plan_merge( + &[survivor.clone(), absorbed.clone()], + Some(&survivor), + MergePolicy::default(), + ) + .unwrap(); + let op = storage.apply_plan(&plan.id, true).unwrap(); + assert_eq!(op.op_type, "merge"); + + // After apply: survivor content merged, absorbed bitemporally invalidated + // but STILL QUERYABLE (never deleted). + let surv = storage.get_node(&survivor).unwrap().unwrap(); + assert!(surv.content.contains("Keep this canonical note")); + assert!(surv.content.contains("Extra detail to fold in")); + assert!(surv.tags.contains(&"y".to_string())); + + let (vu, sb) = storage.read_bitemporal(&absorbed).unwrap(); + assert!(vu.is_some(), "absorbed node stamped valid_until"); + assert_eq!(sb.as_deref(), Some(survivor.as_str())); + // Old node is still fully retrievable for audit. + assert!( + storage.get_node(&absorbed).unwrap().is_some(), + "superseded node remains queryable" + ); + assert!(storage.superseded_node_ids().unwrap().contains(&absorbed)); + + // Undo restores everything. + let undo = storage.merge_undo(&op.id).unwrap(); + assert_eq!(undo.op_type, "undo"); + let surv_after = storage.get_node(&survivor).unwrap().unwrap(); + assert_eq!(surv_after.content, "Keep this canonical note"); + let (vu2, sb2) = storage.read_bitemporal(&absorbed).unwrap(); + assert!(vu2.is_none() && sb2.is_none(), "invalidation cleared on undo"); + assert!(!storage.superseded_node_ids().unwrap().contains(&absorbed)); + + // The original op is now marked reverted; double-undo is rejected. + assert!(storage.merge_undo(&op.id).is_err()); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_supersede_invalidates_old_but_keeps_it_queryable() { + let storage = create_test_storage(); + let old = seed_node(&storage, "LR should be 1e-4", &["ml"], axis_vector(9, 0.02)); + let new = seed_node( + &storage, + "Correction: LR should be 3e-4", + &["ml"], + axis_vector(9, 0.01), + ); + + let plan = storage + .plan_supersede(&old, &new, MergePolicy::default()) + .unwrap(); + // Preview did not mutate. + let (vu0, _) = storage.read_bitemporal(&old).unwrap(); + assert!(vu0.is_none()); + + let op = storage.apply_plan(&plan.id, true).unwrap(); + assert_eq!(op.op_type, "supersede"); + + let (vu, sb) = storage.read_bitemporal(&old).unwrap(); + assert!(vu.is_some(), "old stamped valid_until"); + assert_eq!(sb.as_deref(), Some(new.as_str())); + // New node untouched and valid. + let (vu_new, sb_new) = storage.read_bitemporal(&new).unwrap(); + assert!(vu_new.is_none() && sb_new.is_none()); + // Old still queryable for audit (invalidate, don't delete). + let old_node = storage.get_node(&old).unwrap().unwrap(); + assert_eq!(old_node.content, "LR should be 1e-4"); + + // And reversible. + storage.merge_undo(&op.id).unwrap(); + let (vu_r, sb_r) = storage.read_bitemporal(&old).unwrap(); + assert!(vu_r.is_none() && sb_r.is_none()); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_protect_blocks_merge_away() { + let storage = create_test_storage(); + let pinned = seed_node(&storage, "Load-bearing fact", &["pin"], axis_vector(11, 0.02)); + let other = seed_node( + &storage, + "Load-bearing fact restated", + &["pin"], + axis_vector(11, 0.01), + ); + storage.set_protected(&pinned, true).unwrap(); + assert!(storage.is_protected(&pinned).unwrap()); + + // Protected node may not be merged AWAY (survivor=other). + let err = storage.plan_merge(&[other.clone(), pinned.clone()], Some(&other), MergePolicy::default()); + assert!(err.is_err(), "merging a protected node away must fail"); + + // But it CAN be the survivor. + let ok = storage.plan_merge( + &[pinned.clone(), other.clone()], + Some(&pinned), + MergePolicy::default(), + ); + assert!(ok.is_ok(), "protected node can be the survivor"); + + // Supersede of a protected node is also blocked. + assert!( + storage.plan_supersede(&pinned, &other, MergePolicy::default()).is_err(), + "superseding a protected node must fail" + ); + + // merge_candidates flags the protected member. + let cands = storage.merge_candidates(MergePolicy::default(), 20, &[]).unwrap(); + assert!(cands.iter().all(|c| c.has_protected_member)); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_apply_requires_confirm_for_low_confidence() { + let storage = create_test_storage(); + // Tighten thresholds so a moderate pair lands in 'possible' (needs confirm). + let strict = MergePolicy::new(0.99, 0.5, false); + storage.set_merge_policy(strict).unwrap(); + + let a = seed_node(&storage, "Topic alpha note", &["t"], axis_vector(13, 0.30)); + let b = seed_node(&storage, "Topic alpha aside", &["t"], axis_vector(13, 0.60)); + let plan = storage.plan_merge(&[a, b], None, storage.get_merge_policy().unwrap()).unwrap(); + assert_ne!(plan.classification, MatchClass::Match); + + // Without confirm => rejected. + assert!(storage.apply_plan(&plan.id, false).is_err()); + // With confirm => applied. + assert!(storage.apply_plan(&plan.id, true).is_ok()); + // Re-applying an applied plan => rejected. + assert!(storage.apply_plan(&plan.id, true).is_err()); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_merge_policy_roundtrip_persists() { + let storage = create_test_storage(); + let p = MergePolicy::new(0.9, 0.6, true); + storage.set_merge_policy(p).unwrap(); + let got = storage.get_merge_policy().unwrap(); + assert!((got.match_threshold - 0.9).abs() < 1e-6); + assert!((got.possible_threshold - 0.6).abs() < 1e-6); + assert!(got.auto_apply); + } + + #[test] + fn test_set_protected_unknown_node_errors() { + let storage = create_test_storage(); + assert!(storage.set_protected("does-not-exist", true).is_err()); + } } diff --git a/crates/vestige-mcp/Cargo.toml b/crates/vestige-mcp/Cargo.toml index 6485504..f265ec5 100644 --- a/crates/vestige-mcp/Cargo.toml +++ b/crates/vestige-mcp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vestige-mcp" -version = "2.1.23" +version = "2.1.25" edition = "2024" description = "Cognitive memory MCP server for AI agents - FSRS-6, spreading activation, synaptic tagging, 3D dashboard, and 130 years of memory research" authors = ["samvallad33"] @@ -51,7 +51,7 @@ path = "src/bin/cli.rs" # Only `bundled-sqlite` is always on. `embeddings` and `vector-search` are # toggled via vestige-mcp's own feature flags below so `--no-default-features` # actually works (previously hardcoded here, which silently defeated the flag). -vestige-core = { version = "2.1.23", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] } +vestige-core = { version = "2.1.25", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] } # ============================================================================ # MCP Server Dependencies diff --git a/crates/vestige-mcp/src/server.rs b/crates/vestige-mcp/src/server.rs index a409ff5..890739b 100644 --- a/crates/vestige-mcp/src/server.rs +++ b/crates/vestige-mcp/src/server.rs @@ -328,6 +328,52 @@ impl McpServer { ..Default::default() }, // ================================================================ + // MERGE / SUPERSEDE CONTROLS (v2.1.25 — Phase 3) + // Diff-previewed, confidence-gated, reversible, never silent. + // ================================================================ + ToolDescription { + name: "merge_candidates".to_string(), + description: Some("Surface likely duplicate/overlapping memory clusters with confidence scores and the signals behind each (Fellegi-Sunter match/possible/non-match). Read-only — nothing is changed.".to_string()), + input_schema: tools::merge::merge_candidates_schema(), + ..Default::default() + }, + ToolDescription { + name: "plan_merge".to_string(), + description: Some("Produce a previewable MERGE plan (a diff: combined content/tags/provenance) for 2+ memories WITHOUT applying it. Returns a plan_id for apply_plan. Protected members block the merge.".to_string()), + input_schema: tools::merge::plan_merge_schema(), + ..Default::default() + }, + ToolDescription { + name: "plan_supersede".to_string(), + description: Some("Preview superseding memory A with B — bitemporal invalidation (stamps valid_until, keeps A queryable for audit) WITHOUT applying. Returns a plan_id for apply_plan.".to_string()), + input_schema: tools::merge::plan_supersede_schema(), + ..Default::default() + }, + ToolDescription { + name: "apply_plan".to_string(), + description: Some("Execute a previously-generated merge/supersede plan by id. Recorded as a reversible operation. Old memories are invalidated (never deleted). 'possible'/'non_match' plans require confirm=true.".to_string()), + input_schema: tools::merge::apply_plan_schema(), + ..Default::default() + }, + ToolDescription { + name: "merge_undo".to_string(), + description: Some("Reverse a prior merge/supersede operation (the 'git reflog for your agent's memory'). With no operation_id, lists the reversible operation log so you can pick one.".to_string()), + input_schema: tools::merge::merge_undo_schema(), + ..Default::default() + }, + ToolDescription { + name: "protect".to_string(), + description: Some("Pin a memory so it can never be auto-merged, superseded, or garbage-collected. Pass protected=false to unpin.".to_string()), + input_schema: tools::merge::protect_schema(), + ..Default::default() + }, + ToolDescription { + name: "merge_policy".to_string(), + description: Some("Get or set the per-project merge policy: the two Fellegi-Sunter thresholds (match_threshold, possible_threshold) and auto_apply. No args returns the current policy.".to_string()), + input_schema: tools::merge::merge_policy_schema(), + ..Default::default() + }, + // ================================================================ // COGNITIVE TOOLS (v1.5+) // ================================================================ ToolDescription { @@ -887,6 +933,14 @@ impl McpServer { } "find_duplicates" => tools::dedup::execute(&self.storage, request.arguments).await, + // ================================================================ + // MERGE / SUPERSEDE CONTROLS (v2.1.25 — Phase 3) + // ================================================================ + "merge_candidates" | "plan_merge" | "plan_supersede" | "apply_plan" | "merge_undo" + | "protect" | "merge_policy" => { + tools::merge::execute(&self.storage, request.name.as_str(), request.arguments).await + } + // ================================================================ // COGNITIVE TOOLS (v1.5+) // ================================================================ @@ -1686,8 +1740,10 @@ mod tests { let result = response.result.unwrap(); let tools = result["tools"].as_array().unwrap(); - // v2.1.21: 25 tools (includes first-class contradictions surface) - assert_eq!(tools.len(), 25, "Expected exactly 25 tools in v2.1.21"); + // v2.1.25: 32 tools (25 from v2.1.21 + 7 Phase 3 merge/supersede tools: + // merge_candidates, plan_merge, plan_supersede, apply_plan, merge_undo, + // protect, merge_policy) + assert_eq!(tools.len(), 32, "Expected exactly 32 tools in v2.1.25"); let tool_names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); @@ -1741,6 +1797,15 @@ mod tests { assert!(tool_names.contains(&"importance_score")); assert!(tool_names.contains(&"find_duplicates")); + // Merge / Supersede controls (v2.1.25 — Phase 3) + assert!(tool_names.contains(&"merge_candidates")); + assert!(tool_names.contains(&"plan_merge")); + assert!(tool_names.contains(&"plan_supersede")); + assert!(tool_names.contains(&"apply_plan")); + assert!(tool_names.contains(&"merge_undo")); + assert!(tool_names.contains(&"protect")); + assert!(tool_names.contains(&"merge_policy")); + // Cognitive tools (v1.5) assert!(tool_names.contains(&"dream")); assert!(tool_names.contains(&"explore_connections")); diff --git a/crates/vestige-mcp/src/tools/merge.rs b/crates/vestige-mcp/src/tools/merge.rs new file mode 100644 index 0000000..f836f5f --- /dev/null +++ b/crates/vestige-mcp/src/tools/merge.rs @@ -0,0 +1,530 @@ +//! Merge / Supersede control tools (Phase 3 — v2.1.25) +//! +//! Diff-previewed, confidence-gated, reversible, self-explaining +//! combine/dedupe/supersede on a never-delete (bitemporal) store. The default +//! is always preview/review — these tools never silently mutate memory. +//! +//! Tool surface (each registered as its own MCP tool name, all routed here): +//! +//! - `merge_candidates` — surface likely duplicate clusters with confidence + +//! the signals behind each (Fellegi-Sunter match / possible / non-match). +//! - `plan_merge` — previewable merge PLAN (a diff) without applying it. +//! - `plan_supersede` — preview superseding A with B (bitemporal invalidation, +//! audit-preserving) without applying. +//! - `apply_plan` — execute a previously-generated plan id; recorded as a +//! reversible operation. +//! - `merge_undo` — reverse a prior merge/supersede operation (the reflog). +//! - `protect` — pin a memory so it can never be auto-merged/superseded/forgotten. +//! - `merge_policy` — get/set the two confidence thresholds + auto_apply. +//! +//! The actual logic lives in `vestige_core` (`storage::Storage` + +//! `advanced::merge_supersede`); this layer only validates arguments and shapes +//! JSON. + +use serde_json::{Value, json}; +use std::sync::Arc; +use vestige_core::Storage; + +// ============================================================================ +// SCHEMAS +// ============================================================================ + +/// `merge_candidates` input schema. +pub fn merge_candidates_schema() -> Value { + json!({ + "type": "object", + "properties": { + "limit": { + "type": "integer", + "description": "Max candidate clusters to return (default 20).", + "default": 20, "minimum": 1, "maximum": 100 + }, + "tags": { + "type": "array", + "items": { "type": "string" }, + "description": "Optional: only consider memories with these tags (ANY match)." + } + } + }) +} + +/// `plan_merge` input schema. +pub fn plan_merge_schema() -> Value { + json!({ + "type": "object", + "properties": { + "member_ids": { + "type": "array", + "items": { "type": "string" }, + "description": "IDs of the memories to merge (>= 2). The survivor is kept; the rest are bitemporally invalidated (kept for audit)." + }, + "survivor_id": { + "type": "string", + "description": "Optional: which member to keep. Defaults to the highest-retention member." + } + }, + "required": ["member_ids"] + }) +} + +/// `plan_supersede` input schema. +pub fn plan_supersede_schema() -> Value { + json!({ + "type": "object", + "properties": { + "old_id": { "type": "string", "description": "Memory being superseded (kept, marked invalid)." }, + "new_id": { "type": "string", "description": "Memory that supersedes the old one." } + }, + "required": ["old_id", "new_id"] + }) +} + +/// `apply_plan` input schema. +pub fn apply_plan_schema() -> Value { + json!({ + "type": "object", + "properties": { + "plan_id": { "type": "string", "description": "ID of a plan produced by plan_merge / plan_supersede." }, + "confirm": { + "type": "boolean", + "description": "Required true for 'possible'/'non_match' plans. 'match' plans apply only if the policy has auto_apply=true, else confirm is required too.", + "default": false + } + }, + "required": ["plan_id"] + }) +} + +/// `merge_undo` input schema. +pub fn merge_undo_schema() -> Value { + json!({ + "type": "object", + "properties": { + "operation_id": { + "type": "string", + "description": "ID of the merge/supersede operation to reverse. Omit to list recent operations (the reflog)." + } + } + }) +} + +/// `protect` input schema. +pub fn protect_schema() -> Value { + json!({ + "type": "object", + "properties": { + "id": { "type": "string", "description": "Memory id to protect/unprotect." }, + "protected": { + "type": "boolean", + "description": "true to pin (block auto-merge/supersede/forget), false to unpin. Default true.", + "default": true + } + }, + "required": ["id"] + }) +} + +/// `merge_policy` input schema. +pub fn merge_policy_schema() -> Value { + json!({ + "type": "object", + "properties": { + "match_threshold": { + "type": "number", + "description": "Score >= this => 'match' (auto-merge eligible). 0-1.", + "minimum": 0.0, "maximum": 1.0 + }, + "possible_threshold": { + "type": "number", + "description": "Score in [possible, match) => 'possible' (review). Below => not offered. 0-1.", + "minimum": 0.0, "maximum": 1.0 + }, + "auto_apply": { + "type": "boolean", + "description": "Allow 'match'-class plans to apply without confirm. Default false (review-first)." + } + } + }) +} + +// ============================================================================ +// DISPATCH +// ============================================================================ + +/// Route a merge/supersede tool call by tool name. +pub async fn execute(storage: &Arc, tool: &str, args: Option) -> Result { + match tool { + "merge_candidates" => merge_candidates(storage, args), + "plan_merge" => plan_merge(storage, args), + "plan_supersede" => plan_supersede(storage, args), + "apply_plan" => apply_plan(storage, args), + "merge_undo" => merge_undo(storage, args), + "protect" => protect(storage, args), + "merge_policy" => merge_policy(storage, args), + other => Err(format!("unknown merge tool: {other}")), + } +} + +fn obj(args: &Option) -> serde_json::Map { + args.as_ref() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default() +} + +// ============================================================================ +// merge_candidates +// ============================================================================ + +fn merge_candidates(storage: &Arc, args: Option) -> Result { + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + let a = obj(&args); + let limit = a.get("limit").and_then(|v| v.as_u64()).unwrap_or(20) as usize; + let tags: Vec = a + .get("tags") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|t| t.as_str().map(|s| s.to_string())) + .collect() + }) + .unwrap_or_default(); + + let policy = storage.get_merge_policy().map_err(|e| e.to_string())?; + let candidates = storage + .merge_candidates(policy, limit, &tags) + .map_err(|e| e.to_string())?; + + let out: Vec = candidates + .iter() + .map(|c| { + json!({ + "memberIds": c.member_ids, + "previews": c.previews, + "survivorId": c.survivor_id, + "confidence": format!("{:.3}", c.confidence), + "classification": c.classification.as_str(), + "hasProtectedMember": c.has_protected_member, + "signals": { + "embeddingSimilarity": format!("{:.3}", c.signals.embedding_similarity), + "tagOverlap": format!("{:.3}", c.signals.tag_overlap), + "tokenOverlap": format!("{:.3}", c.signals.token_overlap), + "combinedScore": format!("{:.3}", c.signals.combined_score) + }, + "nextStep": if c.has_protected_member { + "A member is protected — unprotect it or pick it as survivor before plan_merge." + } else { + "Call plan_merge with these memberIds to preview the combined result." + } + }) + }) + .collect(); + + let policy = storage.get_merge_policy().map_err(|e| e.to_string())?; + Ok(json!({ + "candidates": out, + "totalCandidates": out.len(), + "policy": { + "matchThreshold": policy.match_threshold, + "possibleThreshold": policy.possible_threshold, + "autoApply": policy.auto_apply + }, + "note": "Nothing was changed. These are review candidates only." + })) + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + let _ = (storage, args); + Ok(json!({ "error": "Embeddings feature not enabled.", "candidates": [] })) + } +} + +// ============================================================================ +// plan_merge +// ============================================================================ + +fn plan_merge(storage: &Arc, args: Option) -> Result { + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + let a = obj(&args); + let member_ids: Vec = a + .get("member_ids") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|t| t.as_str().map(|s| s.to_string())) + .collect() + }) + .unwrap_or_default(); + if member_ids.len() < 2 { + return Err("member_ids must contain at least 2 ids".into()); + } + let survivor = a.get("survivor_id").and_then(|v| v.as_str()); + let policy = storage.get_merge_policy().map_err(|e| e.to_string())?; + let plan = storage + .plan_merge(&member_ids, survivor, policy) + .map_err(|e| e.to_string())?; + Ok(plan_to_json(&plan, &policy)) + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + let _ = (storage, args); + Err("Embeddings feature not enabled.".into()) + } +} + +// ============================================================================ +// plan_supersede +// ============================================================================ + +fn plan_supersede(storage: &Arc, args: Option) -> Result { + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + let a = obj(&args); + let old_id = a + .get("old_id") + .and_then(|v| v.as_str()) + .ok_or("old_id is required")?; + let new_id = a + .get("new_id") + .and_then(|v| v.as_str()) + .ok_or("new_id is required")?; + let policy = storage.get_merge_policy().map_err(|e| e.to_string())?; + let plan = storage + .plan_supersede(old_id, new_id, policy) + .map_err(|e| e.to_string())?; + Ok(plan_to_json(&plan, &policy)) + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + let _ = (storage, args); + Err("Embeddings feature not enabled.".into()) + } +} + +#[cfg(all(feature = "embeddings", feature = "vector-search"))] +fn plan_to_json(plan: &vestige_core::MergePlan, policy: &vestige_core::MergePolicy) -> Value { + let requires_confirm = plan.classification != vestige_core::MatchClass::Match || !policy.auto_apply; + json!({ + "planId": plan.id, + "kind": plan.kind.as_str(), + "survivorId": plan.survivor_id, + "memberIds": plan.member_ids, + "diff": { + "resultContent": plan.result_content, + "resultTags": plan.result_tags, + "resultSource": plan.result_source, + "invalidatedIds": plan.invalidated_ids + }, + "confidence": format!("{:.3}", plan.confidence), + "classification": plan.classification.as_str(), + "signals": { + "embeddingSimilarity": format!("{:.3}", plan.signals.embedding_similarity), + "tagOverlap": format!("{:.3}", plan.signals.tag_overlap), + "tokenOverlap": format!("{:.3}", plan.signals.token_overlap), + "combinedScore": format!("{:.3}", plan.signals.combined_score) + }, + "explanation": plan.explanation, + "requiresConfirm": requires_confirm, + "nextStep": format!( + "Review the diff. To execute: apply_plan with plan_id='{}'{}.", + plan.id, + if requires_confirm { " and confirm=true" } else { "" } + ), + "note": "Nothing was changed. This is a preview plan — apply_plan applies it; merge_undo reverses it." + }) +} + +// ============================================================================ +// apply_plan +// ============================================================================ + +fn apply_plan(storage: &Arc, args: Option) -> Result { + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + let a = obj(&args); + let plan_id = a + .get("plan_id") + .and_then(|v| v.as_str()) + .ok_or("plan_id is required")?; + let confirm = a.get("confirm").and_then(|v| v.as_bool()).unwrap_or(false); + let op = storage + .apply_plan(plan_id, confirm) + .map_err(|e| e.to_string())?; + Ok(json!({ + "operationId": op.id, + "opType": op.op_type, + "status": op.status, + "survivorId": op.survivor_id, + "affectedIds": op.affected_ids, + "reason": op.reason, + "appliedAt": op.created_at, + "reversible": true, + "nextStep": format!("To reverse this, call merge_undo with operation_id='{}'.", op.id), + "note": "Old memories were bitemporally invalidated (valid_until stamped), NOT deleted. They remain queryable for audit." + })) + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + let _ = (storage, args); + Err("Embeddings feature not enabled.".into()) + } +} + +// ============================================================================ +// merge_undo (also lists the reflog when no id given) +// ============================================================================ + +fn merge_undo(storage: &Arc, args: Option) -> Result { + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + let a = obj(&args); + match a.get("operation_id").and_then(|v| v.as_str()) { + Some(op_id) => { + let op = storage.merge_undo(op_id).map_err(|e| e.to_string())?; + Ok(json!({ + "undoOperationId": op.id, + "revertedOperationId": op.reverts_op_id, + "status": "reverted", + "affectedIds": op.affected_ids, + "reason": op.reason, + "note": "The original operation was reversed: survivor content/tags restored and invalidation cleared. The plan is re-openable." + })) + } + None => { + // No id => return the reflog so the caller can pick one. + let ops = storage.list_merge_operations(20).map_err(|e| e.to_string())?; + let log: Vec = ops + .iter() + .map(|op| { + json!({ + "operationId": op.id, + "opType": op.op_type, + "status": op.status, + "survivorId": op.survivor_id, + "affectedIds": op.affected_ids, + "confidence": op.confidence.map(|c| format!("{:.3}", c)), + "reason": op.reason, + "createdAt": op.created_at, + "revertedAt": op.reverted_at + }) + }) + .collect(); + Ok(json!({ + "operations": log, + "totalOperations": log.len(), + "note": "This is the reversible operation log (the memory reflog). Pass operation_id to reverse one." + })) + } + } + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + let _ = (storage, args); + Err("Embeddings feature not enabled.".into()) + } +} + +// ============================================================================ +// protect +// ============================================================================ + +fn protect(storage: &Arc, args: Option) -> Result { + let a = obj(&args); + let id = a + .get("id") + .and_then(|v| v.as_str()) + .ok_or("id is required")?; + let protected = a.get("protected").and_then(|v| v.as_bool()).unwrap_or(true); + storage + .set_protected(id, protected) + .map_err(|e| e.to_string())?; + Ok(json!({ + "id": id, + "protected": protected, + "note": if protected { + "Memory pinned. It can never be auto-merged, superseded, or garbage-collected until unprotected." + } else { + "Memory unprotected. It is now eligible for merge/supersede/forget again." + } + })) +} + +// ============================================================================ +// merge_policy (get when no args, set otherwise) +// ============================================================================ + +fn merge_policy(storage: &Arc, args: Option) -> Result { + let a = obj(&args); + let current = storage.get_merge_policy().map_err(|e| e.to_string())?; + + let has_update = a.contains_key("match_threshold") + || a.contains_key("possible_threshold") + || a.contains_key("auto_apply"); + + if has_update { + let match_t = a + .get("match_threshold") + .and_then(|v| v.as_f64()) + .map(|v| v as f32) + .unwrap_or(current.match_threshold); + let possible_t = a + .get("possible_threshold") + .and_then(|v| v.as_f64()) + .map(|v| v as f32) + .unwrap_or(current.possible_threshold); + let auto = a + .get("auto_apply") + .and_then(|v| v.as_bool()) + .unwrap_or(current.auto_apply); + let policy = vestige_core::MergePolicy::new(match_t, possible_t, auto); + storage.set_merge_policy(policy).map_err(|e| e.to_string())?; + Ok(json!({ + "updated": true, + "matchThreshold": policy.match_threshold, + "possibleThreshold": policy.possible_threshold, + "autoApply": policy.auto_apply, + "note": "Policy saved. Fellegi-Sunter: score>=match => auto-merge eligible; [possible,match) => review; below => not offered." + })) + } else { + Ok(json!({ + "matchThreshold": current.match_threshold, + "possibleThreshold": current.possible_threshold, + "autoApply": current.auto_apply, + "note": "Two-threshold merge policy. Pass match_threshold / possible_threshold / auto_apply to change it." + })) + } +} + +// ============================================================================ +// TESTS — see tests/merge_supersede_test.rs for full integration coverage. +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schemas_are_objects() { + for s in [ + merge_candidates_schema(), + plan_merge_schema(), + plan_supersede_schema(), + apply_plan_schema(), + merge_undo_schema(), + protect_schema(), + merge_policy_schema(), + ] { + assert_eq!(s["type"], "object"); + } + } + + #[test] + fn plan_merge_requires_two_ids() { + assert!(plan_merge_schema()["required"] + .as_array() + .unwrap() + .iter() + .any(|v| v == "member_ids")); + } +} diff --git a/crates/vestige-mcp/src/tools/mod.rs b/crates/vestige-mcp/src/tools/mod.rs index 260869e..a2c3e24 100644 --- a/crates/vestige-mcp/src/tools/mod.rs +++ b/crates/vestige-mcp/src/tools/mod.rs @@ -24,6 +24,9 @@ pub mod maintenance; pub mod dedup; pub mod importance; +// v2.1.25: Merge / Supersede controls (Phase 3) +pub mod merge; + // v1.5: Cognitive tools pub mod dream; pub mod explore; diff --git a/docs/MERGE_SUPERSEDE.md b/docs/MERGE_SUPERSEDE.md new file mode 100644 index 0000000..a35fb06 --- /dev/null +++ b/docs/MERGE_SUPERSEDE.md @@ -0,0 +1,152 @@ +# Merge / Supersede Controls (Phase 3) + +> Diff-previewed, confidence-gated, reversible, self-explaining +> combine/dedupe/supersede on a never-delete (bitemporal) store. + +Memory systems accumulate duplicates, near-duplicates, and outdated facts. The +naive fixes are all bad: dumb hashing under-merges (misses paraphrases), +aggressive LLM merging over-merges and destroys the audit trail, and +auto-deleting on contradiction silently loses information. Vestige's Phase 3 +takes the opposite stance: + +- **Opt-in, never silent.** The default is preview/review. Nothing mutates your + memory unless you explicitly apply a plan. +- **Diff-previewed.** `plan_merge` / `plan_supersede` show exactly what *would* + change before anything does. +- **Confidence-gated.** A Fellegi-Sunter two-threshold score classifies each + candidate as `match` / `possible` / `non_match`. +- **Reversible.** Every applied operation is recorded with an undo payload — a + *git reflog for your agent's memory*. +- **Self-explaining.** Each candidate carries the signals that explain *why* two + memories were judged duplicates. +- **Audit-preserving.** Superseding does not delete: it stamps `valid_until` and + keeps the old memory queryable (Graphiti-style "invalidate, don't delete"). + +## The bitemporal model: invalidate, don't delete + +Superseding memory A with memory B does **not** erase A. Instead: + +- `A.valid_until` is stamped with the supersede time. +- `A.superseded_by` is set to `B.id` (a lineage pointer). +- A remains fully queryable for audit. Searches and timelines can still surface + it; it is simply marked as no longer the current truth. + +This reuses the existing `valid_from` / `valid_until` columns on +`knowledge_nodes` (migration V2) plus a new `superseded_by` column (migration +V14). Merges work the same way: the survivor absorbs the others' content, and +each absorbed node is bitemporally invalidated rather than deleted. + +## Fellegi-Sunter two-threshold scoring + +Candidate scoring combines three signals into a weighted score in `[0, 1]`: + +| Signal | Weight | Source | +| ----------------------- | -----: | ------------------------------------------ | +| Embedding cosine sim | 0.70 | stored embeddings (`node_embeddings`) | +| Tag overlap (Jaccard) | 0.15 | `knowledge_nodes.tags` | +| Content token overlap | 0.15 | Jaccard over content tokens (len > 2) | + +The combined score is then classified against **two** thresholds: + +``` +score >= match_threshold => "match" (auto-merge eligible) +possible_threshold <= score => "possible" (surfaced for review) +score < possible_threshold => "non_match" (never offered) +``` + +Defaults: `match_threshold = 0.86`, `possible_threshold = 0.72`. The two-band +design means borderline cases are surfaced for review instead of being +force-decided in either direction. + +A cluster's confidence is the **weakest** pairwise score within it (the loosest +link), so a cluster is only as confident as its least-similar member. + +## The reversible operation log (the "memory reflog") + +Every applied merge/supersede writes one row to `merge_operations`: + +- `op_type` — `merge` | `supersede` | `undo` +- `status` — `applied` | `reverted` +- `survivor_id`, `affected_ids` — what was touched +- `confidence`, `signals` — the score and *why* the memories combined +- `reason` — a human-readable explanation +- `undo_payload` — a JSON snapshot capturing everything needed to reverse it + +`merge_undo` consumes the undo payload to restore the survivor's prior +content/tags and clear the bitemporal invalidation on every affected node, then +records a compensating `undo` operation. Calling `merge_undo` with no +`operation_id` returns the operation log so you can pick one. + +## Memory protection (pinning) + +`protect` sets the `protected` flag on a memory. A protected memory: + +- is never offered for auto-merge (it is flagged in `merge_candidates`), +- cannot be merged *away* (it may only be the survivor of a merge), +- cannot be superseded, +- is excluded from garbage collection. + +Pass `protected: false` to unpin. + +## Tool surface + +| Tool | Mutates? | Purpose | +| ------------------ | :------: | ------------------------------------------------------------------------- | +| `merge_candidates` | No | Surface likely duplicate clusters with confidence + signals. | +| `plan_merge` | No | Preview a merge of 2+ memories (a diff). Returns a `plan_id`. | +| `plan_supersede` | No | Preview superseding A with B (bitemporal). Returns a `plan_id`. | +| `apply_plan` | **Yes** | Execute a plan by id; recorded as a reversible operation. | +| `merge_undo` | **Yes** | Reverse an operation, or list the operation log when given no id. | +| `protect` | **Yes** | Pin / unpin a memory so it can never be auto-merged/superseded/forgotten. | +| `merge_policy` | **Yes** | Get/set the two thresholds + `auto_apply`. | + +### Typical flow + +```text +1. merge_candidates -> review clusters + confidence + signals +2. plan_merge { member_ids: [...] } -> inspect the diff, get plan_id +3. apply_plan { plan_id, confirm } -> apply; get operation_id (reversible) +4. merge_undo { operation_id } -> reverse if it was wrong +``` + +`apply_plan` requires `confirm: true` for `possible` / `non_match` plans. A +`match` plan applies without `confirm` only when the policy has +`auto_apply: true` (default `false`). + +## Configuration + +The merge policy persists per project (stored in `fsrs_config`). It can also be +overridden via environment variables: + +| Variable | Meaning | +| ----------------------------------- | ------------------------------------ | +| `VESTIGE_MERGE_MATCH_THRESHOLD` | Score ≥ this ⇒ `match`. | +| `VESTIGE_MERGE_POSSIBLE_THRESHOLD` | Score ≥ this ⇒ at least `possible`. | +| `VESTIGE_MERGE_AUTO_APPLY` | `1`/`true` to allow auto-apply. | + +A persisted policy (set via `merge_policy`) takes precedence over the +environment, which takes precedence over the built-in defaults. When +`vestige.toml` configuration lands, the policy will read from there as well. + +## Schema (migration V14) + +- `knowledge_nodes.protected INTEGER NOT NULL DEFAULT 0` +- `knowledge_nodes.superseded_by TEXT` +- `merge_plans(id, kind, status, created_at, applied_at, survivor_id, + member_ids, confidence, classification, payload)` +- `merge_operations(id, plan_id, op_type, status, created_at, reverted_at, + reverts_op_id, survivor_id, affected_ids, confidence, signals, reason, + undo_payload)` + +The two `ALTER TABLE ... ADD COLUMN` statements are applied with duplicate-column +guards so the migration is idempotent on replay; the rest of V14 uses +`CREATE ... IF NOT EXISTS`. + +## Anti-patterns this design avoids + +- **Silently double-storing contradictions.** Merge composition attributes and + de-duplicates content instead of blindly concatenating or dropping it. +- **Auto-deleting on contradiction.** Supersede invalidates bitemporally; the + old memory is retained and queryable. +- **Trading away the audit trail for auto-merge convenience.** Every operation is + logged and reversible, with provenance for why memories combined. diff --git a/package.json b/package.json index 9d759a6..20bb78a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "vestige", - "version": "2.1.23", + "version": "2.1.25", "private": true, "description": "Cognitive memory for AI - MCP server with FSRS-6 spaced repetition", "author": "Sam Valladares", diff --git a/packages/vestige-init/package.json b/packages/vestige-init/package.json index dce6675..cb9b4f7 100644 --- a/packages/vestige-init/package.json +++ b/packages/vestige-init/package.json @@ -1,6 +1,6 @@ { "name": "@vestige/init", - "version": "2.1.23", + "version": "2.1.25", "description": "Configure Vestige local memory for MCP-compatible AI agents", "bin": { "vestige-init": "bin/init.js" diff --git a/packages/vestige-mcp-npm/package.json b/packages/vestige-mcp-npm/package.json index 1ff860f..a7e7829 100644 --- a/packages/vestige-mcp-npm/package.json +++ b/packages/vestige-mcp-npm/package.json @@ -1,6 +1,6 @@ { "name": "vestige-mcp-server", - "version": "2.1.23", + "version": "2.1.25", "mcpName": "io.github.samvallad33/vestige", "description": "Vestige MCP Server — local cognitive memory for MCP-compatible AI agents", "bin": { diff --git a/server.json b/server.json index e11c5a4..2b9a927 100644 --- a/server.json +++ b/server.json @@ -7,12 +7,12 @@ "url": "https://github.com/samvallad33/vestige", "source": "github" }, - "version": "2.1.23", + "version": "2.1.25", "packages": [ { "registryType": "npm", "identifier": "vestige-mcp-server", - "version": "2.1.23", + "version": "2.1.25", "transport": { "type": "stdio" } From 51f08264f7d18c8e3045f81d85bae55b56a59ecb Mon Sep 17 00:00:00 2001 From: brendon Date: Mon, 15 Jun 2026 13:50:55 -0500 Subject: [PATCH 19/38] fix(storage): tolerate SQLite-native datetime format in parse_timestamp Tolerate SQLite-native timestamps from external writers while preserving RFC3339 as the canonical write format. Verified locally on the merge result: - cargo test -p vestige-core test_parse_timestamp_accepts_rfc3339_and_sqlite_native --no-fail-fast CI/Test Suite on the updated PR branch are green. --- crates/vestige-core/src/storage/sqlite.rs | 71 ++++++++++++++++++----- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index dcd32ad..4cd32e8 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -2,7 +2,7 @@ //! //! Core storage layer with integrated embeddings and vector search. -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; use directories::{BaseDirs, ProjectDirs}; #[cfg(all(feature = "embeddings", feature = "vector-search"))] use lru::LruCache; @@ -1060,20 +1060,41 @@ impl Storage { Ok(node) } - /// Parse RFC3339 timestamp + /// Parse a stored timestamp into a UTC `DateTime`. + /// + /// The canonical on-disk format is RFC 3339 (every Rust writer in this + /// crate uses `DateTime::to_rfc3339()`). However, timestamps can also be + /// written by external tooling that bypasses this storage layer — most + /// notably session hooks or manual maintenance that touch the DB with raw + /// `sqlite3`. SQLite's native `datetime('now')` / `CURRENT_TIMESTAMP` + /// emit a space-separated, timezone-less `YYYY-MM-DD HH:MM:SS[.fff]` + /// string that `parse_from_rfc3339` rejects, which would otherwise make + /// every affected row unreadable. + /// + /// We therefore parse RFC 3339 first and fall back to the SQLite-native + /// format (assumed UTC) so the store stays tolerant of either writer. fn parse_timestamp(value: &str, field_name: &str) -> rusqlite::Result> { - DateTime::parse_from_rfc3339(value) - .map(|dt| dt.with_timezone(&Utc)) - .map_err(|e| { - rusqlite::Error::FromSqlConversionFailure( - 0, - rusqlite::types::Type::Text, - Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid {} timestamp '{}': {}", field_name, value, e), - )), - ) - }) + if let Ok(dt) = DateTime::parse_from_rfc3339(value) { + return Ok(dt.with_timezone(&Utc)); + } + + // Fallback: SQLite-native "YYYY-MM-DD HH:MM:SS" (with optional + // fractional seconds), which has no timezone and is assumed UTC. + if let Ok(naive) = NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f") { + return Ok(naive.and_utc()); + } + + Err(rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "Invalid {} timestamp '{}': not RFC 3339 or SQLite datetime format", + field_name, value + ), + )), + )) } /// Convert a row to KnowledgeNode @@ -7376,6 +7397,28 @@ mod tests { assert_eq!(stats.total_nodes, 0); } + #[test] + fn test_parse_timestamp_accepts_rfc3339_and_sqlite_native() { + use chrono::TimeZone; + + // Canonical writer: RFC 3339 with fractional seconds + offset. + let rfc = Storage::parse_timestamp("2026-06-12T15:07:59.730+00:00", "last_accessed").unwrap(); + assert_eq!(rfc.to_rfc3339(), "2026-06-12T15:07:59.730+00:00"); + + // External writer: SQLite-native `datetime('now')` (space separator, + // no timezone, no fraction) — must be tolerated, assumed UTC. + let sqlite = Storage::parse_timestamp("2026-06-12 15:07:59", "last_accessed").unwrap(); + assert_eq!(sqlite, Utc.with_ymd_and_hms(2026, 6, 12, 15, 7, 59).unwrap()); + + // SQLite-native with fractional seconds. + let sqlite_frac = + Storage::parse_timestamp("2026-06-12 15:07:59.730", "last_accessed").unwrap(); + assert_eq!(sqlite_frac.timestamp_subsec_millis(), 730); + + // Genuinely malformed input still errors. + assert!(Storage::parse_timestamp("not-a-timestamp", "last_accessed").is_err()); + } + #[test] fn test_ingest_and_get() { let storage = create_test_storage(); From 47de61f2d23c531bde070593a36f8a703d789f40 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Mon, 15 Jun 2026 13:49:15 -0500 Subject: [PATCH 20/38] =?UTF-8?q?feat(config):=20Phase=202=20Configurable?= =?UTF-8?q?=20Output=20=E2=80=94=20vestige.toml=20+=20output=20profiles=20?= =?UTF-8?q?(v2.1.26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased on v2.1.25 merge/supersede and bumped the post-release metadata to v2.1.26 so this branch does not roll versions backward. Adds local vestige.toml defaults, output profiles, and MCP response precedence for search, timeline, codebase context, and session context. Verified: - cargo metadata --format-version 1 --locked --no-deps - cargo test -p vestige-core config --no-fail-fast - cargo test -p vestige-mcp config --no-fail-fast --- CHANGELOG.md | 44 ++ Cargo.lock | 4 +- Cargo.toml | 2 +- apps/dashboard/package.json | 2 +- crates/vestige-core/Cargo.toml | 2 +- crates/vestige-core/src/config.rs | 388 ++++++++++++++++++ crates/vestige-core/src/lib.rs | 5 + crates/vestige-mcp/Cargo.toml | 4 +- crates/vestige-mcp/src/server.rs | 82 +++- .../vestige-mcp/src/tools/codebase_unified.rs | 71 +++- .../vestige-mcp/src/tools/search_unified.rs | 206 ++++++++-- .../vestige-mcp/src/tools/session_context.rs | 67 ++- crates/vestige-mcp/src/tools/timeline.rs | 83 ++-- docs/CONFIGURATION.md | 87 ++++ package.json | 2 +- packages/vestige-init/package.json | 2 +- packages/vestige-mcp-npm/package.json | 2 +- server.json | 4 +- 18 files changed, 931 insertions(+), 126 deletions(-) create mode 100644 crates/vestige-core/src/config.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dec084c..5edf533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.1.26] - 2026-06-15 — "Configurable Output" + +Roadmap **Phase 2: Configurable Output**. Users can now control the default +shape and size of high-traffic MCP responses with an optional, local-first +config file — without recompiling and without a cloud service. The default +behavior is unchanged: a fresh install with no `vestige.toml` behaves exactly +as before. + +### Added + +- **`vestige.toml` config file**, loaded from the active Vestige data directory + (`/vestige.toml`, alongside `vestige.db`). A missing or malformed + file falls back to built-in defaults, so existing installs are unaffected. +- **`[defaults]` table** with three keys: `detail_level` + (`brief` | `summary` | `full`), `limit` (default result count for + high-traffic tools), and `profile`. +- **Output profiles** — `lean`, `default`, `audit`, `research` — each presetting + a coherent bundle of detail level, result limit, and whether scores and + timestamps are included: + - `lean`: `brief` detail, limit 5, scores and timestamps dropped (smallest + context cost). + - `default`: historical behavior — `summary` detail, tool's own default + limit, scores and timestamps present. **Unchanged.** + - `audit`: `full` detail with every field, score, and timestamp. + - `research`: `full` detail with a larger default limit (25). +- **Three-layer precedence**, applied per call: an explicit MCP parameter wins + over the config file, which wins over the built-in default. +- **`profile` field** echoed in `search`, `memory_timeline`, `codebase` + (`get_context`), and `session_context` responses so the active profile is + observable. + +### Changed + +- `search`, `memory_timeline`, `codebase` (`get_context`), and + `session_context` now resolve their default detail level and result limit + through the config file when no explicit parameter is supplied. With no + `vestige.toml` present, their output is byte-for-byte identical to v2.1.25. + +### Documentation + +- `docs/CONFIGURATION.md` gains a **Output Configuration (`vestige.toml`)** + section documenting the file location, `[defaults]` keys, profile presets, + and precedence rules. + ## [2.1.25] - 2026-06-12 — "Merge / Supersede Controls" v2.1.25 ships Phase 3: diff-previewed, confidence-gated, reversible, diff --git a/Cargo.lock b/Cargo.lock index 33fe576..0b613a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4629,7 +4629,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vestige-core" -version = "2.1.25" +version = "2.1.26" dependencies = [ "candle-core", "chrono", @@ -4665,7 +4665,7 @@ dependencies = [ [[package]] name = "vestige-mcp" -version = "2.1.25" +version = "2.1.26" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 1c89455..7183f40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ exclude = [ ] [workspace.package] -version = "2.1.25" +version = "2.1.26" edition = "2024" license = "AGPL-3.0-only" repository = "https://github.com/samvallad33/vestige" diff --git a/apps/dashboard/package.json b/apps/dashboard/package.json index b35ef9f..7b826a2 100644 --- a/apps/dashboard/package.json +++ b/apps/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "@vestige/dashboard", - "version": "2.1.25", + "version": "2.1.26", "private": true, "type": "module", "scripts": { diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index c66c369..e878cdb 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vestige-core" -version = "2.1.25" +version = "2.1.26" edition = "2024" rust-version = "1.91" authors = ["Vestige Team"] diff --git a/crates/vestige-core/src/config.rs b/crates/vestige-core/src/config.rs new file mode 100644 index 0000000..f1bb8d5 --- /dev/null +++ b/crates/vestige-core/src/config.rs @@ -0,0 +1,388 @@ +//! Vestige configuration file (`vestige.toml`). +//! +//! Phase 2 "Configurable Output" of the adoption roadmap. A small, optional +//! config file lives alongside the SQLite database in the active Vestige data +//! directory (`/vestige.toml`). It lets users tune the default shape +//! of high-traffic MCP responses (detail level, result limit, output profile) +//! without recompiling and without a cloud service. +//! +//! Precedence, from highest to lowest: +//! +//! 1. An explicit MCP call parameter (e.g. `detail_level` on a `search` call). +//! 2. The config file `[defaults]` (and the selected output profile). +//! 3. The built-in default, which preserves the historical behavior so nothing +//! changes for users who never write a `vestige.toml`. +//! +//! The parser is intentionally a tiny, dependency-free subset of TOML: section +//! headers (`[defaults]`) and `key = value` lines with string or integer +//! values. This keeps the local-first binary lean and avoids pulling a full +//! TOML crate into the dependency tree for a three-key schema. Unknown keys and +//! unknown sections are ignored so the file can grow in future phases without +//! breaking older binaries. + +use std::path::{Path, PathBuf}; + +/// Canonical config file name, resolved inside the active data directory. +pub const CONFIG_FILE: &str = "vestige.toml"; + +/// Output profiles preset a coherent bundle of detail/field choices. +/// +/// `Default` MUST reproduce the pre-Phase-2 behavior exactly so existing users +/// see no change. The other profiles are opt-in presets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum OutputProfile { + /// Smallest responses: brief detail, scores and timestamps suppressed. + /// Use when context budget matters more than provenance. + Lean, + /// Historical behavior. `summary` detail with content + dates. Unchanged. + #[default] + Default, + /// Maximum provenance: `full` detail with every field, score, and timestamp. + /// Use when reviewing or debugging memory state. + Audit, + /// Like `audit` but tuned for larger result sets (higher default limit). + Research, +} + +impl OutputProfile { + /// Parse a profile name. Returns `None` for unknown names so the caller can + /// decide whether that is an error (MCP param) or ignorable (config file). + pub fn from_name(name: &str) -> Option { + match name.trim().to_ascii_lowercase().as_str() { + "lean" => Some(Self::Lean), + "default" => Some(Self::Default), + "audit" => Some(Self::Audit), + "research" => Some(Self::Research), + _ => None, + } + } + + /// Canonical lowercase name, suitable for echoing back in responses. + pub fn as_str(self) -> &'static str { + match self { + Self::Lean => "lean", + Self::Default => "default", + Self::Audit => "audit", + Self::Research => "research", + } + } + + /// The detail level this profile presets when the user has not set one + /// explicitly via an MCP param or `[defaults] detail_level`. + pub fn detail_level(self) -> &'static str { + match self { + Self::Lean => "brief", + Self::Default => "summary", + Self::Audit | Self::Research => "full", + } + } + + /// The result limit this profile presets when the user has not set one + /// explicitly. `None` means "use the tool's own historical default", which + /// keeps `default` fully backward-compatible. + pub fn limit(self) -> Option { + match self { + Self::Lean => Some(5), + Self::Default => None, + Self::Audit => None, + Self::Research => Some(25), + } + } + + /// Whether scores (combined/keyword/semantic) should be shown by default. + /// Lean drops them to save tokens; the rest keep whatever the detail level + /// already includes. + pub fn show_scores(self) -> bool { + !matches!(self, Self::Lean) + } + + /// Whether timestamps should be shown by default. Lean drops them. + pub fn show_timestamps(self) -> bool { + !matches!(self, Self::Lean) + } +} + +/// The `[defaults]` table from `vestige.toml`. All fields optional. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct OutputDefaults { + /// Default detail level (`brief` | `summary` | `full`). Overrides the + /// profile's preset detail level when set. + pub detail_level: Option, + /// Default result limit for high-traffic tools. Overrides the profile's + /// preset limit when set. + pub limit: Option, + /// Selected output profile. Defaults to `default` (historical behavior). + pub profile: OutputProfile, +} + +/// Parsed `vestige.toml`. Currently only the `[defaults]` table is meaningful; +/// the struct exists so future phases can add tables without churn. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct VestigeConfig { + pub defaults: OutputDefaults, +} + +impl VestigeConfig { + /// Resolve the config path for a given data directory. + pub fn path_for_data_dir(data_dir: &Path) -> PathBuf { + data_dir.join(CONFIG_FILE) + } + + /// Load config from a data directory. A missing or unreadable file yields + /// the built-in default (never an error) so a fresh install just works. + /// A present-but-malformed file is parsed leniently: only well-formed lines + /// are honored. + pub fn load_from_data_dir(data_dir: &Path) -> Self { + let path = Self::path_for_data_dir(data_dir); + match std::fs::read_to_string(&path) { + Ok(contents) => Self::parse(&contents), + Err(_) => Self::default(), + } + } + + /// Parse the minimal TOML subset. Lenient by design. + pub fn parse(contents: &str) -> Self { + let mut config = Self::default(); + let mut section = String::new(); + + for raw in contents.lines() { + let line = strip_comment(raw).trim(); + if line.is_empty() { + continue; + } + + if let Some(name) = line.strip_prefix('[').and_then(|s| s.strip_suffix(']')) { + section = name.trim().to_ascii_lowercase(); + continue; + } + + let Some((key, value)) = line.split_once('=') else { + continue; + }; + let key = key.trim().to_ascii_lowercase(); + let value = unquote(value.trim()); + + if section == "defaults" { + match key.as_str() { + "detail_level" => { + let v = value.trim().to_ascii_lowercase(); + if matches!(v.as_str(), "brief" | "summary" | "full") { + config.defaults.detail_level = Some(v); + } + } + "limit" => { + if let Ok(n) = value.trim().parse::() + && n > 0 + { + config.defaults.limit = Some(n); + } + } + "profile" => { + if let Some(p) = OutputProfile::from_name(&value) { + config.defaults.profile = p; + } + } + _ => {} + } + } + } + + config + } + + /// Effective output config after applying the profile, with `[defaults]` + /// detail_level / limit overriding the profile presets. + pub fn output(&self) -> OutputConfig { + let profile = self.defaults.profile; + OutputConfig { + profile, + detail_level: self + .defaults + .detail_level + .clone() + .unwrap_or_else(|| profile.detail_level().to_string()), + limit: self.defaults.limit.or_else(|| profile.limit()), + show_scores: profile.show_scores(), + show_timestamps: profile.show_timestamps(), + } + } +} + +/// The resolved, ready-to-apply output configuration handed to MCP tools. +/// +/// Tools treat each field as the *fallback* used only when the corresponding +/// explicit MCP call parameter is absent, preserving the precedence +/// `MCP param > config file > built-in default`. +#[derive(Debug, Clone, PartialEq)] +pub struct OutputConfig { + pub profile: OutputProfile, + pub detail_level: String, + pub limit: Option, + pub show_scores: bool, + pub show_timestamps: bool, +} + +impl Default for OutputConfig { + /// The built-in default == the historical behavior == the `default` profile. + fn default() -> Self { + VestigeConfig::default().output() + } +} + +impl OutputConfig { + /// Resolve the detail level to use, given an optional explicit MCP param. + /// Explicit param always wins (precedence layer 1). + pub fn resolve_detail_level(&self, explicit: Option<&str>) -> String { + explicit + .map(|s| s.to_string()) + .unwrap_or_else(|| self.detail_level.clone()) + } + + /// Resolve the limit to use, given an optional explicit MCP param and the + /// tool's own built-in fallback (used only when neither param nor config + /// supplies one). + pub fn resolve_limit(&self, explicit: Option, builtin_default: i32) -> i32 { + explicit + .or(self.limit) + .unwrap_or(builtin_default) + } +} + +/// Strip a `#` comment that is not inside a quoted string. +fn strip_comment(line: &str) -> &str { + let mut in_quotes = false; + for (idx, ch) in line.char_indices() { + match ch { + '"' => in_quotes = !in_quotes, + '#' if !in_quotes => return &line[..idx], + _ => {} + } + } + line +} + +/// Remove a single layer of matching surrounding double quotes, if present. +fn unquote(value: &str) -> String { + let bytes = value.as_bytes(); + if bytes.len() >= 2 && bytes[0] == b'"' && bytes[bytes.len() - 1] == b'"' { + value[1..value.len() - 1].to_string() + } else { + value.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_preserves_historical_behavior() { + let out = OutputConfig::default(); + assert_eq!(out.profile, OutputProfile::Default); + assert_eq!(out.detail_level, "summary"); + assert_eq!(out.limit, None); + assert!(out.show_scores); + assert!(out.show_timestamps); + } + + #[test] + fn empty_or_missing_file_is_default() { + assert_eq!(VestigeConfig::parse(""), VestigeConfig::default()); + assert_eq!(VestigeConfig::parse("\n\n# just a comment\n"), VestigeConfig::default()); + } + + #[test] + fn parses_defaults_table() { + let cfg = VestigeConfig::parse( + r#" + [defaults] + detail_level = "full" + limit = 25 + profile = "research" + "#, + ); + assert_eq!(cfg.defaults.detail_level.as_deref(), Some("full")); + assert_eq!(cfg.defaults.limit, Some(25)); + assert_eq!(cfg.defaults.profile, OutputProfile::Research); + } + + #[test] + fn unquoted_and_commented_values() { + let cfg = VestigeConfig::parse( + "[defaults]\nprofile = lean # inline comment\nlimit = 7\n", + ); + assert_eq!(cfg.defaults.profile, OutputProfile::Lean); + assert_eq!(cfg.defaults.limit, Some(7)); + } + + #[test] + fn invalid_values_are_ignored() { + let cfg = VestigeConfig::parse( + "[defaults]\ndetail_level = \"loud\"\nlimit = -3\nprofile = \"galaxy\"\n", + ); + // All invalid -> fall back to defaults. + assert_eq!(cfg.defaults.detail_level, None); + assert_eq!(cfg.defaults.limit, None); + assert_eq!(cfg.defaults.profile, OutputProfile::Default); + } + + #[test] + fn unknown_sections_and_keys_ignored() { + let cfg = VestigeConfig::parse( + "[future_phase]\nfoo = 1\n[defaults]\nprofile = audit\nbar = baz\n", + ); + assert_eq!(cfg.defaults.profile, OutputProfile::Audit); + } + + #[test] + fn profile_presets() { + // lean: brief + dropped scores/timestamps + small limit + let lean = VestigeConfig::parse("[defaults]\nprofile=lean").output(); + assert_eq!(lean.detail_level, "brief"); + assert_eq!(lean.limit, Some(5)); + assert!(!lean.show_scores); + assert!(!lean.show_timestamps); + + // audit: full detail, no forced limit + let audit = VestigeConfig::parse("[defaults]\nprofile=audit").output(); + assert_eq!(audit.detail_level, "full"); + assert_eq!(audit.limit, None); + + // research: full detail, larger limit + let research = VestigeConfig::parse("[defaults]\nprofile=research").output(); + assert_eq!(research.detail_level, "full"); + assert_eq!(research.limit, Some(25)); + } + + #[test] + fn explicit_defaults_override_profile_presets() { + // profile=lean would give brief/limit 5, but explicit keys win. + let out = VestigeConfig::parse( + "[defaults]\nprofile=lean\ndetail_level=\"full\"\nlimit=42\n", + ) + .output(); + assert_eq!(out.detail_level, "full"); + assert_eq!(out.limit, Some(42)); + } + + #[test] + fn precedence_mcp_param_wins() { + let out = VestigeConfig::parse("[defaults]\nprofile=lean").output(); + // Config says brief, but an explicit MCP param wins. + assert_eq!(out.resolve_detail_level(Some("full")), "full"); + // No explicit param -> config (lean -> brief). + assert_eq!(out.resolve_detail_level(None), "brief"); + } + + #[test] + fn precedence_limit_layers() { + let out = VestigeConfig::parse("[defaults]\nprofile=research").output(); + // explicit param wins over everything + assert_eq!(out.resolve_limit(Some(3), 10), 3); + // no param -> config (research -> 25) + assert_eq!(out.resolve_limit(None, 10), 25); + // default profile has no limit -> builtin fallback used + let def = OutputConfig::default(); + assert_eq!(def.resolve_limit(None, 10), 10); + } +} diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index 4c50413..b0afc0b 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -80,6 +80,8 @@ // MODULES // ============================================================================ +/// Optional `vestige.toml` configuration (Phase 2: Configurable Output). +pub mod config; pub mod consolidation; pub mod fsrs; pub mod fts; @@ -152,6 +154,9 @@ pub use fsrs::{ retrievability_with_decay, }; +// Configuration (vestige.toml output profiles / defaults) +pub use config::{OutputConfig, OutputDefaults, OutputProfile, VestigeConfig, CONFIG_FILE}; + // Storage layer pub use storage::{ ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, InsightRecord, diff --git a/crates/vestige-mcp/Cargo.toml b/crates/vestige-mcp/Cargo.toml index f265ec5..bc08a40 100644 --- a/crates/vestige-mcp/Cargo.toml +++ b/crates/vestige-mcp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vestige-mcp" -version = "2.1.25" +version = "2.1.26" edition = "2024" description = "Cognitive memory MCP server for AI agents - FSRS-6, spreading activation, synaptic tagging, 3D dashboard, and 130 years of memory research" authors = ["samvallad33"] @@ -51,7 +51,7 @@ path = "src/bin/cli.rs" # Only `bundled-sqlite` is always on. `embeddings` and `vector-search` are # toggled via vestige-mcp's own feature flags below so `--no-default-features` # actually works (previously hardcoded here, which silently defeated the flag). -vestige-core = { version = "2.1.25", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] } +vestige-core = { version = "2.1.26", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] } # ============================================================================ # MCP Server Dependencies diff --git a/crates/vestige-mcp/src/server.rs b/crates/vestige-mcp/src/server.rs index 890739b..2cb1e5f 100644 --- a/crates/vestige-mcp/src/server.rs +++ b/crates/vestige-mcp/src/server.rs @@ -20,7 +20,7 @@ use crate::protocol::messages::{ use crate::protocol::types::{JsonRpcError, JsonRpcRequest, JsonRpcResponse, MCP_VERSION}; use crate::resources; use crate::tools; -use vestige_core::Storage; +use vestige_core::{OutputConfig, Storage, VestigeConfig}; /// Build the MCP `instructions` string injected into every connecting client's /// system prompt. @@ -77,17 +77,31 @@ pub struct McpServer { tool_call_count: AtomicU64, /// Optional event broadcast channel for dashboard real-time updates. event_tx: Option>, + /// Resolved output config from `/vestige.toml` (Phase 2). Tools + /// use it as the fallback for detail/limit when no explicit MCP param is + /// given; explicit params always win. + output_config: Arc, +} + +/// Load `vestige.toml` from the storage's data directory and resolve it to an +/// effective [`OutputConfig`]. A missing/malformed file yields the built-in +/// default, which preserves historical behavior. +fn load_output_config(storage: &Arc) -> Arc { + let config = VestigeConfig::load_from_data_dir(storage.data_dir()); + Arc::new(config.output()) } impl McpServer { #[allow(dead_code)] pub fn new(storage: Arc, cognitive: Arc>) -> Self { + let output_config = load_output_config(&storage); Self { storage, cognitive, initialized: false, tool_call_count: AtomicU64::new(0), event_tx: None, + output_config, } } @@ -97,12 +111,14 @@ impl McpServer { cognitive: Arc>, event_tx: broadcast::Sender, ) -> Self { + let output_config = load_output_config(&storage); Self { storage, cognitive, initialized: false, tool_call_count: AtomicU64::new(0), event_tx: Some(event_tx), + output_config, } } @@ -537,16 +553,26 @@ impl McpServer { // UNIFIED TOOLS (v1.1+) - Preferred API // ================================================================ "search" => { - tools::search_unified::execute(&self.storage, &self.cognitive, request.arguments) - .await + tools::search_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) + .await } "memory" => { tools::memory_unified::execute(&self.storage, &self.cognitive, request.arguments) .await } "codebase" => { - tools::codebase_unified::execute(&self.storage, &self.cognitive, request.arguments) - .await + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) + .await } "intention" => { tools::intention_unified::execute(&self.storage, &self.cognitive, request.arguments) @@ -661,8 +687,13 @@ impl McpServer { "Tool '{}' is deprecated. Use 'search' instead.", request.name ); - tools::search_unified::execute(&self.storage, &self.cognitive, request.arguments) - .await + tools::search_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) + .await } // ================================================================ @@ -742,7 +773,13 @@ impl McpServer { } None => Some(serde_json::json!({"action": "remember_pattern"})), }; - tools::codebase_unified::execute(&self.storage, &self.cognitive, unified_args).await + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + unified_args, + ) + .await } "remember_decision" => { warn!( @@ -761,7 +798,13 @@ impl McpServer { } None => Some(serde_json::json!({"action": "remember_decision"})), }; - tools::codebase_unified::execute(&self.storage, &self.cognitive, unified_args).await + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + unified_args, + ) + .await } "get_codebase_context" => { warn!( @@ -777,7 +820,13 @@ impl McpServer { } None => Some(serde_json::json!({"action": "get_context"})), }; - tools::codebase_unified::execute(&self.storage, &self.cognitive, unified_args).await + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + unified_args, + ) + .await } // ================================================================ @@ -909,7 +958,9 @@ impl McpServer { // ================================================================ // TEMPORAL TOOLS (v1.2+) // ================================================================ - "memory_timeline" => tools::timeline::execute(&self.storage, request.arguments).await, + "memory_timeline" => { + tools::timeline::execute(&self.storage, &self.output_config, request.arguments).await + } "memory_changelog" => tools::changelog::execute(&self.storage, request.arguments).await, // ================================================================ @@ -967,8 +1018,13 @@ impl McpServer { // CONTEXT PACKETS (v1.8+) // ================================================================ "session_context" => { - tools::session_context::execute(&self.storage, &self.cognitive, request.arguments) - .await + tools::session_context::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) + .await } // ================================================================ diff --git a/crates/vestige-mcp/src/tools/codebase_unified.rs b/crates/vestige-mcp/src/tools/codebase_unified.rs index 726ab4b..70d8007 100644 --- a/crates/vestige-mcp/src/tools/codebase_unified.rs +++ b/crates/vestige-mcp/src/tools/codebase_unified.rs @@ -9,7 +9,9 @@ use std::sync::Arc; use tokio::sync::Mutex; use crate::cognitive::CognitiveEngine; -use vestige_core::{IngestInput, Storage}; +use vestige_core::{IngestInput, OutputConfig, Storage}; + +use super::search_unified::apply_output_masks; /// Input schema for the unified codebase tool pub fn schema() -> Value { @@ -87,6 +89,7 @@ struct CodebaseArgs { pub async fn execute( storage: &Arc, cognitive: &Arc>, + output_config: &OutputConfig, args: Option, ) -> Result { let args: CodebaseArgs = match args { @@ -97,7 +100,7 @@ pub async fn execute( match args.action.as_str() { "remember_pattern" => execute_remember_pattern(storage, cognitive, &args).await, "remember_decision" => execute_remember_decision(storage, cognitive, &args).await, - "get_context" => execute_get_context(storage, cognitive, &args).await, + "get_context" => execute_get_context(storage, cognitive, output_config, &args).await, _ => Err(format!( "Invalid action '{}'. Must be one of: remember_pattern, remember_decision, get_context", args.action @@ -282,9 +285,11 @@ async fn execute_remember_decision( async fn execute_get_context( storage: &Arc, cognitive: &Arc>, + output_config: &OutputConfig, args: &CodebaseArgs, ) -> Result { - let limit = args.limit.unwrap_or(10).clamp(1, 50); + // Precedence: explicit MCP param > config limit > built-in default (10). + let limit = output_config.resolve_limit(args.limit, 10).clamp(1, 50); // Build tag filter for codebase let tag_filter = args.codebase.as_ref().map(|cb| format!("codebase:{}", cb)); @@ -299,7 +304,7 @@ async fn execute_get_context( .get_nodes_by_type_and_tag("decision", tag_filter.as_deref(), limit) .unwrap_or_default(); - let formatted_patterns: Vec = patterns + let mut formatted_patterns: Vec = patterns .iter() .map(|n| { serde_json::json!({ @@ -311,8 +316,9 @@ async fn execute_get_context( }) }) .collect(); + apply_output_masks(&mut formatted_patterns, output_config); - let formatted_decisions: Vec = decisions + let mut formatted_decisions: Vec = decisions .iter() .map(|n| { serde_json::json!({ @@ -324,6 +330,7 @@ async fn execute_get_context( }) }) .collect(); + apply_output_masks(&mut formatted_decisions, output_config); // ==================================================================== // COGNITIVE: Cross-project knowledge discovery @@ -352,6 +359,7 @@ async fn execute_get_context( Ok(serde_json::json!({ "action": "get_context", "codebase": args.codebase, + "profile": output_config.profile.as_str(), "patterns": { "count": formatted_patterns.len(), "items": formatted_patterns, @@ -411,7 +419,7 @@ mod tests { #[tokio::test] async fn test_missing_args_fails() { let (storage, _dir) = test_storage().await; - let result = execute(&storage, &test_cognitive(), None).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), None).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Missing arguments")); } @@ -420,7 +428,7 @@ mod tests { async fn test_invalid_action_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "action": "invalid" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid action")); } @@ -435,7 +443,7 @@ mod tests { "files": ["src/lib.rs"], "codebase": "vestige" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "remember_pattern"); @@ -451,7 +459,7 @@ mod tests { "action": "remember_pattern", "description": "Some description" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'name' is required")); } @@ -463,7 +471,7 @@ mod tests { "action": "remember_pattern", "name": "Test Pattern" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'description' is required")); } @@ -476,7 +484,7 @@ mod tests { "name": " ", "description": "Some description" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -492,7 +500,7 @@ mod tests { "files": ["src/storage.rs"], "codebase": "vestige" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "remember_decision"); @@ -507,7 +515,7 @@ mod tests { "action": "remember_decision", "rationale": "Some rationale" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'decision' is required")); } @@ -519,7 +527,7 @@ mod tests { "action": "remember_decision", "decision": "Use SQLite" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'rationale' is required")); } @@ -532,7 +540,7 @@ mod tests { "decision": " ", "rationale": "Something" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -544,7 +552,7 @@ mod tests { "action": "get_context", "codebase": "nonexistent" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "get_context"); @@ -563,14 +571,14 @@ mod tests { "description": "A test pattern", "codebase": "myproject" }); - execute(&storage, &cog, Some(save_args)).await.unwrap(); + execute(&storage, &cog, &OutputConfig::default(), Some(save_args)).await.unwrap(); // Now retrieve let get_args = serde_json::json!({ "action": "get_context", "codebase": "myproject" }); - let result = execute(&storage, &cog, Some(get_args)).await; + let result = execute(&storage, &cog, &OutputConfig::default(), Some(get_args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert!(value["patterns"]["count"].as_u64().unwrap() >= 1); @@ -580,10 +588,35 @@ mod tests { async fn test_get_context_no_codebase() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "action": "get_context" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "get_context"); assert!(value["codebase"].is_null()); } + + /// Phase 2: the `lean` profile masks the `createdAt` timestamp from + /// get_context items, and the response echoes the active profile. + #[tokio::test] + async fn test_get_context_lean_profile_masks_timestamps() { + let (storage, _dir) = test_storage().await; + let cog = test_cognitive(); + let save_args = serde_json::json!({ + "action": "remember_pattern", + "name": "Lean Pattern", + "description": "A pattern for lean masking", + "codebase": "leanproj" + }); + execute(&storage, &cog, &OutputConfig::default(), Some(save_args)) + .await + .unwrap(); + + let cfg = vestige_core::VestigeConfig::parse("[defaults]\nprofile=lean").output(); + let get_args = serde_json::json!({ "action": "get_context", "codebase": "leanproj" }); + let value = execute(&storage, &cog, &cfg, Some(get_args)).await.unwrap(); + assert_eq!(value["profile"], "lean"); + let item = &value["patterns"]["items"][0]; + assert!(item.get("createdAt").is_none(), "lean must drop createdAt"); + assert!(item.get("content").is_some(), "content still present"); + } } diff --git a/crates/vestige-mcp/src/tools/search_unified.rs b/crates/vestige-mcp/src/tools/search_unified.rs index 3aea11f..0a3c583 100644 --- a/crates/vestige-mcp/src/tools/search_unified.rs +++ b/crates/vestige-mcp/src/tools/search_unified.rs @@ -21,8 +21,8 @@ use tokio::sync::Mutex; use crate::cognitive::CognitiveEngine; use vestige_core::{ - CompetitionCandidate, EncodingContext, MemoryLifecycle, MemorySnapshot, MemoryState, Storage, - TopicalContext, + CompetitionCandidate, EncodingContext, MemoryLifecycle, MemorySnapshot, MemoryState, + OutputConfig, Storage, TopicalContext, }; /// Input schema for unified search tool @@ -143,6 +143,7 @@ struct SearchArgs { pub async fn execute( storage: &Arc, cognitive: &Arc>, + output_config: &OutputConfig, args: Option, ) -> Result { let args: SearchArgs = match args { @@ -154,12 +155,16 @@ pub async fn execute( return Err("Query cannot be empty".to_string()); } - // Validate detail_level - let detail_level = match args.detail_level.as_deref() { - Some("brief") => "brief", - Some("full") => "full", - Some("summary") | None => "summary", - Some(invalid) => { + // Validate detail_level. Precedence: explicit MCP param > config file > + // built-in default. The explicit arg is validated; the config fallback is + // already validated at load time. + let detail_level_owned = + output_config.resolve_detail_level(args.detail_level.as_deref()); + let detail_level = match detail_level_owned.as_str() { + "brief" => "brief", + "full" => "full", + "summary" => "summary", + invalid => { return Err(format!( "Invalid detail_level '{}'. Must be 'brief', 'summary', or 'full'.", invalid @@ -167,8 +172,9 @@ pub async fn execute( } }; - // Clamp all parameters to valid ranges - let limit = args.limit.unwrap_or(10).clamp(1, 100); + // Clamp all parameters to valid ranges. The default limit honors the + // config file (e.g. a `research` profile) when no explicit param is set. + let limit = output_config.resolve_limit(args.limit, 10).clamp(1, 100); let min_retention = args.min_retention.unwrap_or(0.0).clamp(0.0, 1.0); let min_similarity = args.min_similarity.unwrap_or(0.5).clamp(0.0, 1.0); @@ -230,6 +236,7 @@ pub async fn execute( .filter(|r| r.node.retention_strength >= min_retention) .map(|r| format_search_result(r, detail_level)) .collect(); + apply_output_masks(&mut formatted, output_config); let mut budget_expandable: Vec = Vec::new(); let mut budget_tokens_used: Option = None; @@ -261,6 +268,7 @@ pub async fn execute( "retrievalMode": retrieval_mode, "concrete": true, "detailLevel": detail_level, + "profile": output_config.profile.as_str(), "total": formatted.len(), "results": formatted, }); @@ -715,6 +723,7 @@ pub async fn execute( .iter() .map(|r| format_search_result(r, detail_level)) .collect(); + apply_output_masks(&mut formatted, output_config); // ==================================================================== // Token budget enforcement (v1.8.0) @@ -755,6 +764,7 @@ pub async fn execute( "method": "hybrid+cognitive", "retrievalMode": retrieval_mode, "detailLevel": detail_level, + "profile": output_config.profile.as_str(), "total": formatted.len(), "results": formatted, }); @@ -844,6 +854,42 @@ fn tags_match_prefix(tags: &[String], prefix: &str) -> bool { } /// Format a search result based on the requested detail level. +/// Score field keys dropped when an output profile suppresses scores. +const SCORE_FIELDS: &[&str] = &["combinedScore", "keywordScore", "semanticScore"]; +/// Timestamp field keys dropped when an output profile suppresses timestamps. +const TIMESTAMP_FIELDS: &[&str] = &[ + "createdAt", + "updatedAt", + "lastAccessed", + "nextReview", + "validFrom", + "validUntil", +]; + +/// Strip score/timestamp fields from already-formatted result objects according +/// to the active output profile (e.g. the `lean` profile drops both). Tools +/// call this after formatting so the field-mask behavior is centralized and the +/// per-detail-level formatters stay unchanged. +pub fn apply_output_masks(results: &mut [Value], output_config: &OutputConfig) { + if output_config.show_scores && output_config.show_timestamps { + return; + } + for result in results.iter_mut() { + if let Some(obj) = result.as_object_mut() { + if !output_config.show_scores { + for key in SCORE_FIELDS { + obj.remove(*key); + } + } + if !output_config.show_timestamps { + for key in TIMESTAMP_FIELDS { + obj.remove(*key); + } + } + } + } +} + fn format_search_result(r: &vestige_core::SearchResult, detail_level: &str) -> Value { match detail_level { "brief" => serde_json::json!({ @@ -984,7 +1030,7 @@ mod tests { async fn test_search_empty_query_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "query": "" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -993,7 +1039,7 @@ mod tests { async fn test_search_whitespace_only_query_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "query": " \t\n " }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -1001,7 +1047,7 @@ mod tests { #[tokio::test] async fn test_search_missing_arguments_fails() { let (storage, _dir) = test_storage().await; - let result = execute(&storage, &test_cognitive(), None).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), None).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Missing arguments")); } @@ -1010,7 +1056,7 @@ mod tests { async fn test_search_missing_query_field_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "limit": 10 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid arguments")); } @@ -1048,7 +1094,7 @@ mod tests { "query": "OPENAI_API_KEY", "limit": 5 }); - let result = execute(&storage, &test_cognitive(), Some(args)) + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) .await .unwrap(); @@ -1072,7 +1118,7 @@ mod tests { "query": uuid, "limit": 5 }); - let result = execute(&storage, &test_cognitive(), Some(args)) + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) .await .unwrap(); @@ -1098,7 +1144,7 @@ mod tests { "query": "mlx_lm.server", "limit": 5 }); - let result = execute(&storage, &test_cognitive(), Some(args)) + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) .await .unwrap(); @@ -1120,7 +1166,7 @@ mod tests { "query": "test", "limit": 0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); } @@ -1134,7 +1180,7 @@ mod tests { "query": "test", "limit": 1000 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); } @@ -1147,7 +1193,7 @@ mod tests { "query": "test", "limit": -5 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); } @@ -1164,7 +1210,7 @@ mod tests { "query": "test", "min_retention": -0.5 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); } @@ -1177,7 +1223,7 @@ mod tests { "query": "test", "min_retention": 1.5 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; // Should succeed but may return no results (retention > 1.0 clamped to 1.0) assert!(result.is_ok()); } @@ -1195,7 +1241,7 @@ mod tests { "query": "test", "min_similarity": -0.5 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); } @@ -1208,7 +1254,7 @@ mod tests { "query": "test", "min_similarity": 1.5 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; // Should succeed but may return no results assert!(result.is_ok()); } @@ -1223,7 +1269,7 @@ mod tests { ingest_test_content(&storage, "The Rust programming language is memory safe.").await; let args = serde_json::json!({ "query": "rust" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1243,7 +1289,7 @@ mod tests { "query": "python", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1265,7 +1311,7 @@ mod tests { "limit": 2, "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1279,7 +1325,7 @@ mod tests { // Don't ingest anything - database is empty let args = serde_json::json!({ "query": "anything" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1296,7 +1342,7 @@ mod tests { "query": "testing", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1329,7 +1375,7 @@ mod tests { "query": "item", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1415,7 +1461,7 @@ mod tests { "detail_level": "brief", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1444,7 +1490,7 @@ mod tests { "detail_level": "full", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1471,7 +1517,7 @@ mod tests { "query": "default", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1500,7 +1546,7 @@ mod tests { "query": "test", "detail_level": "invalid_level" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid detail_level")); } @@ -1529,7 +1575,7 @@ mod tests { "token_budget": 200, "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1556,7 +1602,7 @@ mod tests { "token_budget": 150, "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1575,7 +1621,7 @@ mod tests { "query": "no budget", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1679,7 +1725,7 @@ mod tests { "tag_prefix": "meeting:", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok(), "{:?}", result); let value = result.unwrap(); let results = value["results"].as_array().unwrap(); @@ -1695,7 +1741,7 @@ mod tests { // depends on the cognitive pipeline's competition/suppression // dynamics, so assert a lower bound. assert!( - results.len() >= 1, + !results.is_empty(), "tag_prefix should leave at least one meeting:* result, got {}", results.len() ); @@ -1722,7 +1768,7 @@ mod tests { "tag_prefix": "project:", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); let results = value["results"].as_array().unwrap(); @@ -1753,13 +1799,13 @@ mod tests { "query": "audit", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); let results = value["results"].as_array().unwrap(); // Both should be retrievable since no tag_prefix is set. assert!( - results.len() >= 1, + !results.is_empty(), "expected at least one result with no tag_prefix" ); } @@ -1786,7 +1832,7 @@ mod tests { "concrete": true, "tag_prefix": "meeting:" }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok(), "{:?}", result); let value = result.unwrap(); assert_eq!(value["method"], "concrete"); @@ -1799,4 +1845,78 @@ mod tests { assert!(has_meeting, "concrete result lacks meeting:* tag: {}", r); } } + + // ======================================================================== + // Phase 2: Configurable Output — precedence tests + // ======================================================================== + + /// Config-file detail_level applies when no explicit MCP param is given. + #[tokio::test] + async fn test_config_detail_level_applies_without_param() { + let (storage, _dir) = test_storage().await; + ingest_test_content(&storage, "Config detail level fallback content.").await; + + // Config selects `full`; the call passes no detail_level. + let cfg = vestige_core::VestigeConfig::parse("[defaults]\ndetail_level=\"full\"").output(); + let args = serde_json::json!({ "query": "config detail", "min_similarity": 0.0 }); + let value = execute(&storage, &test_cognitive(), &cfg, Some(args)) + .await + .unwrap(); + assert_eq!(value["detailLevel"], "full"); + } + + /// Explicit MCP param beats the config file (precedence layer 1 > 2). + #[tokio::test] + async fn test_explicit_param_overrides_config() { + let (storage, _dir) = test_storage().await; + ingest_test_content(&storage, "Explicit overrides config content.").await; + + // Config says `full`, but the call explicitly requests `brief`. + let cfg = vestige_core::VestigeConfig::parse("[defaults]\ndetail_level=\"full\"").output(); + let args = serde_json::json!({ + "query": "explicit override", + "detail_level": "brief", + "min_similarity": 0.0 + }); + let value = execute(&storage, &test_cognitive(), &cfg, Some(args)) + .await + .unwrap(); + assert_eq!(value["detailLevel"], "brief"); + } + + /// The `lean` profile masks scores and timestamps from results. + #[tokio::test] + async fn test_lean_profile_masks_scores_and_timestamps() { + let (storage, _dir) = test_storage().await; + ingest_test_content(&storage, "Lean profile masking content.").await; + + let cfg = vestige_core::VestigeConfig::parse("[defaults]\nprofile=lean").output(); + let args = serde_json::json!({ "query": "lean masking", "min_similarity": 0.0 }); + let value = execute(&storage, &test_cognitive(), &cfg, Some(args)) + .await + .unwrap(); + assert_eq!(value["profile"], "lean"); + if let Some(first) = value["results"].as_array().and_then(|a| a.first()) { + assert!(first.get("combinedScore").is_none(), "lean must drop scores"); + assert!(first.get("createdAt").is_none(), "lean must drop timestamps"); + } + } + + /// The default profile is byte-for-byte the historical behavior: summary + /// detail with scores and timestamps present. + #[tokio::test] + async fn test_default_profile_preserves_behavior() { + let (storage, _dir) = test_storage().await; + ingest_test_content(&storage, "Default profile preserved content.").await; + + let args = serde_json::json!({ "query": "default preserved", "min_similarity": 0.0 }); + let value = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) + .await + .unwrap(); + assert_eq!(value["detailLevel"], "summary"); + assert_eq!(value["profile"], "default"); + if let Some(first) = value["results"].as_array().and_then(|a| a.first()) { + assert!(first.get("createdAt").is_some(), "default keeps timestamps"); + } + } } diff --git a/crates/vestige-mcp/src/tools/session_context.rs b/crates/vestige-mcp/src/tools/session_context.rs index 0a32ce4..68e5c6b 100644 --- a/crates/vestige-mcp/src/tools/session_context.rs +++ b/crates/vestige-mcp/src/tools/session_context.rs @@ -13,7 +13,7 @@ use serde::Deserialize; use serde_json::Value; use crate::cognitive::CognitiveEngine; -use vestige_core::Storage; +use vestige_core::{OutputConfig, Storage}; /// Input schema for session_context tool pub fn schema() -> Value { @@ -98,6 +98,7 @@ fn first_sentence(content: &str) -> String { pub async fn execute( storage: &Arc, cognitive: &Arc>, + output_config: &OutputConfig, args: Option, ) -> Result { let args: SessionContextArgs = match args { @@ -105,6 +106,14 @@ pub async fn execute( None => SessionContextArgs::default(), }; + // Per-query search width honors the active profile (e.g. `research` widens + // it, `lean` narrows it). No explicit MCP param exists here, so the config + // limit (or built-in default of 5) applies. Capped to keep the budgeted + // session response compact. + let per_query_limit = output_config.resolve_limit(None, 5).clamp(1, 25); + // The `lean` profile suppresses the inline memory date to save tokens. + let show_dates = output_config.show_timestamps; + let token_budget = args.token_budget.unwrap_or(1000).clamp(100, 100000) as usize; let budget_chars = token_budget * 4; let include_status = args.include_status.unwrap_or(true); @@ -126,7 +135,7 @@ pub async fn execute( for query in &queries { let results = storage - .hybrid_search(query, 5, 0.3, 0.7) + .hybrid_search(query, per_query_limit, 0.3, 0.7) .map_err(|e| e.to_string())?; for r in results { @@ -134,8 +143,12 @@ pub async fn execute( continue; } let summary = first_sentence(&r.node.content); - let date_str = r.node.updated_at.format("%b %d, %Y").to_string(); - let line = format!("- ({}) {}", date_str, summary); + let line = if show_dates { + let date_str = r.node.updated_at.format("%b %d, %Y").to_string(); + format!("- ({}) {}", date_str, summary) + } else { + format!("- {}", summary) + }; let line_len = line.len() + 1; // +1 for newline if char_count + line_len > budget_chars { @@ -384,6 +397,7 @@ pub async fn execute( Ok(serde_json::json!({ "context": context_text, + "profile": output_config.profile.as_str(), "tokensUsed": tokens_used, "tokenBudget": token_budget, "expandable": expandable_ids, @@ -529,7 +543,7 @@ mod tests { #[tokio::test] async fn test_default_no_args() { let (storage, _dir) = test_storage().await; - let result = execute(&storage, &test_cognitive(), None).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), None).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -554,7 +568,7 @@ mod tests { let args = serde_json::json!({ "queries": ["user preferences", "project context"] }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -582,7 +596,7 @@ mod tests { "queries": ["memory"], "token_budget": 200 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -618,7 +632,7 @@ mod tests { "queries": ["expandable test memory"], "token_budget": 150 }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -629,7 +643,7 @@ mod tests { #[tokio::test] async fn test_automation_triggers_booleans() { let (storage, _dir) = test_storage().await; - let result = execute(&storage, &test_cognitive(), None).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), None).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -649,7 +663,7 @@ mod tests { "include_intentions": false, "include_predictions": false }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -683,7 +697,7 @@ mod tests { "topics": ["performance"] } }); - let result = execute(&storage, &test_cognitive(), Some(args)).await; + let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); @@ -692,6 +706,37 @@ mod tests { assert!(ctx.contains("vestige")); } + /// Phase 2: the response echoes the active profile, and the `lean` profile + /// suppresses inline memory dates to save tokens. + #[tokio::test] + async fn test_session_context_profile_echo_and_lean_dates() { + let (storage, _dir) = test_storage().await; + ingest_test_content(&storage, "Session profile content sentence.", vec![]).await; + + // Default profile -> profile echoed, dates present. + let args = serde_json::json!({ "queries": ["profile content"] }); + let value = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) + .await + .unwrap(); + assert_eq!(value["profile"], "default"); + + // Lean profile -> profile echoed as lean. The memory line must not carry + // the "(Mon DD, YYYY)" inline date prefix. + let cfg = vestige_core::VestigeConfig::parse("[defaults]\nprofile=lean").output(); + let args = serde_json::json!({ "queries": ["profile content"] }); + let value = execute(&storage, &test_cognitive(), &cfg, Some(args)) + .await + .unwrap(); + assert_eq!(value["profile"], "lean"); + let ctx = value["context"].as_str().unwrap(); + if ctx.contains("**Memories:**") { + assert!( + !ctx.contains(", 20"), + "lean profile should omit the inline year in memory dates" + ); + } + } + // ======================================================================== // HELPER TESTS // ======================================================================== diff --git a/crates/vestige-mcp/src/tools/timeline.rs b/crates/vestige-mcp/src/tools/timeline.rs index 14e58bf..5c73357 100644 --- a/crates/vestige-mcp/src/tools/timeline.rs +++ b/crates/vestige-mcp/src/tools/timeline.rs @@ -9,9 +9,9 @@ use serde_json::Value; use std::collections::BTreeMap; use std::sync::Arc; -use vestige_core::Storage; +use vestige_core::{OutputConfig, Storage}; -use super::search_unified::format_node; +use super::search_unified::{apply_output_masks, format_node}; /// Input schema for memory_timeline tool pub fn schema() -> Value { @@ -87,7 +87,11 @@ fn parse_datetime(s: &str) -> Result, String> { } /// Execute memory_timeline tool -pub async fn execute(storage: &Arc, args: Option) -> Result { +pub async fn execute( + storage: &Arc, + output_config: &OutputConfig, + args: Option, +) -> Result { let args: TimelineArgs = match args { Some(v) => serde_json::from_value(v).map_err(|e| format!("Invalid arguments: {}", e))?, None => TimelineArgs { @@ -100,12 +104,13 @@ pub async fn execute(storage: &Arc, args: Option) -> Result "brief", - Some("full") => "full", - Some("summary") | None => "summary", - Some(invalid) => { + // Validate detail_level. Precedence: explicit MCP param > config > default. + let detail_level_owned = output_config.resolve_detail_level(args.detail_level.as_deref()); + let detail_level = match detail_level_owned.as_str() { + "brief" => "brief", + "full" => "full", + "summary" => "summary", + invalid => { return Err(format!( "Invalid detail_level '{}'. Must be 'brief', 'summary', or 'full'.", invalid @@ -124,7 +129,8 @@ pub async fn execute(storage: &Arc, args: Option) -> Result Some(now), }; - let limit = args.limit.unwrap_or(50).clamp(1, 200); + // Precedence: explicit MCP param > config limit > built-in default (50). + let limit = output_config.resolve_limit(args.limit, 50).clamp(1, 200); // Query memories in time range with filters pushed into SQL. Rust-side // `retain` after `LIMIT` was unsafe for sparse types/tags — a dominant @@ -140,14 +146,15 @@ pub async fn execute(storage: &Arc, args: Option) -> Result> = BTreeMap::new(); for node in &results { let date = node.created_at.date_naive(); - by_day - .entry(date) - .or_default() - .push(format_node(node, detail_level)); + let mut formatted = [format_node(node, detail_level)]; + apply_output_masks(&mut formatted, output_config); + let [formatted] = formatted; + by_day.entry(date).or_default().push(formatted); } // Build timeline (newest first) @@ -173,6 +180,7 @@ pub async fn execute(storage: &Arc, args: Option) -> Result= 1); @@ -296,7 +304,7 @@ mod tests { async fn test_timeline_invalid_detail_level() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "detail_level": "invalid" }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid detail_level")); } @@ -306,7 +314,7 @@ mod tests { let (storage, _dir) = test_storage().await; ingest_test_memory(&storage, "Brief test memory").await; let args = serde_json::json!({ "detail_level": "brief" }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["detailLevel"], "brief"); @@ -317,7 +325,7 @@ mod tests { let (storage, _dir) = test_storage().await; ingest_test_memory(&storage, "Full test memory").await; let args = serde_json::json!({ "detail_level": "full" }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["detailLevel"], "full"); @@ -327,7 +335,7 @@ mod tests { async fn test_timeline_limit_clamped() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "limit": 0 }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); // limit clamped to 1, no error } @@ -339,7 +347,7 @@ mod tests { "start": "2020-01-01", "end": "2030-12-31" }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; assert!(result.is_ok()); let value = result.unwrap(); assert!(value["totalMemories"].as_u64().unwrap() >= 1); @@ -350,7 +358,7 @@ mod tests { let (storage, _dir) = test_storage().await; ingest_test_memory(&storage, "A fact memory").await; let args = serde_json::json!({ "node_type": "concept" }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; let value = result.unwrap(); // Ingested as "fact", filtering for "concept" should yield 0 assert_eq!(value["totalMemories"], 0); @@ -361,7 +369,7 @@ mod tests { let (storage, _dir) = test_storage().await; ingest_test_memory(&storage, "Tagged memory").await; let args = serde_json::json!({ "tags": ["timeline-test"] }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; let value = result.unwrap(); assert!(value["totalMemories"].as_u64().unwrap() >= 1); } @@ -371,7 +379,7 @@ mod tests { let (storage, _dir) = test_storage().await; ingest_test_memory(&storage, "Tagged memory").await; let args = serde_json::json!({ "tags": ["nonexistent-tag"] }); - let result = execute(&storage, Some(args)).await; + let result = execute(&storage, &OutputConfig::default(), Some(args)).await; let value = result.unwrap(); assert_eq!(value["totalMemories"], 0); } @@ -409,7 +417,7 @@ mod tests { // Limit 5 against 12 total — before the fix, `retain` on `concept` // would operate on the 5 most recent rows (all `fact`) and find 0. let args = serde_json::json!({ "node_type": "concept", "limit": 5 }); - let value = execute(&storage, Some(args)).await.unwrap(); + let value = execute(&storage, &OutputConfig::default(), Some(args)).await.unwrap(); assert_eq!( value["totalMemories"], 2, "Both sparse concepts should survive a limit smaller than the dominant set" @@ -447,7 +455,7 @@ mod tests { } let args = serde_json::json!({ "tags": ["rare"], "limit": 5 }); - let value = execute(&storage, Some(args)).await.unwrap(); + let value = execute(&storage, &OutputConfig::default(), Some(args)).await.unwrap(); assert_eq!( value["totalMemories"], 2, "Both sparse-tag matches should survive a limit smaller than the dominant set" @@ -479,4 +487,23 @@ mod tests { assert_eq!(nodes.len(), 1, "Only the exact-tag match should return"); assert_eq!(nodes[0].content, "Exact tag hit"); } + + /// Phase 2: config-file detail_level applies when no explicit param is set, + /// and an explicit param overrides it. + #[tokio::test] + async fn test_timeline_config_detail_precedence() { + let (storage, _dir) = test_storage().await; + ingest_test_memory(&storage, "Timeline config precedence content.").await; + + let cfg = vestige_core::VestigeConfig::parse("[defaults]\ndetail_level=\"full\"").output(); + + // No explicit param -> config wins. + let value = execute(&storage, &cfg, None).await.unwrap(); + assert_eq!(value["detailLevel"], "full"); + + // Explicit param -> overrides config. + let args = serde_json::json!({ "detail_level": "brief" }); + let value = execute(&storage, &cfg, Some(args)).await.unwrap(); + assert_eq!(value["detailLevel"], "brief"); + } } diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 64b9c06..e9bcc0e 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -50,6 +50,93 @@ Qwen3 currently uses Hugging Face Hub's Candle loader directly, so use the stand --- +## Output Configuration (`vestige.toml`) + +> Added in **v2.1.26** (Roadmap Phase 2: Configurable Output). + +You can control the default shape and size of high-traffic MCP responses with an +optional config file. It is **local-first** — no cloud service is involved — and +**fully backward-compatible**: with no file present, Vestige behaves exactly as +it did before. + +### Location + +The config file lives in the active Vestige data directory, alongside the +database: + +``` +/vestige.toml # e.g. ~/Library/Application Support/com.vestige.core/vestige.toml +``` + +The data directory is resolved with the same precedence as storage +(`--data-dir` > `VESTIGE_DATA_DIR` > OS per-user data dir). A missing file, or a +file with no recognized keys, falls back to built-in defaults. The parser is +lenient: unknown keys and unknown sections are ignored, so the file can grow in +future releases without breaking older binaries. + +### `[defaults]` table + +```toml +[defaults] +# Detail level for high-traffic tools: "brief" | "summary" | "full" +detail_level = "summary" + +# Default result count for high-traffic tools (positive integer) +limit = 10 + +# Output profile: "lean" | "default" | "audit" | "research" +profile = "default" +``` + +All three keys are optional. `detail_level` and `limit`, when set, override the +selected profile's presets. + +### Output profiles + +A profile presets a coherent bundle of detail level, default limit, and whether +scores and timestamps are included: + +| Profile | Detail | Default limit | Scores | Timestamps | Use when | +|---------|--------|---------------|--------|------------|----------| +| `lean` | `brief` | 5 | dropped | dropped | Context budget matters most | +| `default` | `summary` | tool default | shown | shown | **Historical behavior (unchanged)** | +| `audit` | `full` | tool default | shown | shown | Reviewing or debugging memory state | +| `research` | `full` | 25 | shown | shown | Wide, detailed result sets | + +### Precedence + +Resolved per call, highest to lowest: + +1. **Explicit MCP parameter** (e.g. `detail_level` / `limit` on a `search` + call) — always wins. +2. **`vestige.toml`** — the `[defaults]` keys and the selected profile. +3. **Built-in default** — the `default` profile, identical to pre-v2.1.26 + behavior. + +### Affected tools + +`search`, `memory_timeline`, `codebase` (`get_context`), and `session_context` +resolve their default detail level and result limit through this config. Each of +these tools also echoes the active `profile` in its response so you can confirm +what was applied. Tools that take no `detail_level`/`limit` are unaffected. + +### Example: minimize context cost + +```toml +[defaults] +profile = "lean" +``` + +### Example: detailed audits without changing the profile + +```toml +[defaults] +detail_level = "full" +limit = 50 +``` + +--- + ## Command-Line Options ```bash diff --git a/package.json b/package.json index 20bb78a..05c69e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "vestige", - "version": "2.1.25", + "version": "2.1.26", "private": true, "description": "Cognitive memory for AI - MCP server with FSRS-6 spaced repetition", "author": "Sam Valladares", diff --git a/packages/vestige-init/package.json b/packages/vestige-init/package.json index cb9b4f7..7347fe9 100644 --- a/packages/vestige-init/package.json +++ b/packages/vestige-init/package.json @@ -1,6 +1,6 @@ { "name": "@vestige/init", - "version": "2.1.25", + "version": "2.1.26", "description": "Configure Vestige local memory for MCP-compatible AI agents", "bin": { "vestige-init": "bin/init.js" diff --git a/packages/vestige-mcp-npm/package.json b/packages/vestige-mcp-npm/package.json index a7e7829..7d44863 100644 --- a/packages/vestige-mcp-npm/package.json +++ b/packages/vestige-mcp-npm/package.json @@ -1,6 +1,6 @@ { "name": "vestige-mcp-server", - "version": "2.1.25", + "version": "2.1.26", "mcpName": "io.github.samvallad33/vestige", "description": "Vestige MCP Server — local cognitive memory for MCP-compatible AI agents", "bin": { diff --git a/server.json b/server.json index 2b9a927..300eed9 100644 --- a/server.json +++ b/server.json @@ -7,12 +7,12 @@ "url": "https://github.com/samvallad33/vestige", "source": "github" }, - "version": "2.1.25", + "version": "2.1.26", "packages": [ { "registryType": "npm", "identifier": "vestige-mcp-server", - "version": "2.1.25", + "version": "2.1.26", "transport": { "type": "stdio" } From 6c7d56b4cf658217519d3f10c93e604dd5d3deb2 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Mon, 15 Jun 2026 17:06:01 -0500 Subject: [PATCH 21/38] Add OpenCode integration and safer startup --- README.md | 8 +- crates/vestige-core/Cargo.toml | 6 +- crates/vestige-mcp/src/main.rs | 51 ++-- docs/AGENT-MEMORY-PROTOCOL.md | 2 +- docs/CONFIGURATION.md | 30 ++- docs/FAQ.md | 24 +- docs/ROADMAP.md | 142 +++++++++++ docs/VESTIGE_STATE_AND_PLAN.md | 4 +- docs/integrations/codex.md | 1 + docs/integrations/cursor.md | 1 + docs/integrations/jetbrains.md | 1 + docs/integrations/opencode.md | 233 ++++++++++++++++++ docs/integrations/vscode.md | 1 + docs/integrations/windsurf.md | 1 + docs/integrations/xcode.md | 1 + docs/launch/opencode-adoption.md | 123 +++++++++ packages/vestige-init/bin/init.js | 54 +++- packages/vestige-init/package.json | 1 + packages/vestige-mcp-npm/README.md | 34 +++ packages/vestige-mcp-npm/package.json | 1 + .../vestige-mcp-npm/scripts/postinstall.js | 1 + 21 files changed, 676 insertions(+), 44 deletions(-) create mode 100644 docs/ROADMAP.md create mode 100644 docs/integrations/opencode.md create mode 100644 docs/launch/opencode-adoption.md diff --git a/README.md b/README.md index f747715..cec3daf 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Built on proven memory and retrieval ideas — FSRS-6 spaced repetition, prediction error gating, synaptic tagging, spreading activation, and memory consolidation — all running in a single Rust binary with a local dashboard. 100% local. Zero cloud. -[Quick Start](#quick-start) | [Dashboard](#-3d-memory-dashboard) | [How It Works](#-the-cognitive-science-stack) | [Tools](#-25-mcp-tools) | [Docs](docs/) +[Quick Start](#quick-start) | [Dashboard](#-3d-memory-dashboard) | [How It Works](#-the-cognitive-science-stack) | [Tools](#-25-mcp-tools) | [Docs](docs/) | [Roadmap](docs/ROADMAP.md) @@ -48,6 +48,9 @@ claude mcp add vestige vestige-mcp -s user # Codex codex mcp add vestige -- vestige-mcp +# OpenCode +npx @vestige/init + # 3. Test it # "Remember that I prefer TypeScript over JavaScript" # ...new session... @@ -142,6 +145,7 @@ Vestige speaks MCP, so any client that can register a stdio MCP server can use i | **Xcode 26.3** | [Integration guide](docs/integrations/xcode.md) | | **Cursor** | [Integration guide](docs/integrations/cursor.md) | | **VS Code (Copilot)** | [Integration guide](docs/integrations/vscode.md) | +| **OpenCode** | [Integration guide](docs/integrations/opencode.md) | | **JetBrains** | [Integration guide](docs/integrations/jetbrains.md) | | **Windsurf** | [Integration guide](docs/integrations/windsurf.md) | @@ -421,7 +425,7 @@ vestige dashboard # Open 3D dashboard in browser | [Storage Modes](docs/STORAGE.md) | Global, per-project, multi-instance | | [CLAUDE.md Setup](docs/CLAUDE-SETUP.md) | Templates for proactive memory | | [Configuration](docs/CONFIGURATION.md) | CLI commands, environment variables | -| [Integrations](docs/integrations/) | Codex, Xcode, Cursor, VS Code, JetBrains, Windsurf | +| [Integrations](docs/integrations/) | Codex, Xcode, Cursor, VS Code, OpenCode, JetBrains, Windsurf | | [Changelog](CHANGELOG.md) | Version history | --- diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index e878cdb..0a9c748 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -121,7 +121,11 @@ candle-core = { version = "0.10.2", optional = true } # its memory_mapping_allocator_gt template references the POSIX MAP_FAILED # macro from , which doesn't exist on MSVC. Tracked upstream in # unum-cloud/usearch#746. Unpin when the upstream fix lands. -usearch = { version = "=2.23.0", optional = true } +# +# Disable default features so release binaries do not include SimSIMD's +# Haswell+/AVX2/FMA dispatch targets. Those kernels can trigger illegal +# instructions on older x86_64 CPUs that Vestige otherwise supports. +usearch = { version = "=2.23.0", default-features = false, optional = true } # LRU cache for query embeddings lru = "0.16" diff --git a/crates/vestige-mcp/src/main.rs b/crates/vestige-mcp/src/main.rs index 916c441..062e750 100644 --- a/crates/vestige-mcp/src/main.rs +++ b/crates/vestige-mcp/src/main.rs @@ -301,21 +301,6 @@ async fn main() { let storage = match Storage::new(storage_path) { Ok(s) => { info!("Storage initialized successfully"); - - // Try to initialize embeddings early and log any issues - #[cfg(feature = "embeddings")] - { - if let Err(e) = s.init_embeddings() { - error!("Failed to initialize embedding service: {}", e); - error!("Smart ingest will fall back to regular ingest without deduplication"); - error!( - "Hint: Check FASTEMBED_CACHE_PATH or ensure ~/.cache/vestige/fastembed is writable" - ); - } else { - info!("Embedding service initialized successfully"); - } - } - Arc::new(s) } Err(e) => { @@ -324,6 +309,40 @@ async fn main() { } }; + // Initialize embeddings in the background so MCP clients can complete the + // stdio handshake quickly. First-run model downloads can otherwise exceed + // short client startup timeouts. + #[cfg(feature = "embeddings")] + { + let storage_clone = Arc::clone(&storage); + tokio::task::spawn_blocking(move || { + if let Err(e) = storage_clone.init_embeddings() { + error!("Failed to initialize embedding service: {}", e); + error!("Smart ingest will fall back to regular ingest without deduplication"); + error!( + "Hint: Check FASTEMBED_CACHE_PATH or ensure ~/.cache/vestige/fastembed is writable" + ); + } else { + info!("Embedding service initialized successfully"); + + #[cfg(feature = "vector-search")] + match storage_clone.generate_embeddings(None, false) { + Ok(result) => { + if result.successful > 0 || result.failed > 0 { + info!( + embeddings_generated = result.successful, + embeddings_failed = result.failed, + embeddings_skipped = result.skipped, + "Background embedding backfill complete" + ); + } + } + Err(e) => warn!("Background embedding backfill failed: {}", e), + } + } + }); + } + // Spawn periodic auto-consolidation so FSRS-6 decay scores stay fresh. // Runs on startup (if needed) and then every N hours (default: 6). // Configurable via VESTIGE_CONSOLIDATION_INTERVAL_HOURS env var. @@ -506,7 +525,7 @@ async fn main() { } // Load cross-encoder reranker in the background (downloads ~150MB on first run) - #[cfg(feature = "vector-search")] + #[cfg(all(feature = "vector-search", feature = "embeddings"))] { let cog_clone = Arc::clone(&cognitive); tokio::spawn(async move { diff --git a/docs/AGENT-MEMORY-PROTOCOL.md b/docs/AGENT-MEMORY-PROTOCOL.md index 367ca4b..6096982 100644 --- a/docs/AGENT-MEMORY-PROTOCOL.md +++ b/docs/AGENT-MEMORY-PROTOCOL.md @@ -76,6 +76,6 @@ call `memory` with `action="purge"` and `confirm=true`. ## Portability Notes The same protocol applies to Claude Code, Codex, Cursor, VS Code, Xcode, -JetBrains, Windsurf, and any other client that can run a stdio MCP server. Claude +OpenCode, JetBrains, Windsurf, and any other client that can run a stdio MCP server. Claude Code's Cognitive Sandwich hooks are optional companion files; they are not required for normal Vestige memory. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index e9bcc0e..450cb71 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -228,20 +228,42 @@ Add to `%APPDATA%\Claude\claude_desktop_config.json`: } ``` -### Opencode TUI/Desktop +### OpenCode -You can put it at [various different](https://opencode.ai/docs/config/#locations) locations. I recommend `opencode.json` in the project folder. +OpenCode supports global and project-local config. For a project-local setup, add to `opencode.json`: ```json { - "mcpServers": { + "$schema": "https://opencode.ai/config.json", + "mcp": { "vestige": { - "command": "vestige-mcp" + "type": "local", + "command": ["vestige-mcp"], + "enabled": true, + "timeout": 10000 } } } ``` +For isolated per-project memory, pass the data directory in the command array: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["vestige-mcp", "--data-dir", "./.vestige"], + "enabled": true, + "timeout": 10000 + } + } +} +``` + +See the [OpenCode integration guide](integrations/opencode.md) for global config, verification, and troubleshooting. + --- ## Custom Data Directory diff --git a/docs/FAQ.md b/docs/FAQ.md index 93005b8..5f93f25 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -782,22 +782,16 @@ This helps trace why you know something.
"What's planned for future versions?" -Based on codebase exploration, these features exist in various stages: +See the public [Vestige Roadmap](ROADMAP.md) for the current adoption plan. The +near-term focus is reducing first-user confusion before expanding the feature +surface: -| Feature | Status | Description | -|---------|--------|-------------| -| Memory Dreams | Partial | Automated offline consolidation | -| Reconsolidation | Planned | Update memories when accessed | -| Memory Chains | Partial | Link related memories explicitly | -| Adaptive Embedding | Planned | Re-embed old memories with better models | -| Cross-Project Learning | Planned | Share patterns across codebases | - -**Community wishlist** (from Reddit): -- Stream ingestion mode -- GUI for memory browsing -- Export/import formats -- Sync between devices (encrypted) -- Team collaboration features +- first-time memory migration and atomic memory guidance +- configurable MCP output fields and output profiles +- clearer merge/supersede controls +- code/docstring memory workflows +- goals and milestones distinct from intentions +- guided import dry runs and review queues Contributions welcome!
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..6578cef --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,142 @@ +# Vestige Roadmap + +> Public adoption roadmap for making Vestige easier to start, easier to trust, +> and easier to configure. + +Last updated: June 7, 2026 + +Vestige already has the core primitives for durable local memory: `search`, +`session_context`, `smart_ingest`, `memory`, `intention`, `codebase`, +`deep_reference`, suppression, portable storage, and the dashboard. The next +product step is reducing first-user confusion so more people can get value from +those primitives without inventing their own fragile memory vocabulary. + +This roadmap turns early community feedback into a staged plan. + +## Principles + +- Make first use obvious. A new user should know what to import, how atomic each + memory should be, and which tool to use for current session context. +- Keep memory legible. Agents and humans should understand whether a memory was + created, reinforced, updated, superseded, suppressed, or purged. +- Prefer progressive disclosure. The default MCP response should be lean, with + explicit ways to request more detail. +- Keep local-first behavior. New onboarding, code memory, and configuration + features must not require a cloud service. +- Optimize for many users. Defaults should work for non-experts, while power + users can tune fields, merge behavior, and formats. + +## Already Shipped, Needs Clearer Guidance + +| Area | Current State | Next Documentation Fix | +|------|---------------|------------------------| +| Session startup | `session_context` combines memories, intentions, status, predictions, and codebase context. | Update all agent setup templates to make `session_context` the default startup call. | +| Batch memory saves | `smart_ingest` batch mode defaults to `batchMergePolicy="force_create"` so caller-separated items stay separate. | Document when to use batch force-create vs smart merge. | +| Device migration | `portable-export`, `portable-import`, and `sync` preserve exact Vestige storage state. | Separate device migration from first-time document import so users do not confuse them. | +| Supersede semantics | Supersede demotes the old memory and creates a new one; it does not purge the old memory. | Add plain-language vocabulary for create, update, supersede, suppress, demote, and purge. | + +## Phase 1: Onboarding And Memory Hygiene + +Target: make the first 30 minutes with Vestige hard to mess up. + +| Work | Outcome | +|------|---------| +| First-time memory migration guide | Users can import notes/docs without Claude tagging everything as `verified` or flattening unrelated facts together. | +| Atomic memory guide | Clear examples for one fact, one preference, one decision, one bug fix, one source note, and one code pattern per memory. | +| Default tag vocabulary | Recommended tags for source quality, confidence, project, type, urgency, and lifecycle without overloading words like `verified`. | +| Smart vs force-create guide | Agents know when to use `forceCreate`, `batchMergePolicy="force_create"`, or normal PE gating. | +| Updated agent templates | Claude, Codex, Cursor, VS Code, Xcode, OpenCode, JetBrains, and Windsurf templates start with `session_context` and use the same memory vocabulary. | + +Planned docs: + +- `docs/MIGRATION.md` +- `docs/MEMORY-HYGIENE.md` +- revised `docs/AGENT-MEMORY-PROTOCOL.md` +- revised `docs/CLAUDE-SETUP.md` + +## Phase 2: Configurable Output + +Target: let users control context cost without losing important evidence. + +| Work | Outcome | +|------|---------| +| Field masks for MCP results | Users can drop fields they never want in model context, such as temporal hints, scores, or timestamps. | +| Output profiles | Presets like `lean`, `default`, `audit`, and `research` tune result size and metadata detail. | +| Markdown output mode | Users can request compact Markdown summaries when that is more context-efficient than JSON. | +| Context reinstatement controls | `contextReinstatement` becomes opt-in or configurable, and temporal hints are based on stored memory context when available. | +| Per-tool defaults | Users can define default detail level, result limit, and response shape for search, timeline, codebase, and session context. | + +Likely implementation paths: + +- config file under the active Vestige data directory +- environment-variable override for simple deployments +- MCP parameters still win over defaults for one-off calls + +## Phase 3: Merge And Supersede Controls + +Target: make memory mutation predictable. + +| Work | Outcome | +|------|---------| +| Merge policy configuration | Users can keep some tags or node types atomic while allowing others to merge. | +| Prediction Error threshold knobs | Advanced users can tune create/update/reinforce boundaries without recompiling. | +| Merge previews before mutation | Agents can show what would change before updating an existing durable memory. | +| Safer consolidation dedup | Consolidation respects user-configured atomic tags and source boundaries. | +| Friendlier lifecycle labels | Agent-facing copy explains that superseded memories are old versions, not destroyed records. | + +## Phase 4: Code Memory + +Target: make code memories useful without blending source code, docstrings, and +human project notes into one noisy search space. + +| Work | Outcome | +|------|---------| +| Code memory import guide | Developers know when to save patterns/decisions versus code entities or docstrings. | +| Exposed code entity workflow | The existing core `CodeEntity` concept becomes usable through MCP or CLI. | +| Docstring/code symbol ingestion | Users can ingest functions, types, modules, docstrings, and call-site notes with source file provenance. | +| Code/prose retrieval separation | Search can filter or rank code memories separately from user preferences and project decisions. | +| Codebase dashboard review | Developers can inspect imported code memories and remove noisy entries. | + +## Phase 5: Goals And Milestones + +Target: support durable direction without pretending every future task is just a +reminder. + +| Work | Outcome | +|------|---------| +| Goal primitive | Non-fading, manually pivoted goals that survive normal memory decay. | +| Milestone tracking | Goals can have milestones, status, evidence, and blockers. | +| Goal-aware session context | `session_context` can include active goals when relevant. | +| Manual pivot semantics | Agents can update goals only when the user explicitly pivots, completes, or cancels them. | +| Dashboard surface | Users can inspect active, completed, paused, and cancelled goals. | + +This is distinct from `intention`: intentions are reminders triggered by time, +topic, file, event, or context. Goals are longer-lived direction and should not +fire as reminders unless the user attaches an intention. + +## Phase 6: Guided Import Tools + +Target: turn "I have 300 notes" into a reliable workflow. + +| Work | Outcome | +|------|---------| +| Import dry run | Vestige previews proposed memories, tags, node types, and merge decisions before writing. | +| Source-aware import | Imported memories keep file/source provenance and confidence metadata. | +| Chunking strategies | Users choose atomic facts, section summaries, decision records, or source notes. | +| Review queue | Users can approve, edit, split, merge, or reject proposed memories. | +| Post-import health pass | Vestige recommends consolidation, duplicate review, or tag cleanup after import. | + +## Non-Goals + +- Do not auto-store every conversation turn by default. +- Do not require cloud services for memory creation, search, or configuration. +- Do not hide irreversible deletion. `purge` must stay explicit. +- Do not make code ingestion pollute general personal memory by default. +- Do not make advanced tuning required for ordinary users. + +## How To Read This Roadmap + +This is directional, not a release guarantee. The priority is adoption: fewer +surprises, clearer defaults, and better tool descriptions before adding complex +new surfaces. Community feedback that reveals a confusing first-use path should +usually become either a documentation fix, a safer default, or a guided workflow. diff --git a/docs/VESTIGE_STATE_AND_PLAN.md b/docs/VESTIGE_STATE_AND_PLAN.md index 3e3a001..5f22469 100644 --- a/docs/VESTIGE_STATE_AND_PLAN.md +++ b/docs/VESTIGE_STATE_AND_PLAN.md @@ -73,8 +73,8 @@ Vestige is organized as: HTTP MCP is disabled unless the user passes `--http`, passes `--http-port`, or sets `VESTIGE_HTTP_ENABLED=1`. The stdio MCP server remains the portable default -for Claude Code, Codex, Cursor, VS Code, Xcode, JetBrains, Windsurf, and other -clients. +for Claude Code, Codex, Cursor, VS Code, Xcode, OpenCode, JetBrains, Windsurf, +and other clients. Purge is implemented transactionally in storage and surfaced through the MCP `memory` tool. `memory(action="purge", confirm=true)` is the explicit hard diff --git a/docs/integrations/codex.md b/docs/integrations/codex.md index 7d9b17b..9413175 100644 --- a/docs/integrations/codex.md +++ b/docs/integrations/codex.md @@ -145,6 +145,7 @@ codex mcp add vestige -- /usr/local/bin/vestige-mcp | Xcode 26.3 | [Setup](./xcode.md) | | Cursor | [Setup](./cursor.md) | | VS Code (Copilot) | [Setup](./vscode.md) | +| OpenCode | [Setup](./opencode.md) | | JetBrains | [Setup](./jetbrains.md) | | Windsurf | [Setup](./windsurf.md) | | Claude Code | [Setup](../CONFIGURATION.md#claude-code-one-liner) | diff --git a/docs/integrations/cursor.md b/docs/integrations/cursor.md index 1e22943..4b7467b 100644 --- a/docs/integrations/cursor.md +++ b/docs/integrations/cursor.md @@ -135,6 +135,7 @@ Cursor does not surface MCP server errors in the UI. Test by running the command | Xcode 26.3 | [Setup](./xcode.md) | | Codex | [Setup](./codex.md) | | VS Code (Copilot) | [Setup](./vscode.md) | +| OpenCode | [Setup](./opencode.md) | | JetBrains | [Setup](./jetbrains.md) | | Windsurf | [Setup](./windsurf.md) | | Claude Code | [Setup](../CONFIGURATION.md#claude-code-one-liner) | diff --git a/docs/integrations/jetbrains.md b/docs/integrations/jetbrains.md index 3424539..1582a34 100644 --- a/docs/integrations/jetbrains.md +++ b/docs/integrations/jetbrains.md @@ -123,6 +123,7 @@ In **Settings > Tools > MCP Server**, click the expansion arrow next to your cli | Cursor | [Setup](./cursor.md) | | VS Code (Copilot) | [Setup](./vscode.md) | | Codex | [Setup](./codex.md) | +| OpenCode | [Setup](./opencode.md) | | Windsurf | [Setup](./windsurf.md) | | Claude Code | [Setup](../CONFIGURATION.md#claude-code-one-liner) | | Claude Desktop | [Setup](../CONFIGURATION.md#claude-desktop-macos) | diff --git a/docs/integrations/opencode.md b/docs/integrations/opencode.md new file mode 100644 index 0000000..3c28e9f --- /dev/null +++ b/docs/integrations/opencode.md @@ -0,0 +1,233 @@ +# OpenCode + +> Give OpenCode persistent local memory across TUI, CLI, and desktop sessions. + +OpenCode supports local MCP servers through its `mcp` config. Add Vestige once and your OpenCode agents can remember project decisions, architecture context, preferences, and previous fixes between sessions. + +Verified with OpenCode `1.16.2` on June 8, 2026. + +--- + +## Why OpenCode Users Add Vestige + +OpenCode is strong at driving real coding work from the terminal. The painful gap is continuity: the next session often has to rediscover what the previous session already learned. Vestige gives OpenCode a local memory layer through MCP, so the agent can reuse the project context that should not be trapped in one chat transcript. + +Useful memories include: + +- project decisions: "we use Axum handlers thinly and keep database logic in storage modules" +- preferences: "prefer small focused PRs and explicit verification receipts" +- architecture context: "the dashboard talks to the MCP server through the Axum backend and WebSocket events" +- bug fixes: "OpenCode rejects `mcpServers`; use top-level `mcp.vestige` with a command array" +- workflow state: "PR #67 was merged, but the config shape needed correction before promotion" + +Vestige is local-first. Memories are stored in SQLite on your machine, can be scoped globally or per project, and are retrieved with tools like `vestige_session_context`, `vestige_search`, `vestige_smart_ingest`, and `vestige_deep_reference`. + +--- + +## Setup + +### 1. Install Vestige + +```bash +npm install -g vestige-mcp-server@latest +``` + +Verify the binary: + +```bash +vestige-mcp --version +``` + +If you prefer not to install globally, use `npx` directly in the OpenCode command array: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["npx", "-y", "-p", "vestige-mcp-server@latest", "vestige-mcp"], + "enabled": true, + "timeout": 60000 + } + } +} +``` + +The higher timeout is for the first cold `npx` run, which may need to download the npm package before OpenCode can connect. If you install `vestige-mcp-server` globally, `10000` is enough for normal startup. + +If `npx` times out against an older published Vestige build, install globally once and use `command: ["vestige-mcp"]`. The current integration keeps the MCP handshake fast by moving embedding startup work into the background. + +### 2. Add Vestige To OpenCode + +For global use across projects, create or edit: + +```bash +mkdir -p ~/.config/opencode +${EDITOR:-vi} ~/.config/opencode/opencode.json +``` + +Add: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["vestige-mcp"], + "enabled": true, + "timeout": 10000 + } + } +} +``` + +OpenCode also supports project-local config. Put the same block in `opencode.json` at the repo root when you want the setting checked in with a project. + +For a custom config file, set `OPENCODE_CONFIG=/path/to/opencode.json` before launching OpenCode. + +### 3. Verify + +Restart OpenCode, then validate the resolved config and MCP server list: + +```bash +opencode debug config +opencode mcp list +``` + +You should see `vestige` listed. In a session, ask: + +> "What MCP tools can you use?" + +Vestige tools should be available with the `vestige_` prefix, such as `vestige_search`, `vestige_smart_ingest`, `vestige_session_context`, and `vestige_deep_reference`. + +--- + +## First Use + +In OpenCode: + +> "Remember that this project uses Rust with Axum and SQLite." + +Start a new OpenCode session, then ask: + +> "What stack does this project use?" + +It remembers. + +--- + +## Project-Specific Memory + +To isolate memory per repo, add `--data-dir` to OpenCode's command array: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["vestige-mcp", "--data-dir", "./.vestige"], + "enabled": true, + "timeout": 10000 + } + } +} +``` + +For an absolute path: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["/usr/local/bin/vestige-mcp", "--data-dir", "/Users/you/projects/my-app/.vestige"], + "enabled": true, + "timeout": 10000 + } + } +} +``` + +--- + +## Automatic Setup + +If `opencode` is installed or `~/.config/opencode` exists, Vestige's installer can add the global config automatically: + +```bash +npx @vestige/init +``` + +The installer writes a backup before modifying an existing config file. It also migrates Vestige entries copied from older `mcpServers` examples into OpenCode's current `mcp.vestige` shape. + +--- + +## Troubleshooting + +
+Vestige tools do not appear + +1. Verify OpenCode can see configured MCP servers: + ```bash + opencode debug config + opencode mcp list + ``` +2. Verify the binary is on your path: + ```bash + which vestige-mcp + ``` +3. Use an absolute binary path if OpenCode cannot resolve `vestige-mcp`. +4. Restart OpenCode after changing `opencode.json`. +5. Keep `timeout` at `10000` or higher for installed binaries. If you use the direct `npx` command, use `60000` so the first cold npm download does not fail OpenCode startup. +
+ +
+Config does not validate + +OpenCode uses the top-level `mcp` key. Do not use the `mcpServers` shape from Claude Desktop, Cursor, or Windsurf. + +If you copied an older Vestige example that used `mcpServers`, rerun: + +```bash +npx @vestige/init +``` + +Correct: + +```json +{ + "mcp": { + "vestige": { + "type": "local", + "command": ["vestige-mcp"], + "timeout": 10000 + } + } +} +``` +
+ +
+Too many MCP tools in context + +OpenCode loads MCP tools alongside built-in tools. If you have many MCP servers enabled, disable unused servers or restrict MCP tools per agent in your OpenCode config. +
+ +--- + +## Also Works With + +| IDE | Guide | +|-----|-------| +| Codex | [Setup](./codex.md) | +| Cursor | [Setup](./cursor.md) | +| VS Code (Copilot) | [Setup](./vscode.md) | +| JetBrains | [Setup](./jetbrains.md) | +| Windsurf | [Setup](./windsurf.md) | +| Xcode 26.3 | [Setup](./xcode.md) | +| Claude Code | [Setup](../CONFIGURATION.md#claude-code-one-liner) | +| Claude Desktop | [Setup](../CONFIGURATION.md#claude-desktop-macos) | diff --git a/docs/integrations/vscode.md b/docs/integrations/vscode.md index 556e784..0211f87 100644 --- a/docs/integrations/vscode.md +++ b/docs/integrations/vscode.md @@ -153,6 +153,7 @@ Every team member with Vestige installed will automatically get memory-enabled C | Xcode 26.3 | [Setup](./xcode.md) | | Cursor | [Setup](./cursor.md) | | Codex | [Setup](./codex.md) | +| OpenCode | [Setup](./opencode.md) | | JetBrains | [Setup](./jetbrains.md) | | Windsurf | [Setup](./windsurf.md) | | Claude Code | [Setup](../CONFIGURATION.md#claude-code-one-liner) | diff --git a/docs/integrations/windsurf.md b/docs/integrations/windsurf.md index 3a0aca1..ec00dea 100644 --- a/docs/integrations/windsurf.md +++ b/docs/integrations/windsurf.md @@ -149,6 +149,7 @@ If you have many MCP servers and exceed 100 total tools, Cascade will ignore exc | Cursor | [Setup](./cursor.md) | | VS Code (Copilot) | [Setup](./vscode.md) | | Codex | [Setup](./codex.md) | +| OpenCode | [Setup](./opencode.md) | | JetBrains | [Setup](./jetbrains.md) | | Claude Code | [Setup](../CONFIGURATION.md#claude-code-one-liner) | | Claude Desktop | [Setup](../CONFIGURATION.md#claude-desktop-macos) | diff --git a/docs/integrations/xcode.md b/docs/integrations/xcode.md index 3856a5e..5164ec1 100644 --- a/docs/integrations/xcode.md +++ b/docs/integrations/xcode.md @@ -252,6 +252,7 @@ Vestige uses the MCP standard — the same memory works across all your tools: | Claude Desktop | [Setup](../CONFIGURATION.md#claude-desktop-macos) | | Cursor | [Setup](./cursor.md) | | VS Code (Copilot) | [Setup](./vscode.md) | +| OpenCode | [Setup](./opencode.md) | | JetBrains | [Setup](./jetbrains.md) | | Windsurf | [Setup](./windsurf.md) | diff --git a/docs/launch/opencode-adoption.md b/docs/launch/opencode-adoption.md new file mode 100644 index 0000000..bb46a44 --- /dev/null +++ b/docs/launch/opencode-adoption.md @@ -0,0 +1,123 @@ +# OpenCode Adoption Plan + +Status: Vestige was tested with OpenCode `1.16.2` on June 8, 2026. The working config uses OpenCode's top-level `mcp.vestige` schema, not `mcpServers`. + +Public promotion started: + +- Vestige PR #70: `https://github.com/samvallad33/vestige/pull/70` +- OpenCode issue: `https://github.com/anomalyco/opencode/issues/31402` +- OpenCode docs/ecosystem PR: `https://github.com/anomalyco/opencode/pull/31405` +- awesome-opencode PR: `https://github.com/awesome-opencode/awesome-opencode/pull/418` +- opencode.cafe listing request: `https://github.com/R44VC0RP/opencode.cafe/issues/6` +- OpenCode persistent memory comment: `https://github.com/anomalyco/opencode/issues/16077#issuecomment-4652064625` + +## Release Gate + +- PR #67 is merged upstream and should be treated as the contributor-driven starting point. +- Ship the corrected OpenCode config docs and `@vestige/init` migration from stale `mcpServers.vestige` to `mcp.vestige`. +- Ship the background embedding initialization fix before making direct `npx` the main OpenCode install path. A cold published `2.1.23` package can still time out while OpenCode waits for tools. +- After release, verify all three OpenCode paths again: + - installed binary: `command: ["vestige-mcp"]` + - project memory: `command: ["vestige-mcp", "--data-dir", "./.vestige"]` + - direct npm: `command: ["npx", "-y", "-p", "vestige-mcp-server@latest", "vestige-mcp"]` with `timeout: 60000` + +## Official OpenCode PR + +Target repo: `https://github.com/anomalyco/opencode` + +Files: + +- `packages/web/src/content/docs/mcp-servers.mdx` +- `packages/web/src/content/docs/ecosystem.mdx` + +MCP docs snippet: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["npx", "-y", "-p", "vestige-mcp-server@latest", "vestige-mcp"], + "enabled": true, + "timeout": 60000 + } + } +} +``` + +Ecosystem row: + +```md +| [Vestige](https://github.com/samvallad33/vestige) | Local MCP memory server for OpenCode that remembers project decisions, preferences, and previous fixes across sessions | +``` + +Positioning: local, inspectable MCP memory for OpenCode. Avoid claiming Vestige fixes OpenCode's process memory or session resume behavior. + +## Awesome OpenCode + +Target repo: `https://github.com/awesome-opencode/awesome-opencode` + +Suggested entry, with category to confirm against maintainer preference (`data/projects/vestige.yaml` or `data/resources/vestige.yaml`): + +```yaml +name: Vestige +repo: https://github.com/samvallad33/vestige +tagline: Local persistent memory for OpenCode +description: Local MCP server that lets OpenCode remember project decisions, preferences, architecture context, and previous fixes across sessions. +scope: + - global + - project +tags: + - mcp + - memory + - local-first + - sqlite + - opencode +min_version: 1.16.2 +homepage: https://github.com/samvallad33/vestige/blob/main/docs/integrations/opencode.md +installation: | + npm install -g vestige-mcp-server@latest + npx @vestige/init +``` + +## MCP Directories + +Current state: + +- Official MCP Registry already lists `io.github.samvallad33/vestige` at `https://registry.modelcontextprotocol.io/v0/servers?search=vestige`. +- Smithery already lists Vestige and indexes 25 tools: `https://smithery.ai/server/@samvallad33/vestige`. +- Glama already lists Vestige, but the listing needs a refresh/fix if it shows no tools: `https://glama.ai/mcp/servers/samvallad33/vestige`. +- `mcp.so` does not show Vestige under the expected slugs yet; submit manually at `https://mcp.so/submit`. + +Priority order: + +1. Official MCP Registry: `https://github.com/modelcontextprotocol/registry` +2. Awesome MCP Servers: `https://github.com/punkpeye/awesome-mcp-servers` +3. Glama MCP directory: `https://glama.ai/mcp/servers` +4. Smithery: `https://smithery.ai` +5. PulseMCP: `https://www.pulsemcp.com` + +Registry metadata is mostly ready: `server.json` exists and `packages/vestige-mcp-npm/package.json` has `mcpName: "io.github.samvallad33/vestige"`. Publish only when the package version and `server.json` version match the released npm package. + +## Community Launch + +Use tested technical copy, not hype: + +> Vestige now works with OpenCode as a local MCP memory server. It gives OpenCode persistent memory for project decisions, preferences, architecture context, and previous fixes across sessions. Install with `npm install -g vestige-mcp-server@latest`, run `npx @vestige/init`, then verify with `opencode mcp list`. + +High-signal channels after release: + +- OpenCode Discord: `https://opencode.ai/discord` +- opencode.cafe MCP Server listing: `https://opencode.cafe` +- OpenCode memory-related GitHub issues, only where directly relevant +- Hacker News and Lobsters with a technical post about the tested OpenCode integration and failure modes +- npm keyword/discovery after the next package release includes `opencode` + +## Proof Checklist + +- `opencode debug config` accepts `mcp.vestige`. +- `opencode mcp list` shows `vestige connected`. +- Stale `mcpServers.vestige` examples fail in OpenCode and are migrated by `@vestige/init`. +- OpenCode tools are prefixed as `vestige_search`, `vestige_smart_ingest`, `vestige_session_context`, and `vestige_deep_reference`. +- The OpenCode guide says `timeout: 60000` for direct `npx` and `timeout: 10000` for installed binaries. diff --git a/packages/vestige-init/bin/init.js b/packages/vestige-init/bin/init.js index d7c5da2..cb62f81 100755 --- a/packages/vestige-init/bin/init.js +++ b/packages/vestige-init/bin/init.js @@ -105,6 +105,21 @@ const IDE_CONFIGS = { note: 'Tip: For project-level config, create .vscode/mcp.json with {"servers": {"vestige": ...}}', }, + 'OpenCode': { + detect: () => { + try { + execSync(PLATFORM === 'win32' ? 'where opencode' : 'which opencode', { stdio: 'ignore' }); + return true; + } catch { + return fs.existsSync(path.join(HOME, '.config', 'opencode')); + } + }, + configPath: () => path.join(HOME, '.config', 'opencode', 'opencode.json'), + format: 'opencode', + key: 'mcp', + note: 'Tip: For project-level memory, add the same mcp.vestige block to an opencode.json in your repo root.', + }, + 'Xcode 26.3': { detect: () => { if (PLATFORM !== 'darwin') return false; @@ -152,7 +167,10 @@ function findBinary() { // npm global install location (() => { try { - const npmPrefix = execSync('npm prefix -g', { encoding: 'utf8' }).trim(); + const npmPrefix = execSync('npm prefix -g', { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); return path.join(npmPrefix, 'bin', 'vestige-mcp'); } catch { return null; } })(), @@ -164,7 +182,11 @@ function findBinary() { encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'], }).trim(); - if (result) candidates.unshift(result); + const firstMatch = result + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)[0]; + if (firstMatch) candidates.unshift(firstMatch); } catch {} for (const candidate of candidates) { @@ -272,6 +294,16 @@ function buildVestigeConfig(binaryPath) { }; } +function buildOpenCodeConfig(binaryPath) { + return { + type: 'local', + command: [binaryPath], + enabled: true, + timeout: 10000, + environment: {}, + }; +} + function buildXcodeConfig(binaryPath) { return { projects: { @@ -324,6 +356,22 @@ function injectConfig(ide, ideName, binaryPath) { return false; } config.mcp.servers.vestige = buildVestigeConfig(binaryPath); + } else if (ide.format === 'opencode') { + // OpenCode uses top-level "mcp" entries with command arrays. + if (!config.$schema) config.$schema = 'https://opencode.ai/config.json'; + if (!config.mcp) config.mcp = {}; + if (config.mcp.vestige) { + console.log(` [skip] ${ideName} — already configured`); + return false; + } + if (config.mcpServers && config.mcpServers.vestige) { + delete config.mcpServers.vestige; + if (Object.keys(config.mcpServers).length === 0) { + delete config.mcpServers; + } + console.log(` [migrate] ${ideName} — moved vestige from mcpServers to mcp`); + } + config.mcp.vestige = buildOpenCodeConfig(binaryPath); } else { // Standard mcpServers format (Cursor, Claude Desktop, JetBrains, Windsurf) const key = ide.key || 'mcpServers'; @@ -383,7 +431,7 @@ function main() { if (detected.length === 0) { console.log(' No supported IDEs found.'); console.log(''); - console.log('Supported: Claude Code, Claude Desktop, Cursor, VS Code, Xcode, JetBrains, Windsurf'); + console.log('Supported: Claude Code, Claude Desktop, Cursor, VS Code, OpenCode, Xcode, JetBrains, Windsurf'); process.exit(1); } diff --git a/packages/vestige-init/package.json b/packages/vestige-init/package.json index 7347fe9..430d886 100644 --- a/packages/vestige-init/package.json +++ b/packages/vestige-init/package.json @@ -13,6 +13,7 @@ "claude", "copilot", "cursor", + "opencode", "xcode", "jetbrains", "windsurf", diff --git a/packages/vestige-mcp-npm/README.md b/packages/vestige-mcp-npm/README.md index 98e6575..7bc9e2c 100644 --- a/packages/vestige-mcp-npm/README.md +++ b/packages/vestige-mcp-npm/README.md @@ -54,6 +54,40 @@ codex mcp add vestige -- vestige-mcp Then restart your MCP client. +**OpenCode** + +Add to `~/.config/opencode/opencode.json` or a project-local `opencode.json`: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["vestige-mcp"], + "enabled": true, + "timeout": 10000 + } + } +} +``` + +Prefer the installed `vestige-mcp` command for OpenCode. If you run Vestige directly through `npx`, use a longer first-run timeout because npm may need to download the package before OpenCode can connect: + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "vestige": { + "type": "local", + "command": ["npx", "-y", "-p", "vestige-mcp-server@latest", "vestige-mcp"], + "enabled": true, + "timeout": 60000 + } + } +} +``` + ## Usage with Claude Desktop Add to your Claude Desktop configuration: diff --git a/packages/vestige-mcp-npm/package.json b/packages/vestige-mcp-npm/package.json index 7d44863..90a0e28 100644 --- a/packages/vestige-mcp-npm/package.json +++ b/packages/vestige-mcp-npm/package.json @@ -14,6 +14,7 @@ "keywords": [ "mcp", "claude", + "opencode", "ai", "memory", "vestige", diff --git a/packages/vestige-mcp-npm/scripts/postinstall.js b/packages/vestige-mcp-npm/scripts/postinstall.js index 65fe54b..76e678b 100644 --- a/packages/vestige-mcp-npm/scripts/postinstall.js +++ b/packages/vestige-mcp-npm/scripts/postinstall.js @@ -258,6 +258,7 @@ async function main() { console.log(' 1. Add vestige-mcp to any MCP-compatible agent.'); console.log(' Claude Code: claude mcp add vestige vestige-mcp -s user'); console.log(' Codex: codex mcp add vestige -- vestige-mcp'); + console.log(' OpenCode: npx @vestige/init, or add mcp.vestige to ~/.config/opencode/opencode.json'); console.log(' 2. Restart your MCP client.'); console.log(' 3. Test with: "remember that my preferred editor is VS Code"'); console.log(''); From efbea2513310982751b1ca21122ed61e6e94f3d5 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 15:59:57 -0500 Subject: [PATCH 22/38] Add ComposedGraph composition ledger --- README.md | 3 +- crates/vestige-core/src/lib.rs | 18 +- crates/vestige-core/src/storage/migrations.rs | 98 +- crates/vestige-core/src/storage/mod.rs | 8 +- crates/vestige-core/src/storage/sqlite.rs | 1968 ++++++++++++++++- crates/vestige-mcp/README.md | 2 +- crates/vestige-mcp/src/server.rs | 19 +- .../vestige-mcp/src/tools/composed_graph.rs | 906 ++++++++ .../vestige-mcp/src/tools/cross_reference.rs | 252 ++- crates/vestige-mcp/src/tools/mod.rs | 1 + docs/COMPOSED_GRAPH.md | 159 ++ 11 files changed, 3375 insertions(+), 59 deletions(-) create mode 100644 crates/vestige-mcp/src/tools/composed_graph.rs create mode 100644 docs/COMPOSED_GRAPH.md diff --git a/README.md b/README.md index f747715..ec3ba29 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,7 @@ This isn't a key-value store with an embedding model bolted on. Vestige implemen --- -## 🛠 25 MCP Tools +## 🛠 MCP Tools ### Context Packets | Tool | What It Does | @@ -272,6 +272,7 @@ This isn't a key-value store with an embedding model bolted on. Vestige implemen |------|-------------| | `memory_health` | Retention dashboard — distribution, trends, recommendations | | `memory_graph` | Knowledge graph export — force-directed layout, up to 200 nodes | +| `composed_graph` | Composition ledger — recent composed memory sets, neighbors, outcome labels, bounty/research lanes, and never-composed frontier candidates | ### Scoring & Dedup | Tool | What It Does | diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index b0afc0b..b8b0154 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -155,13 +155,15 @@ pub use fsrs::{ }; // Configuration (vestige.toml output profiles / defaults) -pub use config::{OutputConfig, OutputDefaults, OutputProfile, VestigeConfig, CONFIG_FILE}; +pub use config::{CONFIG_FILE, OutputConfig, OutputDefaults, OutputProfile, VestigeConfig}; // Storage layer pub use storage::{ - ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, InsightRecord, - IntentionRecord, PORTABLE_ARCHIVE_FORMAT, PortableArchive, PortableImportMode, - PortableImportReport, Result, SmartIngestResult, StateTransitionRecord, Storage, StorageError, + CompositionEventRecord, CompositionMemberRecord, CompositionNeighborRecord, + CompositionOutcomeRecord, ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, + InsightRecord, IntentionRecord, NeverComposedCandidate, PORTABLE_ARCHIVE_FORMAT, + PortableArchive, PortableImportMode, PortableImportReport, Result, SmartIngestResult, + StateTransitionRecord, Storage, StorageError, }; // Consolidation (sleep-inspired memory processing) @@ -220,6 +222,9 @@ pub use advanced::{ LabileState, Language, MaintenanceType, + // Merge / Supersede controls (Phase 3) + MatchClass, + MatchSignals, // Memory chains MemoryChainBuilder, // Memory compression @@ -230,18 +235,15 @@ pub use advanced::{ MemoryPath, MemoryReplay, MemorySnapshot, - // Merge / Supersede controls (Phase 3) - MatchClass, - MatchSignals, MergeCandidate, MergeOperation, MergePlan, MergePolicy, MergeStrategy, Modification, - PlanKind, Pattern, PatternType, + PlanKind, PredictedMemory, PredictionContext, PredictionErrorConfig, diff --git a/crates/vestige-core/src/storage/migrations.rs b/crates/vestige-core/src/storage/migrations.rs index 3be941c..127bc84 100644 --- a/crates/vestige-core/src/storage/migrations.rs +++ b/crates/vestige-core/src/storage/migrations.rs @@ -74,6 +74,11 @@ pub const MIGRATIONS: &[Migration] = &[ description: "v2.1.25 Merge/Supersede: reversible operation log, merge plans, bitemporal lineage, protected pins", up: MIGRATION_V14_UP, }, + Migration { + version: 15, + description: "ComposedGraph: composition events, members, outcomes", + up: MIGRATION_V15_UP, + }, ]; /// A database migration @@ -813,6 +818,67 @@ CREATE INDEX IF NOT EXISTS idx_merge_operations_survivor ON merge_operations(sur UPDATE schema_version SET version = 14, applied_at = datetime('now'); "#; +/// V15: ComposedGraph persistence for memory composition outcomes. +/// +/// These tables record which memories were used together, which tool/query +/// produced the composition, and what happened afterward. `memory_id` values +/// are intentionally historical references instead of foreign keys to +/// `knowledge_nodes`: purging or superseding a memory must not erase the fact +/// that a bounty lane or reasoning path was previously composed. +const MIGRATION_V15_UP: &str = r#" +CREATE TABLE IF NOT EXISTS composition_events ( + id TEXT PRIMARY KEY, + created_at TEXT NOT NULL, + tool TEXT NOT NULL, + mode TEXT NOT NULL DEFAULT 'deep_reference', + query TEXT, + query_hash TEXT, + confidence REAL, + status TEXT, + output_preview TEXT, + metadata TEXT NOT NULL DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_composition_events_created_at ON composition_events(created_at); +CREATE INDEX IF NOT EXISTS idx_composition_events_tool ON composition_events(tool); +CREATE INDEX IF NOT EXISTS idx_composition_events_mode ON composition_events(mode); +CREATE INDEX IF NOT EXISTS idx_composition_events_query_hash ON composition_events(query_hash); + +CREATE TABLE IF NOT EXISTS composition_members ( + event_id TEXT NOT NULL, + memory_id TEXT NOT NULL, + role TEXT NOT NULL, -- primary | supporting | contradicting | superseded | related + rank INTEGER NOT NULL DEFAULT 0, + trust REAL, + score REAL, + preview TEXT, + metadata TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (event_id, memory_id, role), + FOREIGN KEY (event_id) REFERENCES composition_events(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_composition_members_memory ON composition_members(memory_id); +CREATE INDEX IF NOT EXISTS idx_composition_members_role ON composition_members(role); + +CREATE TABLE IF NOT EXISTS composition_outcomes ( + id TEXT PRIMARY KEY, + event_id TEXT NOT NULL, + outcome_type TEXT NOT NULL, + labeled_at TEXT NOT NULL, + label_source TEXT NOT NULL DEFAULT 'tool', + confidence_delta REAL, + notes TEXT, + metadata TEXT NOT NULL DEFAULT '{}', + FOREIGN KEY (event_id) REFERENCES composition_events(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_composition_outcomes_event ON composition_outcomes(event_id); +CREATE INDEX IF NOT EXISTS idx_composition_outcomes_type ON composition_outcomes(outcome_type); +CREATE INDEX IF NOT EXISTS idx_composition_outcomes_labeled_at ON composition_outcomes(labeled_at); + +UPDATE schema_version SET version = 15, applied_at = datetime('now'); +"#; + /// Get current schema version from database pub fn get_current_version(conn: &rusqlite::Connection) -> rusqlite::Result { conn.query_row( @@ -829,7 +895,9 @@ pub fn get_current_version(conn: &rusqlite::Connection) -> rusqlite::Result fn add_column_if_missing(conn: &rusqlite::Connection, sql: &str) -> rusqlite::Result<()> { match conn.execute(sql, []) { Ok(_) => Ok(()), - Err(rusqlite::Error::SqliteFailure(_, Some(msg))) if msg.contains("duplicate column name") => { + Err(rusqlite::Error::SqliteFailure(_, Some(msg))) + if msg.contains("duplicate column name") => + { Ok(()) } Err(e) => Err(e), @@ -890,17 +958,17 @@ mod tests { /// version after `apply_migrations` runs all migrations end-to-end, and /// neither of the dead tables V11 drops must exist afterwards. #[test] - fn test_apply_migrations_advances_to_v14_and_drops_dead_tables() { + fn test_apply_migrations_advances_to_v15_and_drops_dead_tables() { let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); // Pre-requisite: schema_version must be bootstrapped by V1. apply_migrations(&conn).expect("apply_migrations succeeds"); - // 1. schema_version advanced to V14 + // 1. schema_version advanced to V15 let version = get_current_version(&conn).expect("read schema_version"); assert_eq!( - version, 14, - "schema_version must be 14 after all migrations" + version, 15, + "schema_version must be 15 after all migrations" ); // 2. knowledge_edges is gone (V11 drops it) @@ -967,7 +1035,23 @@ mod tests { assert_eq!(rows, 1, "{table} table must be created by V14"); } - // 7. knowledge_nodes gains `protected` + `superseded_by` (V14) + // 7. ComposedGraph tables exist (V15) + for table in [ + "composition_events", + "composition_members", + "composition_outcomes", + ] { + let rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1", + [table], + |row| row.get(0), + ) + .expect("query sqlite_master"); + assert_eq!(rows, 1, "{table} table must be created by V15"); + } + + // 8. knowledge_nodes gains `protected` + `superseded_by` (V14) let node_cols: Vec = { let mut stmt = conn .prepare("PRAGMA table_info(knowledge_nodes)") @@ -1006,6 +1090,6 @@ mod tests { apply_migrations(&conn).expect("V11 replay must be idempotent"); let version = get_current_version(&conn).expect("read schema_version"); - assert_eq!(version, 14, "schema_version back at 14 after replay"); + assert_eq!(version, 15, "schema_version back at 15 after replay"); } } diff --git a/crates/vestige-core/src/storage/mod.rs b/crates/vestige-core/src/storage/mod.rs index 1660529..282228d 100644 --- a/crates/vestige-core/src/storage/mod.rs +++ b/crates/vestige-core/src/storage/mod.rs @@ -16,7 +16,9 @@ pub use portable::{ PortableTable, PortableValue, }; pub use sqlite::{ - ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, FilePortableSyncBackend, - InsightRecord, IntentionRecord, PortableSyncBackend, PortableSyncReport, Result, - SmartIngestResult, StateTransitionRecord, Storage, StorageError, + CompositionEventRecord, CompositionMemberRecord, CompositionNeighborRecord, + CompositionOutcomeRecord, ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, + FilePortableSyncBackend, InsightRecord, IntentionRecord, NeverComposedCandidate, + PortableSyncBackend, PortableSyncReport, Result, SmartIngestResult, StateTransitionRecord, + Storage, StorageError, }; diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 4cd32e8..a9840a1 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -260,6 +260,9 @@ const PORTABLE_TABLES: &[&str] = &[ "retention_snapshots", "sync_tombstones", "deletion_tombstones", + "composition_events", + "composition_members", + "composition_outcomes", ]; const PORTABLE_USER_DATA_TABLES: &[&str] = &[ @@ -278,6 +281,9 @@ const PORTABLE_USER_DATA_TABLES: &[&str] = &[ "retention_snapshots", "sync_tombstones", "deletion_tombstones", + "composition_events", + "composition_members", + "composition_outcomes", ]; #[derive(Default)] @@ -1950,10 +1956,7 @@ impl Storage { // future migrations that switch. chrono::NaiveDateTime::parse_from_str(&s, "%Y-%m-%d %H:%M:%S") .map(|naive| naive.and_utc()) - .or_else(|_| { - DateTime::parse_from_rfc3339(&s) - .map(|dt| dt.with_timezone(&Utc)) - }) + .or_else(|_| DateTime::parse_from_rfc3339(&s).map(|dt| dt.with_timezone(&Utc))) .ok() }); @@ -2106,6 +2109,11 @@ impl Storage { params![id], )? as i64; + tx.execute( + "UPDATE composition_members SET preview = NULL WHERE memory_id = ?1", + params![id], + )?; + let tags_json = serde_json::to_string(&node.tags).unwrap_or_else(|_| "[]".to_string()); tx.execute( "INSERT INTO deletion_tombstones ( @@ -4035,7 +4043,966 @@ pub struct DreamHistoryRecord { pub creative_connections_found: Option, } +/// Composition event envelope for ComposedGraph. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompositionEventRecord { + pub id: String, + pub created_at: DateTime, + pub tool: String, + pub mode: String, + pub query: Option, + pub query_hash: Option, + pub confidence: Option, + pub status: Option, + pub output_preview: Option, + pub metadata: serde_json::Value, +} + +/// Memory participating in a composition event. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompositionMemberRecord { + pub event_id: String, + pub memory_id: String, + pub role: String, + pub rank: i32, + pub trust: Option, + pub score: Option, + pub preview: Option, + pub metadata: serde_json::Value, +} + +/// Outcome label attached to a composition event. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompositionOutcomeRecord { + pub id: String, + pub event_id: String, + pub outcome_type: String, + pub labeled_at: DateTime, + pub label_source: String, + pub confidence_delta: Option, + pub notes: Option, + pub metadata: serde_json::Value, +} + +/// Memory most often composed with another memory. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompositionNeighborRecord { + pub memory_id: String, + pub composed_count: i64, + pub latest_event_at: DateTime, +} + +/// Candidate memory pair that shares useful shape but has never been composed. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NeverComposedCandidate { + pub first_id: String, + pub second_id: String, + pub score: f64, + pub novelty_score: f64, + pub bridge_score: f64, + pub trust_score: f64, + pub outcome_score_adjustment: f64, + pub shared_tags: Vec, + pub boundary_tags: Vec, + pub shared_terms: Vec, + pub prior_outcomes: Vec, + pub outcome_signal: String, + pub first_node_type: String, + pub second_node_type: String, + pub first_preview: String, + pub second_preview: String, + pub reason: String, + pub composition_question: String, +} + impl Storage { + // ======================================================================== + // COMPOSEDGRAPH PERSISTENCE + // ======================================================================== + + /// Save a complete composition event with members and optional outcomes in one transaction. + pub fn save_composition( + &self, + event: &CompositionEventRecord, + members: &[CompositionMemberRecord], + outcomes: &[CompositionOutcomeRecord], + ) -> Result<()> { + let mut writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + let tx = writer.transaction()?; + + let metadata_json = + serde_json::to_string(&event.metadata).unwrap_or_else(|_| "{}".to_string()); + tx.execute( + "INSERT OR REPLACE INTO composition_events ( + id, created_at, tool, mode, query, query_hash, confidence, status, + output_preview, metadata + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + event.id, + event.created_at.to_rfc3339(), + event.tool, + event.mode, + event.query, + event.query_hash, + event.confidence, + event.status, + event.output_preview, + metadata_json, + ], + )?; + + for member in members { + let mut member = member.clone(); + Self::snapshot_composition_member_tags(&tx, &mut member)?; + Self::insert_composition_member(&tx, &member)?; + } + for outcome in outcomes { + Self::insert_composition_outcome(&tx, outcome)?; + } + + tx.commit()?; + Ok(()) + } + + /// Add one outcome label to an existing composition event. + pub fn record_composition_outcome(&self, outcome: &CompositionOutcomeRecord) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + Self::insert_composition_outcome(&writer, outcome) + } + + /// Get one composition event by id. + pub fn get_composition_event(&self, id: &str) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare("SELECT * FROM composition_events WHERE id = ?1")?; + stmt.query_row(params![id], Self::row_to_composition_event) + .optional() + .map_err(StorageError::from) + } + + /// Get recent composition events. + pub fn get_recent_composition_events(&self, limit: i32) -> Result> { + self.get_recent_composition_events_page(limit, 0) + } + + /// Get recent composition events with explicit pagination. + pub fn get_recent_composition_events_page( + &self, + limit: i32, + offset: i32, + ) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT * FROM composition_events + ORDER BY created_at DESC + LIMIT ?1 OFFSET ?2", + )?; + let rows = stmt.query_map( + params![limit.max(1), offset.max(0)], + Self::row_to_composition_event, + )?; + let mut result = Vec::new(); + for row in rows { + result.push(row?); + } + Ok(result) + } + + /// Get all members for a composition event. + pub fn get_composition_members(&self, event_id: &str) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT * FROM composition_members + WHERE event_id = ?1 + ORDER BY rank ASC, role ASC, memory_id ASC", + )?; + let rows = stmt.query_map(params![event_id], Self::row_to_composition_member)?; + let mut result = Vec::new(); + for row in rows { + result.push(row?); + } + Ok(result) + } + + /// Get all outcomes for a composition event. + pub fn get_composition_outcomes( + &self, + event_id: &str, + ) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT * FROM composition_outcomes + WHERE event_id = ?1 + ORDER BY labeled_at DESC", + )?; + let rows = stmt.query_map(params![event_id], Self::row_to_composition_outcome)?; + let mut result = Vec::new(); + for row in rows { + result.push(row?); + } + Ok(result) + } + + /// Get composition events containing a memory id. + pub fn get_compositions_for_memory( + &self, + memory_id: &str, + limit: i32, + ) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT DISTINCT e.* + FROM composition_events e + JOIN composition_members m ON m.event_id = e.id + WHERE m.memory_id = ?1 + ORDER BY e.created_at DESC + LIMIT ?2", + )?; + let rows = stmt.query_map( + params![memory_id, limit.max(1)], + Self::row_to_composition_event, + )?; + let mut result = Vec::new(); + for row in rows { + result.push(row?); + } + Ok(result) + } + + /// Return memories most frequently composed with the requested memory. + pub fn get_composition_neighbors( + &self, + memory_id: &str, + limit: i32, + ) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "WITH distinct_members AS ( + SELECT DISTINCT event_id, memory_id FROM composition_members + ) + SELECT other.memory_id, COUNT(DISTINCT other.event_id) AS composed_count, MAX(e.created_at) AS latest_event_at + FROM distinct_members self + JOIN distinct_members other + ON other.event_id = self.event_id AND other.memory_id != self.memory_id + JOIN composition_events e ON e.id = self.event_id + WHERE self.memory_id = ?1 + GROUP BY other.memory_id + ORDER BY composed_count DESC, latest_event_at DESC + LIMIT ?2", + )?; + let rows = stmt.query_map(params![memory_id, limit.max(1)], |row| { + Ok(CompositionNeighborRecord { + memory_id: row.get(0)?, + composed_count: row.get(1)?, + latest_event_at: Self::parse_timestamp( + &row.get::<_, String>(2)?, + "latest_event_at", + )?, + }) + })?; + let mut result = Vec::new(); + for row in rows { + result.push(row?); + } + Ok(result) + } + + /// Generate ranked memory pairs that share useful tags but have not yet been composed. + pub fn get_never_composed_candidates( + &self, + limit: i32, + tag_filter: Option<&[String]>, + ) -> Result> { + let nodes = self.composition_candidate_nodes(tag_filter)?; + let composed_pairs = self.composed_pair_set()?; + let composition_degrees = self.composition_degree_map()?; + let outcome_map = self.composition_outcome_map()?; + let mut candidates = Vec::new(); + + for i in 0..nodes.len() { + for j in (i + 1)..nodes.len() { + let a = &nodes[i]; + let b = &nodes[j]; + let pair = Self::pair_key(&a.id, &b.id); + if composed_pairs.contains(&pair) { + continue; + } + + if let Some(filter) = tag_filter + && !filter.is_empty() + && !Self::node_pair_matches_tag_filter(a, b, filter) + { + continue; + } + + let shared_tags = Self::shared_tags(&a.tags, &b.tags); + let shared_terms = Self::shared_content_terms(&a.content, &b.content, 8); + if shared_tags.is_empty() && shared_terms.is_empty() { + continue; + } + + let boundary_tags = Self::boundary_tags_for_pair(&a.tags, &b.tags); + let trust_score = + ((a.retention_strength + b.retention_strength) / 2.0).clamp(0.0, 1.0); + let degree_a = composition_degrees.get(&a.id).copied().unwrap_or(0) as f64; + let degree_b = composition_degrees.get(&b.id).copied().unwrap_or(0) as f64; + let novelty_score = ((1.0 / (1.0 + degree_a)) + (1.0 / (1.0 + degree_b))) / 2.0; + let bridge_score = Self::composition_bridge_score( + a, + b, + &shared_tags, + &shared_terms, + &boundary_tags, + ); + let anchor_score = + (shared_tags.len() as f64 * 0.45) + (shared_terms.len().min(5) as f64 * 0.25); + let prior_outcomes = Self::pair_prior_outcomes(&outcome_map, &a.id, &b.id); + let outcome_signal = Self::outcome_signal(&prior_outcomes); + let outcome_score_adjustment = Self::outcome_score_adjustment(&prior_outcomes); + let score = anchor_score + + (bridge_score * 2.0) + + (novelty_score * 1.5) + + trust_score + + outcome_score_adjustment; + if score < 1.6 { + continue; + } + + let reason = if !boundary_tags.is_empty() { + format!( + "Untried bridge across {} with {}", + boundary_tags.join(", "), + Self::anchor_summary(&shared_tags, &shared_terms) + ) + } else if a.node_type != b.node_type { + format!( + "Untried {} -> {} composition with {}", + a.node_type, + b.node_type, + Self::anchor_summary(&shared_tags, &shared_terms) + ) + } else { + format!( + "Never composed despite {}", + Self::anchor_summary(&shared_tags, &shared_terms) + ) + }; + let composition_question = + Self::composition_question(a, b, &shared_tags, &shared_terms, &boundary_tags); + candidates.push(NeverComposedCandidate { + first_id: a.id.clone(), + second_id: b.id.clone(), + score, + novelty_score, + bridge_score, + trust_score, + outcome_score_adjustment, + shared_tags, + boundary_tags, + shared_terms, + prior_outcomes, + outcome_signal, + first_node_type: a.node_type.clone(), + second_node_type: b.node_type.clone(), + first_preview: preview(&a.content, 160), + second_preview: preview(&b.content, 160), + reason, + composition_question, + }); + } + } + + candidates.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + candidates.truncate(limit.max(1) as usize); + Ok(candidates) + } + + fn insert_composition_member( + conn: &Connection, + member: &CompositionMemberRecord, + ) -> Result<()> { + let metadata_json = + serde_json::to_string(&member.metadata).unwrap_or_else(|_| "{}".to_string()); + conn.execute( + "INSERT OR REPLACE INTO composition_members ( + event_id, memory_id, role, rank, trust, score, preview, metadata + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + member.event_id, + member.memory_id, + member.role, + member.rank, + member.trust, + member.score, + member.preview, + metadata_json, + ], + )?; + Ok(()) + } + + fn snapshot_composition_member_tags( + conn: &Connection, + member: &mut CompositionMemberRecord, + ) -> Result<()> { + if member.metadata.get("tags").is_some() { + return Ok(()); + } + + let tags_json: Option = conn + .query_row( + "SELECT tags FROM knowledge_nodes WHERE id = ?1", + params![member.memory_id], + |row| row.get(0), + ) + .optional()?; + let Some(tags_json) = tags_json else { + return Ok(()); + }; + let Ok(tags) = serde_json::from_str::>(&tags_json) else { + return Ok(()); + }; + if tags.is_empty() { + return Ok(()); + } + + if let Some(object) = member.metadata.as_object_mut() { + object.insert("tags".to_string(), serde_json::json!(tags)); + } else { + member.metadata = serde_json::json!({ "tags": tags }); + } + Ok(()) + } + + fn insert_composition_outcome( + conn: &Connection, + outcome: &CompositionOutcomeRecord, + ) -> Result<()> { + let metadata_json = + serde_json::to_string(&outcome.metadata).unwrap_or_else(|_| "{}".to_string()); + conn.execute( + "INSERT OR REPLACE INTO composition_outcomes ( + id, event_id, outcome_type, labeled_at, label_source, + confidence_delta, notes, metadata + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + outcome.id, + outcome.event_id, + outcome.outcome_type, + outcome.labeled_at.to_rfc3339(), + outcome.label_source, + outcome.confidence_delta, + outcome.notes, + metadata_json, + ], + )?; + Ok(()) + } + + fn row_to_composition_event(row: &rusqlite::Row) -> rusqlite::Result { + let metadata_json: String = row.get("metadata")?; + Ok(CompositionEventRecord { + id: row.get("id")?, + created_at: Self::parse_timestamp(&row.get::<_, String>("created_at")?, "created_at")?, + tool: row.get("tool")?, + mode: row.get("mode")?, + query: row.get("query").ok().flatten(), + query_hash: row.get("query_hash").ok().flatten(), + confidence: row.get("confidence").ok().flatten(), + status: row.get("status").ok().flatten(), + output_preview: row.get("output_preview").ok().flatten(), + metadata: serde_json::from_str(&metadata_json) + .unwrap_or_else(|_| serde_json::json!({})), + }) + } + + fn row_to_composition_member(row: &rusqlite::Row) -> rusqlite::Result { + let metadata_json: String = row.get("metadata")?; + Ok(CompositionMemberRecord { + event_id: row.get("event_id")?, + memory_id: row.get("memory_id")?, + role: row.get("role")?, + rank: row.get("rank").unwrap_or(0), + trust: row.get("trust").ok().flatten(), + score: row.get("score").ok().flatten(), + preview: row.get("preview").ok().flatten(), + metadata: serde_json::from_str(&metadata_json) + .unwrap_or_else(|_| serde_json::json!({})), + }) + } + + fn row_to_composition_outcome( + row: &rusqlite::Row, + ) -> rusqlite::Result { + let metadata_json: String = row.get("metadata")?; + Ok(CompositionOutcomeRecord { + id: row.get("id")?, + event_id: row.get("event_id")?, + outcome_type: row.get("outcome_type")?, + labeled_at: Self::parse_timestamp(&row.get::<_, String>("labeled_at")?, "labeled_at")?, + label_source: row + .get("label_source") + .unwrap_or_else(|_| "tool".to_string()), + confidence_delta: row.get("confidence_delta").ok().flatten(), + notes: row.get("notes").ok().flatten(), + metadata: serde_json::from_str(&metadata_json) + .unwrap_or_else(|_| serde_json::json!({})), + }) + } + + fn composition_event_exists(conn: &Connection, id: &str) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM composition_events WHERE id = ?1", + params![id], + |row| row.get(0), + )?; + Ok(count > 0) + } + + fn composed_pair_set(&self) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT event_id, memory_id + FROM composition_members + ORDER BY event_id, memory_id", + )?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + let mut grouped: HashMap> = HashMap::new(); + for row in rows { + let (event_id, memory_id) = row?; + grouped.entry(event_id).or_default().push(memory_id); + } + + let mut pairs = HashSet::new(); + for ids in grouped.values_mut() { + ids.sort(); + ids.dedup(); + for i in 0..ids.len() { + for j in (i + 1)..ids.len() { + pairs.insert(Self::pair_key(&ids[i], &ids[j])); + } + } + } + Ok(pairs) + } + + fn pair_key(a: &str, b: &str) -> (String, String) { + if a <= b { + (a.to_string(), b.to_string()) + } else { + (b.to_string(), a.to_string()) + } + } + + fn shared_tags(a: &[String], b: &[String]) -> Vec { + let b_set: HashSet<&str> = b.iter().map(String::as_str).collect(); + let mut shared = a + .iter() + .filter(|tag| b_set.contains(tag.as_str())) + .cloned() + .collect::>(); + shared.sort(); + shared.dedup(); + shared + } + + fn node_pair_matches_tag_filter( + a: &KnowledgeNode, + b: &KnowledgeNode, + tag_filter: &[String], + ) -> bool { + a.tags.iter().chain(b.tags.iter()).any(|tag| { + tag_filter + .iter() + .any(|wanted| wanted == tag || tag.starts_with(&format!("{wanted}:"))) + }) + } + + fn boundary_tags_for_pair(a: &[String], b: &[String]) -> Vec { + let mut tags = a + .iter() + .chain(b.iter()) + .filter(|tag| Self::is_boundary_tag(tag)) + .cloned() + .collect::>(); + tags.sort(); + tags.dedup(); + tags + } + + fn composition_bridge_score( + a: &KnowledgeNode, + b: &KnowledgeNode, + shared_tags: &[String], + shared_terms: &[String], + boundary_tags: &[String], + ) -> f64 { + let tag_distance = Self::tag_distance(&a.tags, &b.tags); + let node_type_bridge = if a.node_type != b.node_type { 1.0 } else { 0.0 }; + let boundary_bridge = (boundary_tags.len() as f64 / 4.0).min(1.0); + let lexical_anchor = if shared_terms.is_empty() { 0.0 } else { 1.0 }; + let tag_anchor = if shared_tags.is_empty() { 0.0 } else { 1.0 }; + + (tag_distance * 0.30 + + node_type_bridge * 0.20 + + boundary_bridge * 0.25 + + lexical_anchor * 0.15 + + tag_anchor * 0.10) + .clamp(0.0, 1.0) + } + + fn tag_distance(a: &[String], b: &[String]) -> f64 { + let a_set = a.iter().map(String::as_str).collect::>(); + let b_set = b.iter().map(String::as_str).collect::>(); + let union = a_set.union(&b_set).count(); + if union == 0 { + return 0.0; + } + let intersection = a_set.intersection(&b_set).count(); + 1.0 - (intersection as f64 / union as f64) + } + + fn shared_content_terms(a: &str, b: &str, limit: usize) -> Vec { + let a_terms = Self::content_terms(a); + let b_terms = Self::content_terms(b); + let mut shared = a_terms + .intersection(&b_terms) + .cloned() + .collect::>(); + shared.sort_by(|left, right| { + Self::term_specificity_score(right) + .cmp(&Self::term_specificity_score(left)) + .then_with(|| left.cmp(right)) + }); + shared.truncate(limit); + shared + } + + fn content_terms(content: &str) -> HashSet { + const STOPWORDS: &[&str] = &[ + "about", "after", "again", "against", "because", "before", "between", "could", "every", + "first", "from", "have", "into", "memory", "needs", "should", "their", "there", + "these", "thing", "through", "using", "where", "which", "while", "would", + ]; + content + .to_ascii_lowercase() + .split(|c: char| !c.is_ascii_alphanumeric() && c != '-' && c != '_') + .filter(|term| term.len() >= 5 && !STOPWORDS.contains(term)) + .map(ToOwned::to_owned) + .collect() + } + + fn term_specificity_score(term: &str) -> usize { + term.len() + + term.chars().filter(|ch| ch.is_ascii_digit()).count() * 2 + + usize::from(term.contains('-')) * 2 + + usize::from(term.contains('_')) * 2 + } + + fn anchor_summary(shared_tags: &[String], shared_terms: &[String]) -> String { + if !shared_tags.is_empty() && !shared_terms.is_empty() { + format!( + "shared tags ({}) and shared terms ({})", + shared_tags.join(", "), + shared_terms + .iter() + .take(4) + .cloned() + .collect::>() + .join(", ") + ) + } else if !shared_tags.is_empty() { + format!("shared tags ({})", shared_tags.join(", ")) + } else { + format!( + "shared terms ({})", + shared_terms + .iter() + .take(4) + .cloned() + .collect::>() + .join(", ") + ) + } + } + + fn composition_question( + a: &KnowledgeNode, + b: &KnowledgeNode, + shared_tags: &[String], + shared_terms: &[String], + boundary_tags: &[String], + ) -> String { + let anchor = if !boundary_tags.is_empty() { + boundary_tags.join(", ") + } else if !shared_tags.is_empty() { + shared_tags.join(", ") + } else { + shared_terms + .iter() + .take(3) + .cloned() + .collect::>() + .join(", ") + }; + format!( + "What changes if a {} memory and a {} memory are composed through {}?", + a.node_type, b.node_type, anchor + ) + } + + fn composition_degree_map(&self) -> Result> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT memory_id, COUNT(DISTINCT event_id) AS composition_count + FROM composition_members + GROUP BY memory_id", + )?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut result = HashMap::new(); + for row in rows { + let (memory_id, count) = row?; + result.insert(memory_id, count); + } + Ok(result) + } + + fn composition_candidate_nodes( + &self, + tag_filter: Option<&[String]>, + ) -> Result> { + const BASE_SCAN_LIMIT: i32 = 750; + const TAGGED_SCAN_LIMIT: i32 = 1500; + + let mut nodes = self.get_all_nodes(BASE_SCAN_LIMIT, 0)?; + if let Some(filter) = tag_filter + && !filter.is_empty() + { + let tagged_nodes = self.get_nodes_matching_any_tag_prefix(filter, TAGGED_SCAN_LIMIT)?; + let mut by_id = HashMap::new(); + for node in nodes.into_iter().chain(tagged_nodes.into_iter()) { + by_id.entry(node.id.clone()).or_insert(node); + } + nodes = by_id.into_values().collect(); + nodes.sort_by(|a, b| { + b.retention_strength + .partial_cmp(&a.retention_strength) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| b.created_at.cmp(&a.created_at)) + }); + } + Ok(nodes) + } + + fn get_nodes_matching_any_tag_prefix( + &self, + tag_filter: &[String], + limit: i32, + ) -> Result> { + let mut patterns = Vec::new(); + for wanted in tag_filter + .iter() + .map(|tag| tag.trim()) + .filter(|tag| !tag.is_empty()) + { + patterns.push(format!("%\"{}\"%", wanted)); + patterns.push(format!("%\"{}:%", wanted)); + } + if patterns.is_empty() { + return Ok(Vec::new()); + } + + let clauses = std::iter::repeat_n("tags LIKE ?", patterns.len()) + .collect::>() + .join(" OR "); + let sql = format!( + "SELECT * FROM knowledge_nodes + WHERE {clauses} + ORDER BY retention_strength DESC, created_at DESC + LIMIT {}", + limit.clamp(1, 5000) + ); + + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare(&sql)?; + let rows = stmt.query_map(params_from_iter(patterns.iter()), Self::row_to_node)?; + let mut result = Vec::new(); + for row in rows { + result.push(row?); + } + Ok(result) + } + + fn composition_outcome_map(&self) -> Result>> { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT DISTINCT m.memory_id, o.outcome_type + FROM composition_members m + JOIN composition_outcomes o ON o.event_id = m.event_id", + )?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + let mut result: HashMap> = HashMap::new(); + for row in rows { + let (memory_id, outcome) = row?; + result.entry(memory_id).or_default().insert(outcome); + } + Ok(result) + } + + fn pair_prior_outcomes( + outcome_map: &HashMap>, + first_id: &str, + second_id: &str, + ) -> Vec { + let mut outcomes = outcome_map + .get(first_id) + .into_iter() + .chain(outcome_map.get(second_id)) + .flat_map(|values| values.iter().cloned()) + .collect::>(); + outcomes.sort(); + outcomes.dedup(); + outcomes + } + + fn outcome_signal(prior_outcomes: &[String]) -> String { + if prior_outcomes.is_empty() { + return "clean".to_string(); + } + + let has_closed = prior_outcomes.iter().any(|outcome| { + matches!( + outcome.as_str(), + "dead_end" + | "rejected" + | "bad_severity" + | "user_demoted" + | "closed_by_scope" + | "closed_by_false_assumption" + | "closed_by_user" + | "expired_lane" + ) + }); + let has_duplicate = prior_outcomes + .iter() + .any(|outcome| matches!(outcome.as_str(), "duplicate_risk" | "closed_by_duplicate")); + let has_success = prior_outcomes.iter().any(|outcome| { + matches!( + outcome.as_str(), + "accepted" | "helpful" | "submitted" | "user_promoted" + ) + }); + let has_needs_poc = prior_outcomes.iter().any(|outcome| outcome == "needs_poc"); + + if (has_closed || has_duplicate) && has_success { + "mixed_prior_outcomes".to_string() + } else if has_closed { + "prior_closed_door".to_string() + } else if has_duplicate { + "prior_duplicate_risk".to_string() + } else if has_success { + "prior_success".to_string() + } else if has_needs_poc { + "prior_needs_poc".to_string() + } else { + "prior_outcome".to_string() + } + } + + fn outcome_score_adjustment(prior_outcomes: &[String]) -> f64 { + let mut adjustment: f64 = 0.0; + for outcome in prior_outcomes { + adjustment += match outcome.as_str() { + "accepted" => 0.35, + "helpful" => 0.25, + "submitted" => 0.15, + "user_promoted" => 0.20, + "needs_poc" => -0.05, + "duplicate_risk" => -0.35, + "closed_by_duplicate" => -0.40, + "dead_end" + | "rejected" + | "bad_severity" + | "closed_by_scope" + | "closed_by_false_assumption" + | "closed_by_user" + | "expired_lane" => -0.45, + "user_demoted" => -0.20, + _ => 0.0, + }; + } + adjustment.clamp(-0.8, 0.5) + } + + fn is_boundary_tag(tag: &str) -> bool { + let lowered = tag.to_ascii_lowercase(); + lowered.starts_with("boundary-") + || matches!( + lowered.as_str(), + "time" + | "chain" + | "role" + | "oracle" + | "queue" + | "settlement" + | "keeper" + | "upgrade" + | "pause" + | "accounting" + | "scope" + ) + } + // ======================================================================== // INTENTIONS PERSISTENCE // ======================================================================== @@ -5213,6 +6180,17 @@ impl Storage { | "consolidation_history" | "dream_history" | "retention_snapshots" => Self::merge_append_only_table(tx, table_name, table, report), + "composition_events" | "composition_outcomes" => { + Self::merge_keyed_table(tx, table_name, table, &["id"], report, state) + } + "composition_members" => Self::merge_keyed_table( + tx, + table_name, + table, + &["event_id", "memory_id", "role"], + report, + state, + ), "node_embeddings" => { Self::merge_keyed_table(tx, table_name, table, &["node_id"], report, state) } @@ -5363,6 +6341,10 @@ impl Storage { (None, _) => false, }; if should_delete { + tx.execute( + "UPDATE composition_members SET preview = NULL WHERE memory_id = ?1", + params![row_id], + )?; let deleted = tx.execute("DELETE FROM knowledge_nodes WHERE id = ?1", params![row_id])?; report.rows_deleted += deleted; @@ -5534,6 +6516,20 @@ impl Storage { .unwrap_or(false); Ok(source_exists && target_exists) } + "composition_members" => { + let event_exists = Self::portable_text(table, row, "event_id") + .map(|id| Self::composition_event_exists(tx, id)) + .transpose()? + .unwrap_or(false); + Ok(event_exists) + } + "composition_outcomes" => { + let event_exists = Self::portable_text(table, row, "event_id") + .map(|id| Self::composition_event_exists(tx, id)) + .transpose()? + .unwrap_or(false); + Ok(event_exists) + } _ => Ok(true), } } @@ -5557,6 +6553,8 @@ impl Storage { fn merge_key_columns(table_name: &str) -> &'static [&'static str] { match table_name { "knowledge_nodes" | "intentions" | "insights" | "sessions" => &["id"], + "composition_events" | "composition_outcomes" => &["id"], + "composition_members" => &["event_id", "memory_id", "role"], "node_embeddings" => &["node_id"], "fsrs_cards" | "memory_states" | "deletion_tombstones" => &["memory_id"], "memory_connections" => &["source_id", "target_id"], @@ -6377,7 +7375,10 @@ impl Storage { let possible_threshold = read_key("merge_possible_threshold") .map(|v| v as f32) .unwrap_or_else(|| { - env_f32("VESTIGE_MERGE_POSSIBLE_THRESHOLD", default.possible_threshold) + env_f32( + "VESTIGE_MERGE_POSSIBLE_THRESHOLD", + default.possible_threshold, + ) }); let auto_apply = match read_key("merge_auto_apply") { Some(v) => v != 0.0, @@ -6495,8 +7496,10 @@ impl Storage { } // Best pair score per resulting cluster member, for the explanation. - let mut pair_score: std::collections::HashMap<(usize, usize), crate::advanced::MatchSignals> = - std::collections::HashMap::new(); + let mut pair_score: std::collections::HashMap< + (usize, usize), + crate::advanced::MatchSignals, + > = std::collections::HashMap::new(); for i in 0..n { for j in (i + 1)..n { @@ -6697,10 +7700,12 @@ impl Storage { .map(|n| (n.id.clone(), n.content.clone())) .collect(); let result_content = compose_merged_content(&members); - let result_tags = compose_merged_tags( - &nodes.iter().map(|n| n.tags.clone()).collect::>(), - ); - let result_source = nodes.iter().find(|n| n.id == survivor).and_then(|n| n.source.clone()); + let result_tags = + compose_merged_tags(&nodes.iter().map(|n| n.tags.clone()).collect::>()); + let result_source = nodes + .iter() + .find(|n| n.id == survivor) + .and_then(|n| n.source.clone()); let invalidated_ids: Vec = nodes .iter() .filter(|n| n.id != survivor) @@ -6968,11 +7973,7 @@ impl Storage { undo.insert("absorbed".into(), serde_json::json!(absorbed)); // Apply: rewrite survivor, invalidate absorbed. - self.rewrite_survivor( - &plan.survivor_id, - &plan.result_content, - &plan.result_tags, - )?; + self.rewrite_survivor(&plan.survivor_id, &plan.result_content, &plan.result_tags)?; for id in &plan.invalidated_ids { self.invalidate_node(id, &plan.survivor_id, now)?; } @@ -7046,9 +8047,7 @@ impl Storage { ))); } if op.op_type == "undo" { - return Err(StorageError::Init( - "cannot undo an undo operation".into(), - )); + return Err(StorageError::Init("cannot undo an undo operation".into())); } let undo: serde_json::Value = { @@ -7180,9 +8179,7 @@ impl Storage { Ok(op) } - fn row_to_operation( - row: &rusqlite::Row, - ) -> rusqlite::Result { + fn row_to_operation(row: &rusqlite::Row) -> rusqlite::Result { let affected: String = row.get("affected_ids")?; let affected_ids: Vec = serde_json::from_str(&affected).unwrap_or_default(); Ok(crate::advanced::MergeOperation { @@ -7195,7 +8192,11 @@ impl Storage { reverts_op_id: row.get("reverts_op_id").ok().flatten(), survivor_id: row.get("survivor_id").ok().flatten(), affected_ids, - confidence: row.get::<_, Option>("confidence").ok().flatten().map(|v| v as f32), + confidence: row + .get::<_, Option>("confidence") + .ok() + .flatten() + .map(|v| v as f32), reason: row.get("reason").ok().flatten(), }) } @@ -7402,13 +8403,17 @@ mod tests { use chrono::TimeZone; // Canonical writer: RFC 3339 with fractional seconds + offset. - let rfc = Storage::parse_timestamp("2026-06-12T15:07:59.730+00:00", "last_accessed").unwrap(); + let rfc = + Storage::parse_timestamp("2026-06-12T15:07:59.730+00:00", "last_accessed").unwrap(); assert_eq!(rfc.to_rfc3339(), "2026-06-12T15:07:59.730+00:00"); // External writer: SQLite-native `datetime('now')` (space separator, // no timezone, no fraction) — must be tolerated, assumed UTC. let sqlite = Storage::parse_timestamp("2026-06-12 15:07:59", "last_accessed").unwrap(); - assert_eq!(sqlite, Utc.with_ymd_and_hms(2026, 6, 12, 15, 7, 59).unwrap()); + assert_eq!( + sqlite, + Utc.with_ymd_and_hms(2026, 6, 12, 15, 7, 59).unwrap() + ); // SQLite-native with fractional seconds. let sqlite_frac = @@ -7490,6 +8495,622 @@ mod tests { assert!(storage.get_node(&node.id).unwrap().is_none()); } + #[test] + fn test_composition_save_query_outcome_and_never_composed() { + let storage = create_test_storage(); + let first = storage + .ingest(IngestInput { + content: "Oracle drift can break delayed settlement.".to_string(), + node_type: "fact".to_string(), + tags: vec![ + "protocolgate".to_string(), + "boundary-oracle".to_string(), + "settlement".to_string(), + ], + ..Default::default() + }) + .unwrap(); + let second = storage + .ingest(IngestInput { + content: "Withdrawal queues can settle stale claims.".to_string(), + node_type: "pattern".to_string(), + tags: vec![ + "protocolgate".to_string(), + "boundary-queue".to_string(), + "settlement".to_string(), + ], + ..Default::default() + }) + .unwrap(); + let third = storage + .ingest(IngestInput { + content: "Keeper roles can drift from local validation paths.".to_string(), + node_type: "pattern".to_string(), + tags: vec![ + "protocolgate".to_string(), + "boundary-role".to_string(), + "settlement".to_string(), + ], + ..Default::default() + }) + .unwrap(); + + let before = storage + .get_never_composed_candidates(10, Some(&["protocolgate".to_string()])) + .unwrap(); + let first_second_before = before + .iter() + .find(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&first.id, &second.id) + }) + .expect("uncomposed first/second pair should be ranked before any event"); + assert!( + first_second_before.bridge_score > 0.0, + "candidate should expose a bridge score" + ); + assert!( + first_second_before.novelty_score > 0.0, + "candidate should expose a novelty score" + ); + assert_eq!( + first_second_before.outcome_signal, "clean", + "new candidate should start without prior outcome context" + ); + assert!( + first_second_before + .composition_question + .contains("composed through"), + "candidate should include a promptable composition question" + ); + + let event = CompositionEventRecord { + id: "composition-test-1".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "bounty".to_string(), + query: Some("oracle drift delayed settlement".to_string()), + query_hash: Some("sha256:test".to_string()), + confidence: Some(0.87), + status: Some("resolved".to_string()), + output_preview: Some("Compose oracle drift with withdrawal queue.".to_string()), + metadata: serde_json::json!({"workflow": "test"}), + }; + let members = vec![ + CompositionMemberRecord { + event_id: event.id.clone(), + memory_id: first.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.9), + preview: Some(preview(&first.content, 120)), + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: event.id.clone(), + memory_id: second.id.clone(), + role: "supporting".to_string(), + rank: 1, + trust: Some(0.7), + score: Some(0.75), + preview: Some(preview(&second.content, 120)), + metadata: serde_json::json!({}), + }, + ]; + storage.save_composition(&event, &members, &[]).unwrap(); + + let outcome = CompositionOutcomeRecord { + id: "composition-outcome-1".to_string(), + event_id: event.id.clone(), + outcome_type: "submitted".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: Some(0.1), + notes: Some("Report submitted".to_string()), + metadata: serde_json::json!({"severity": "high"}), + }; + storage.record_composition_outcome(&outcome).unwrap(); + + let fetched = storage.get_composition_event(&event.id).unwrap().unwrap(); + assert_eq!(fetched.mode, "bounty"); + assert_eq!(fetched.metadata["workflow"], "test"); + + let fetched_members = storage.get_composition_members(&event.id).unwrap(); + assert_eq!(fetched_members.len(), 2); + assert_eq!(fetched_members[0].role, "primary"); + + let fetched_outcomes = storage.get_composition_outcomes(&event.id).unwrap(); + assert_eq!(fetched_outcomes.len(), 1); + assert_eq!(fetched_outcomes[0].outcome_type, "submitted"); + + let for_memory = storage.get_compositions_for_memory(&first.id, 5).unwrap(); + assert_eq!(for_memory.len(), 1); + assert_eq!(for_memory[0].id, event.id); + + let neighbors = storage.get_composition_neighbors(&first.id, 5).unwrap(); + assert_eq!(neighbors.len(), 1); + assert_eq!(neighbors[0].memory_id, second.id); + + let after = storage + .get_never_composed_candidates(10, Some(&["protocolgate".to_string()])) + .unwrap(); + assert!( + !after.iter().any(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&first.id, &second.id) + }), + "already-composed first/second pair should be removed" + ); + assert!( + after.iter().any(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&first.id, &third.id) + || pair == Storage::pair_key(&second.id, &third.id) + }), + "other protocolgate pairs should remain candidates" + ); + } + + #[test] + fn test_composition_neighbors_count_distinct_events_not_member_roles() { + let storage = create_test_storage(); + let first = storage + .ingest(IngestInput { + content: "Oracle role appears once in the event.".to_string(), + node_type: "fact".to_string(), + tags: vec!["protocolgate".to_string(), "settlement".to_string()], + ..Default::default() + }) + .unwrap(); + let second = storage + .ingest(IngestInput { + content: "Queue role appears under two evidence roles.".to_string(), + node_type: "fact".to_string(), + tags: vec!["protocolgate".to_string(), "settlement".to_string()], + ..Default::default() + }) + .unwrap(); + + storage + .save_composition( + &CompositionEventRecord { + id: "multi-role-neighbor-event".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "bounty".to_string(), + query: Some("multi role neighbor".to_string()), + query_hash: Some("fnv1a64:neighbor".to_string()), + confidence: Some(0.7), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[ + CompositionMemberRecord { + event_id: "multi-role-neighbor-event".to_string(), + memory_id: first.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.9), + preview: None, + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: "multi-role-neighbor-event".to_string(), + memory_id: second.id.clone(), + role: "supporting".to_string(), + rank: 1, + trust: Some(0.7), + score: Some(0.8), + preview: None, + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: "multi-role-neighbor-event".to_string(), + memory_id: second.id.clone(), + role: "related".to_string(), + rank: 2, + trust: Some(0.7), + score: Some(0.6), + preview: None, + metadata: serde_json::json!({}), + }, + ], + &[], + ) + .unwrap(); + + let neighbors = storage.get_composition_neighbors(&first.id, 10).unwrap(); + assert_eq!(neighbors.len(), 1); + assert_eq!(neighbors[0].memory_id, second.id); + assert_eq!( + neighbors[0].composed_count, 1, + "one event with multiple member roles should count as one composition" + ); + } + + #[test] + fn test_never_composed_tag_filter_includes_older_tagged_candidates() { + let storage = create_test_storage(); + let first = storage + .ingest(IngestInput { + content: "Older Vestige composition frontier about outcome-shaped recall." + .to_string(), + node_type: "fact".to_string(), + tags: vec!["project:vestige".to_string(), "composition".to_string()], + ..Default::default() + }) + .unwrap(); + let second = storage + .ingest(IngestInput { + content: "Older Vestige composition frontier about never-composed recall." + .to_string(), + node_type: "pattern".to_string(), + tags: vec!["project:vestige".to_string(), "composition".to_string()], + ..Default::default() + }) + .unwrap(); + + for idx in 0..751 { + storage + .ingest(IngestInput { + content: format!("Unrelated recent memory {idx} for scan-window pressure."), + node_type: "fact".to_string(), + tags: vec!["unrelated".to_string()], + ..Default::default() + }) + .unwrap(); + } + + let candidates = storage + .get_never_composed_candidates(10, Some(&["project".to_string()])) + .unwrap(); + assert!( + candidates.iter().any(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&first.id, &second.id) + }), + "tag-filtered frontier should include older namespaced-tag memories outside the base scan window" + ); + } + + #[test] + fn test_never_composed_carries_prior_outcome_signal() { + let storage = create_test_storage(); + let first = storage + .ingest(IngestInput { + content: "Oracle drift lane previously looked duplicate-prone.".to_string(), + node_type: "fact".to_string(), + tags: vec![ + "protocolgate".to_string(), + "boundary-oracle".to_string(), + "settlement".to_string(), + ], + ..Default::default() + }) + .unwrap(); + let second = storage + .ingest(IngestInput { + content: "Withdrawal queue lane had weak proof.".to_string(), + node_type: "fact".to_string(), + tags: vec![ + "protocolgate".to_string(), + "boundary-queue".to_string(), + "settlement".to_string(), + ], + ..Default::default() + }) + .unwrap(); + let third = storage + .ingest(IngestInput { + content: "Keeper settlement lane has not been composed with oracle drift." + .to_string(), + node_type: "pattern".to_string(), + tags: vec![ + "protocolgate".to_string(), + "boundary-role".to_string(), + "settlement".to_string(), + ], + ..Default::default() + }) + .unwrap(); + + let event = CompositionEventRecord { + id: "prior-outcome-composition".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "bounty".to_string(), + query: Some("oracle withdrawal duplicate risk".to_string()), + query_hash: Some("fnv1a64:prior".to_string()), + confidence: Some(0.4), + status: Some("closed".to_string()), + output_preview: Some("Prior composition was labeled duplicate risk.".to_string()), + metadata: serde_json::json!({}), + }; + storage + .save_composition( + &event, + &[ + CompositionMemberRecord { + event_id: event.id.clone(), + memory_id: first.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.7), + score: Some(0.8), + preview: None, + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: event.id.clone(), + memory_id: second.id.clone(), + role: "supporting".to_string(), + rank: 1, + trust: Some(0.7), + score: Some(0.8), + preview: None, + metadata: serde_json::json!({}), + }, + ], + &[CompositionOutcomeRecord { + id: "prior-outcome-label".to_string(), + event_id: event.id.clone(), + outcome_type: "duplicate_risk".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: Some(-0.2), + notes: Some("Duplicate family in prior lane.".to_string()), + metadata: serde_json::json!({}), + }], + ) + .unwrap(); + + let candidates = storage + .get_never_composed_candidates(10, Some(&["protocolgate".to_string()])) + .unwrap(); + let candidate = candidates + .iter() + .find(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&first.id, &third.id) + }) + .expect("untried first/third pair should remain a frontier candidate"); + + assert!( + candidate + .prior_outcomes + .iter() + .any(|outcome| outcome == "duplicate_risk"), + "frontier candidate should expose prior outcome labels from either member" + ); + assert_eq!(candidate.outcome_signal, "prior_duplicate_risk"); + assert!( + candidate.outcome_score_adjustment < 0.0, + "duplicate-risk history should reduce but not hide the untried lane" + ); + } + + #[test] + fn test_never_composed_marks_mixed_prior_outcomes() { + let storage = create_test_storage(); + let successful = storage + .ingest(IngestInput { + content: "Accepted release lane linked rollback evidence to install telemetry." + .to_string(), + node_type: "decision".to_string(), + tags: vec![ + "project:vestige".to_string(), + "release".to_string(), + "telemetry".to_string(), + ], + ..Default::default() + }) + .unwrap(); + let closed = storage + .ingest(IngestInput { + content: "Closed release lane linked install telemetry to out-of-scope claims." + .to_string(), + node_type: "incident".to_string(), + tags: vec![ + "project:vestige".to_string(), + "release".to_string(), + "telemetry".to_string(), + ], + ..Default::default() + }) + .unwrap(); + let success_helper = storage + .ingest(IngestInput { + content: "Helper memory for an accepted release composition.".to_string(), + node_type: "fact".to_string(), + tags: vec!["project:vestige".to_string(), "release".to_string()], + ..Default::default() + }) + .unwrap(); + let closed_helper = storage + .ingest(IngestInput { + content: "Helper memory for a closed release composition.".to_string(), + node_type: "fact".to_string(), + tags: vec!["project:vestige".to_string(), "release".to_string()], + ..Default::default() + }) + .unwrap(); + + storage + .save_composition( + &CompositionEventRecord { + id: "prior-success-composition".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "release".to_string(), + query: Some("accepted release lane".to_string()), + query_hash: Some("fnv1a64:success".to_string()), + confidence: Some(0.9), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[ + CompositionMemberRecord { + event_id: "prior-success-composition".to_string(), + memory_id: successful.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.9), + score: Some(0.9), + preview: None, + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: "prior-success-composition".to_string(), + memory_id: success_helper.id, + role: "supporting".to_string(), + rank: 1, + trust: Some(0.7), + score: Some(0.6), + preview: None, + metadata: serde_json::json!({}), + }, + ], + &[CompositionOutcomeRecord { + id: "prior-success-label".to_string(), + event_id: "prior-success-composition".to_string(), + outcome_type: "accepted".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: Some(0.2), + notes: None, + metadata: serde_json::json!({}), + }], + ) + .unwrap(); + + storage + .save_composition( + &CompositionEventRecord { + id: "prior-closed-composition".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "release".to_string(), + query: Some("closed release lane".to_string()), + query_hash: Some("fnv1a64:closed".to_string()), + confidence: Some(0.3), + status: Some("closed".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[ + CompositionMemberRecord { + event_id: "prior-closed-composition".to_string(), + memory_id: closed.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.7), + preview: None, + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: "prior-closed-composition".to_string(), + memory_id: closed_helper.id, + role: "supporting".to_string(), + rank: 1, + trust: Some(0.7), + score: Some(0.6), + preview: None, + metadata: serde_json::json!({}), + }, + ], + &[CompositionOutcomeRecord { + id: "prior-closed-label".to_string(), + event_id: "prior-closed-composition".to_string(), + outcome_type: "closed_by_scope".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: Some(-0.3), + notes: None, + metadata: serde_json::json!({}), + }], + ) + .unwrap(); + + let candidates = storage + .get_never_composed_candidates(10, Some(&["project".to_string()])) + .unwrap(); + let candidate = candidates + .iter() + .find(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&successful.id, &closed.id) + }) + .expect("untried success/closed pair should remain a frontier candidate"); + + assert_eq!(candidate.outcome_signal, "mixed_prior_outcomes"); + assert!( + candidate + .prior_outcomes + .iter() + .any(|outcome| outcome == "accepted") + ); + assert!( + candidate + .prior_outcomes + .iter() + .any(|outcome| outcome == "closed_by_scope") + ); + } + + #[test] + fn test_never_composed_surfaces_weak_tie_shared_terms_without_shared_tags() { + let storage = create_test_storage(); + let incident = storage + .ingest(IngestInput { + content: + "OpenCode handshake stalls when embedding startup blocks stdio negotiation." + .to_string(), + node_type: "incident".to_string(), + tags: vec!["opencode".to_string(), "startup".to_string()], + ..Default::default() + }) + .unwrap(); + let mitigation = storage + .ingest(IngestInput { + content: "JetBrains startup should keep embedding backfill behind the handshake." + .to_string(), + node_type: "mitigation".to_string(), + tags: vec!["jetbrains".to_string(), "background-work".to_string()], + ..Default::default() + }) + .unwrap(); + + let candidates = storage.get_never_composed_candidates(10, None).unwrap(); + let candidate = candidates + .iter() + .find(|candidate| { + let pair = Storage::pair_key(&candidate.first_id, &candidate.second_id); + pair == Storage::pair_key(&incident.id, &mitigation.id) + }) + .expect("shared terms should surface a weak-tie candidate without shared tags"); + + assert!( + candidate.shared_tags.is_empty(), + "test fixture intentionally has no shared tags" + ); + assert!( + candidate + .shared_terms + .iter() + .any(|term| term == "embedding" || term == "startup" || term == "handshake"), + "shared terms should explain the candidate" + ); + assert!( + candidate.bridge_score > 0.5, + "different tags and node types should create a bridge signal" + ); + } + #[test] fn test_dream_history_save_and_get_last() { let storage = create_test_storage(); @@ -7586,6 +9207,54 @@ mod tests { activation_count: 1, }) .unwrap(); + source + .save_composition( + &CompositionEventRecord { + id: "portable-composition-1".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "bounty".to_string(), + query: Some("portable composition".to_string()), + query_hash: Some("sha256:portable".to_string()), + confidence: Some(0.9), + status: Some("resolved".to_string()), + output_preview: Some("Portable composition event".to_string()), + metadata: serde_json::json!({}), + }, + &[ + CompositionMemberRecord { + event_id: "portable-composition-1".to_string(), + memory_id: first.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.9), + score: Some(1.0), + preview: Some("alpha".to_string()), + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: "portable-composition-1".to_string(), + memory_id: second.id.clone(), + role: "supporting".to_string(), + rank: 1, + trust: Some(0.8), + score: Some(0.8), + preview: Some("beta".to_string()), + metadata: serde_json::json!({}), + }, + ], + &[CompositionOutcomeRecord { + id: "portable-composition-outcome-1".to_string(), + event_id: "portable-composition-1".to_string(), + outcome_type: "helpful".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: None, + notes: None, + metadata: serde_json::json!({}), + }], + ) + .unwrap(); let archive = source.export_portable_archive().unwrap(); assert_eq!(archive.archive_format, PORTABLE_ARCHIVE_FORMAT); @@ -7596,6 +9265,16 @@ mod tests { .iter() .any(|table| table.name == "knowledge_nodes" && table.rows.len() == 2) ); + for table_name in [ + "composition_events", + "composition_members", + "composition_outcomes", + ] { + assert!( + archive.tables.iter().any(|table| table.name == table_name), + "{table_name} must be included in portable archive" + ); + } let target = create_test_storage_at(&target_dir, "target.db"); let report = target @@ -7614,6 +9293,26 @@ mod tests { assert_eq!(connections.len(), 1); assert_eq!(connections[0].target_id, second.id); + let composition = target + .get_composition_event("portable-composition-1") + .unwrap() + .unwrap(); + assert_eq!(composition.mode, "bounty"); + assert_eq!( + target + .get_composition_members("portable-composition-1") + .unwrap() + .len(), + 2 + ); + assert_eq!( + target + .get_composition_outcomes("portable-composition-1") + .unwrap() + .len(), + 1 + ); + let results = target.search("alpha", 10).unwrap(); assert_eq!(results.len(), 1); assert_eq!(results[0].id, first.id); @@ -7919,6 +9618,84 @@ mod tests { assert_eq!(access_count, 42); } + #[test] + fn test_portable_merge_import_keeps_composition_members_for_newer_local_memory() { + let source_dir = tempdir().unwrap(); + let target_dir = tempdir().unwrap(); + let source = create_test_storage_at(&source_dir, "source.db"); + let target = create_test_storage_at(&target_dir, "target.db"); + + let node = source + .ingest(IngestInput { + content: "Shared memory with historical composition".to_string(), + node_type: "fact".to_string(), + tags: vec!["protocolgate".to_string()], + ..Default::default() + }) + .unwrap(); + source + .save_composition( + &CompositionEventRecord { + id: "merge-composition-1".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "bounty".to_string(), + query: Some("historical composition".to_string()), + query_hash: Some("sha256:historical".to_string()), + confidence: Some(0.7), + status: Some("resolved".to_string()), + output_preview: Some("Historical composition survives merge".to_string()), + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "merge-composition-1".to_string(), + memory_id: node.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.9), + preview: Some("historical".to_string()), + metadata: serde_json::json!({}), + }], + &[], + ) + .unwrap(); + + let archive = source.export_portable_archive().unwrap(); + target + .import_portable_archive(&archive, PortableImportMode::EmptyOnly) + .unwrap(); + + let local_time = (Utc::now() + Duration::hours(1)).to_rfc3339(); + { + let writer = target.writer.lock().unwrap(); + writer + .execute( + "DELETE FROM composition_members WHERE event_id = ?1", + params!["merge-composition-1"], + ) + .unwrap(); + writer + .execute( + "UPDATE knowledge_nodes SET content = ?1, updated_at = ?2 WHERE id = ?3", + params!["Newer local content", &local_time, &node.id], + ) + .unwrap(); + } + + target + .import_portable_archive(&archive, PortableImportMode::Merge) + .unwrap(); + + let restored = target.get_node(&node.id).unwrap().unwrap(); + assert_eq!(restored.content, "Newer local content"); + let members = target + .get_composition_members("merge-composition-1") + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!(members[0].memory_id, node.id); + } + #[test] fn test_portable_merge_import_applies_delete_tombstones() { let source_dir = tempdir().unwrap(); @@ -7964,22 +9741,71 @@ mod tests { ..Default::default() }) .unwrap(); + source + .save_composition( + &CompositionEventRecord { + id: "portable-purge-composition".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "sync".to_string(), + query: Some("portable purge preview".to_string()), + query_hash: Some("fnv1a64:portable-purge".to_string()), + confidence: Some(0.7), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "portable-purge-composition".to_string(), + memory_id: node.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.8), + preview: Some("Portable purge composition preview leak".to_string()), + metadata: serde_json::json!({}), + }], + &[], + ) + .unwrap(); let archive = source.export_portable_archive().unwrap(); target .import_portable_archive(&archive, PortableImportMode::EmptyOnly) .unwrap(); assert!(target.get_node(&node.id).unwrap().is_some()); + assert_eq!( + target + .get_composition_members("portable-purge-composition") + .unwrap()[0] + .preview + .as_deref(), + Some("Portable purge composition preview leak") + ); source .purge_node(&node.id, Some("sync purge test")) .unwrap(); let purge_archive = source.export_portable_archive().unwrap(); + assert!( + !serde_json::to_string(&purge_archive) + .unwrap() + .contains("Portable purge composition preview leak"), + "source portable archive should not retain purged composition previews" + ); let report = target .import_portable_archive(&purge_archive, PortableImportMode::Merge) .unwrap(); assert!(report.rows_deleted >= 1); assert!(target.get_node(&node.id).unwrap().is_none()); + assert!( + target + .get_composition_members("portable-purge-composition") + .unwrap()[0] + .preview + .is_none(), + "portable purge merge should scrub target composition previews" + ); let writer = target.writer.lock().unwrap(); let tombstone_count: i64 = writer @@ -8348,6 +10174,34 @@ mod tests { .unwrap(); } + storage + .save_composition( + &CompositionEventRecord { + id: "purge-composition-preview-test".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "audit".to_string(), + query: Some("purge preview leak".to_string()), + query_hash: Some("fnv1a64:purge".to_string()), + confidence: Some(0.7), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "purge-composition-preview-test".to_string(), + memory_id: doomed.id.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.9), + preview: Some("Sensitive purge target memory preview leak".to_string()), + metadata: serde_json::json!({}), + }], + &[], + ) + .unwrap(); + let report = storage .purge_node(&doomed.id, Some("user requested hard purge")) .unwrap(); @@ -8387,6 +10241,21 @@ mod tests { .unwrap(); assert_eq!(tombstone_count, 1); + let members = storage + .get_composition_members("purge-composition-preview-test") + .unwrap(); + assert_eq!(members.len(), 1); + assert!( + members[0].preview.is_none(), + "purge should scrub composition member previews for the purged memory" + ); + let archive_json = + serde_json::to_string(&storage.export_portable_archive().unwrap()).unwrap(); + assert!( + !archive_json.contains("Sensitive purge target memory preview leak"), + "portable archive should not retain purged memory content through composition previews" + ); + let has_content_column: i64 = writer .query_row( "SELECT COUNT(*) FROM pragma_table_info('deletion_tombstones') WHERE name = 'content'", @@ -8496,7 +10365,12 @@ mod tests { #[test] fn test_plan_merge_is_preview_only_no_mutation() { let storage = create_test_storage(); - let a = seed_node(&storage, "Fact A about caching", &["perf"], axis_vector(5, 0.02)); + let a = seed_node( + &storage, + "Fact A about caching", + &["perf"], + axis_vector(5, 0.02), + ); let b = seed_node( &storage, "Fact A about caching, expanded", @@ -8524,14 +10398,22 @@ mod tests { assert!(vu_b.is_none() && sb_b.is_none()); // Plan persisted as pending. - assert_eq!(storage.plan_status(&plan.id).unwrap().as_deref(), Some("pending")); + assert_eq!( + storage.plan_status(&plan.id).unwrap().as_deref(), + Some("pending") + ); } #[cfg(all(feature = "embeddings", feature = "vector-search"))] #[test] fn test_apply_then_undo_merge_is_reversible() { let storage = create_test_storage(); - let survivor = seed_node(&storage, "Keep this canonical note", &["x"], axis_vector(7, 0.02)); + let survivor = seed_node( + &storage, + "Keep this canonical note", + &["x"], + axis_vector(7, 0.02), + ); let absorbed = seed_node( &storage, "Extra detail to fold in", @@ -8572,7 +10454,10 @@ mod tests { let surv_after = storage.get_node(&survivor).unwrap().unwrap(); assert_eq!(surv_after.content, "Keep this canonical note"); let (vu2, sb2) = storage.read_bitemporal(&absorbed).unwrap(); - assert!(vu2.is_none() && sb2.is_none(), "invalidation cleared on undo"); + assert!( + vu2.is_none() && sb2.is_none(), + "invalidation cleared on undo" + ); assert!(!storage.superseded_node_ids().unwrap().contains(&absorbed)); // The original op is now marked reverted; double-undo is rejected. @@ -8621,7 +10506,12 @@ mod tests { #[test] fn test_protect_blocks_merge_away() { let storage = create_test_storage(); - let pinned = seed_node(&storage, "Load-bearing fact", &["pin"], axis_vector(11, 0.02)); + let pinned = seed_node( + &storage, + "Load-bearing fact", + &["pin"], + axis_vector(11, 0.02), + ); let other = seed_node( &storage, "Load-bearing fact restated", @@ -8632,7 +10522,11 @@ mod tests { assert!(storage.is_protected(&pinned).unwrap()); // Protected node may not be merged AWAY (survivor=other). - let err = storage.plan_merge(&[other.clone(), pinned.clone()], Some(&other), MergePolicy::default()); + let err = storage.plan_merge( + &[other.clone(), pinned.clone()], + Some(&other), + MergePolicy::default(), + ); assert!(err.is_err(), "merging a protected node away must fail"); // But it CAN be the survivor. @@ -8645,12 +10539,16 @@ mod tests { // Supersede of a protected node is also blocked. assert!( - storage.plan_supersede(&pinned, &other, MergePolicy::default()).is_err(), + storage + .plan_supersede(&pinned, &other, MergePolicy::default()) + .is_err(), "superseding a protected node must fail" ); // merge_candidates flags the protected member. - let cands = storage.merge_candidates(MergePolicy::default(), 20, &[]).unwrap(); + let cands = storage + .merge_candidates(MergePolicy::default(), 20, &[]) + .unwrap(); assert!(cands.iter().all(|c| c.has_protected_member)); } @@ -8664,7 +10562,9 @@ mod tests { let a = seed_node(&storage, "Topic alpha note", &["t"], axis_vector(13, 0.30)); let b = seed_node(&storage, "Topic alpha aside", &["t"], axis_vector(13, 0.60)); - let plan = storage.plan_merge(&[a, b], None, storage.get_merge_policy().unwrap()).unwrap(); + let plan = storage + .plan_merge(&[a, b], None, storage.get_merge_policy().unwrap()) + .unwrap(); assert_ne!(plan.classification, MatchClass::Match); // Without confirm => rejected. diff --git a/crates/vestige-mcp/README.md b/crates/vestige-mcp/README.md index 92f53d8..2547e42 100644 --- a/crates/vestige-mcp/README.md +++ b/crates/vestige-mcp/README.md @@ -61,7 +61,7 @@ The server exposes the current unified MCP tools from - `search`, `smart_ingest`, `memory`, `codebase`, `intention` - `deep_reference`, `cross_reference`, `contradictions` - `dream`, `explore_connections`, `predict` -- `memory_health`, `memory_graph`, `system_status` +- `memory_health`, `memory_graph`, `composed_graph`, `system_status` - `importance_score`, `find_duplicates` - `consolidate`, `memory_timeline`, `memory_changelog` - `backup`, `export`, `restore`, `gc`, `suppress` diff --git a/crates/vestige-mcp/src/server.rs b/crates/vestige-mcp/src/server.rs index 2cb1e5f..7682441 100644 --- a/crates/vestige-mcp/src/server.rs +++ b/crates/vestige-mcp/src/server.rs @@ -443,6 +443,12 @@ impl McpServer { input_schema: tools::graph::schema(), ..Default::default() }, + ToolDescription { + name: "composed_graph".to_string(), + description: Some("ComposedGraph memory topology. Reads durable composition events, members, and outcome labels; returns recent/already-composed lanes, neighbors, never-composed pairs, bounty-mode lanes, and lets users label outcomes such as helpful, submitted, accepted, rejected, duplicate_risk, needs_poc, or dead_end.".to_string()), + input_schema: tools::composed_graph::schema(), + ..Default::default() + }, // ================================================================ // DEEP REFERENCE (v2.0.4+) — replaces cross_reference // ================================================================ @@ -959,7 +965,8 @@ impl McpServer { // TEMPORAL TOOLS (v1.2+) // ================================================================ "memory_timeline" => { - tools::timeline::execute(&self.storage, &self.output_config, request.arguments).await + tools::timeline::execute(&self.storage, &self.output_config, request.arguments) + .await } "memory_changelog" => tools::changelog::execute(&self.storage, request.arguments).await, @@ -1032,6 +1039,9 @@ impl McpServer { // ================================================================ "memory_health" => tools::health::execute(&self.storage, request.arguments).await, "memory_graph" => tools::graph::execute(&self.storage, request.arguments).await, + "composed_graph" => { + tools::composed_graph::execute(&self.storage, request.arguments).await + } "deep_reference" | "cross_reference" => { tools::cross_reference::execute(&self.storage, &self.cognitive, request.arguments) .await @@ -1796,10 +1806,10 @@ mod tests { let result = response.result.unwrap(); let tools = result["tools"].as_array().unwrap(); - // v2.1.25: 32 tools (25 from v2.1.21 + 7 Phase 3 merge/supersede tools: + // 33 tools: 25 from v2.1.21 + 7 Phase 3 merge/supersede tools: // merge_candidates, plan_merge, plan_supersede, apply_plan, merge_undo, - // protect, merge_policy) - assert_eq!(tools.len(), 32, "Expected exactly 32 tools in v2.1.25"); + // protect, merge_policy, composed_graph) + assert_eq!(tools.len(), 33, "Expected exactly 33 tools"); let tool_names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); @@ -1874,6 +1884,7 @@ mod tests { // Autonomic tools (v1.9) assert!(tool_names.contains(&"memory_health")); assert!(tool_names.contains(&"memory_graph")); + assert!(tool_names.contains(&"composed_graph")); // Deep reference + cross_reference alias (v2.0.4) assert!(tool_names.contains(&"deep_reference")); diff --git a/crates/vestige-mcp/src/tools/composed_graph.rs b/crates/vestige-mcp/src/tools/composed_graph.rs new file mode 100644 index 0000000..ee69d93 --- /dev/null +++ b/crates/vestige-mcp/src/tools/composed_graph.rs @@ -0,0 +1,906 @@ +//! composed_graph tool — durable composition history and bounty-mode lane queue. + +use chrono::Utc; +use serde::Deserialize; +use serde_json::Value; +use std::sync::Arc; +use uuid::Uuid; +use vestige_core::{CompositionOutcomeRecord, Storage}; + +const OUTCOME_TYPES: &[&str] = &[ + "helpful", + "dead_end", + "submitted", + "accepted", + "rejected", + "duplicate_risk", + "needs_poc", + "bad_severity", + "user_promoted", + "user_demoted", + "closed_by_scope", + "closed_by_duplicate", + "closed_by_false_assumption", + "closed_by_user", + "expired_lane", +]; + +pub fn schema() -> Value { + serde_json::json!({ + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["recent", "get", "memory", "neighbors", "never_composed", "bounty_mode", "label"], + "description": "ComposedGraph action to run." + }, + "event_id": { + "type": "string", + "description": "Composition event id for get/label actions." + }, + "memory_id": { + "type": "string", + "description": "Memory id for memory/neighbors actions." + }, + "limit": { + "type": "integer", + "description": "Maximum rows to return (default 10, max 100).", + "default": 10, + "minimum": 1, + "maximum": 100 + }, + "tags": { + "type": "array", + "items": { "type": "string" }, + "description": "Optional tag filter for never_composed and bounty_mode." + }, + "outcome_type": { + "type": "string", + "enum": ["helpful", "dead_end", "submitted", "accepted", "rejected", "duplicate_risk", "needs_poc", "bad_severity", "user_promoted", "user_demoted", "closed_by_scope", "closed_by_duplicate", "closed_by_false_assumption", "closed_by_user", "expired_lane"], + "description": "Outcome label for label action." + }, + "notes": { + "type": "string", + "description": "Optional outcome notes." + }, + "label_source": { + "type": "string", + "description": "Where the outcome label came from (default: user)." + }, + "confidence_delta": { + "type": "number", + "description": "Optional confidence adjustment for this outcome." + } + }, + "required": ["action"] + }) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +struct ComposedGraphArgs { + action: String, + event_id: Option, + memory_id: Option, + limit: Option, + tags: Option>, + outcome_type: Option, + notes: Option, + label_source: Option, + confidence_delta: Option, +} + +pub async fn execute(storage: &Arc, args: Option) -> Result { + let args: ComposedGraphArgs = match args { + Some(value) => { + serde_json::from_value(value).map_err(|e| format!("Invalid arguments: {}", e))? + } + None => return Err("Missing arguments".to_string()), + }; + let limit = args.limit.unwrap_or(10).clamp(1, 100); + + match args.action.as_str() { + "recent" => recent(storage, limit), + "get" => { + let event_id = args + .event_id + .as_deref() + .ok_or_else(|| "event_id is required for get".to_string())?; + get(storage, event_id) + } + "memory" => { + let memory_id = args + .memory_id + .as_deref() + .ok_or_else(|| "memory_id is required for memory".to_string())?; + memory(storage, memory_id, limit) + } + "neighbors" => { + let memory_id = args + .memory_id + .as_deref() + .ok_or_else(|| "memory_id is required for neighbors".to_string())?; + neighbors(storage, memory_id, limit) + } + "never_composed" => never_composed(storage, limit, args.tags.as_deref()), + "bounty_mode" => bounty_mode(storage, limit, args.tags.as_deref()), + "label" => label(storage, &args), + other => Err(format!("Unknown composed_graph action: {}", other)), + } +} + +fn recent(storage: &Storage, limit: i32) -> Result { + let events = storage + .get_recent_composition_events(limit) + .map_err(|e| e.to_string())?; + Ok(serde_json::json!({ + "action": "recent", + "events": events, + })) +} + +fn get(storage: &Storage, event_id: &str) -> Result { + let event = storage + .get_composition_event(event_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("composition event not found: {}", event_id))?; + let members = storage + .get_composition_members(event_id) + .map_err(|e| e.to_string())?; + let outcomes = storage + .get_composition_outcomes(event_id) + .map_err(|e| e.to_string())?; + Ok(serde_json::json!({ + "action": "get", + "event": event, + "members": members, + "outcomes": outcomes, + })) +} + +fn memory(storage: &Storage, memory_id: &str, limit: i32) -> Result { + let events = storage + .get_compositions_for_memory(memory_id, limit) + .map_err(|e| e.to_string())?; + Ok(serde_json::json!({ + "action": "memory", + "memoryId": memory_id, + "events": events, + })) +} + +fn neighbors(storage: &Storage, memory_id: &str, limit: i32) -> Result { + let neighbors = storage + .get_composition_neighbors(memory_id, limit) + .map_err(|e| e.to_string())?; + Ok(serde_json::json!({ + "action": "neighbors", + "memoryId": memory_id, + "neighbors": neighbors, + })) +} + +fn never_composed(storage: &Storage, limit: i32, tags: Option<&[String]>) -> Result { + let candidates = storage + .get_never_composed_candidates(limit, tags) + .map_err(|e| e.to_string())?; + Ok(serde_json::json!({ + "action": "never_composed", + "candidates": candidates, + })) +} + +fn bounty_mode(storage: &Storage, limit: i32, tags: Option<&[String]>) -> Result { + const PAGE_SIZE: i32 = 100; + const MAX_SCAN_EVENTS: i32 = 1_000; + + let mut offset = 0; + let mut scanned = 0; + let mut already_composed = Vec::new(); + let mut closed_doors = Vec::new(); + let mut duplicate_risk_lanes = Vec::new(); + let mut needs_poc_lanes = Vec::new(); + + loop { + let events = storage + .get_recent_composition_events_page(PAGE_SIZE, offset) + .map_err(|e| e.to_string())?; + if events.is_empty() { + break; + } + scanned += events.len() as i32; + + for event in events { + let outcomes = storage + .get_composition_outcomes(&event.id) + .map_err(|e| e.to_string())?; + let members = storage + .get_composition_members(&event.id) + .map_err(|e| e.to_string())?; + if !composition_matches_tags(storage, &event, &members, tags)? { + continue; + } + let item = serde_json::json!({ + "event": event, + "members": members, + "outcomes": outcomes, + }); + let outcome_types = item["outcomes"] + .as_array() + .map(|values| { + values + .iter() + .filter_map(|value| value.get("outcomeType").and_then(|v| v.as_str())) + .collect::>() + }) + .unwrap_or_default(); + + if outcome_types.iter().any(|kind| { + matches!( + *kind, + "dead_end" + | "rejected" + | "bad_severity" + | "closed_by_scope" + | "closed_by_duplicate" + | "closed_by_false_assumption" + | "closed_by_user" + | "expired_lane" + ) + }) { + push_limited(&mut closed_doors, item.clone(), limit); + } + if outcome_types + .iter() + .any(|kind| matches!(*kind, "duplicate_risk" | "closed_by_duplicate")) + { + push_limited(&mut duplicate_risk_lanes, item.clone(), limit); + } + if outcome_types.iter().any(|kind| *kind == "needs_poc") { + push_limited(&mut needs_poc_lanes, item.clone(), limit); + } + if already_composed.len() < limit as usize { + already_composed.push(item); + } + if bounty_mode_lanes_full( + limit, + &already_composed, + &closed_doors, + &duplicate_risk_lanes, + &needs_poc_lanes, + ) { + break; + } + } + + if bounty_mode_lanes_full( + limit, + &already_composed, + &closed_doors, + &duplicate_risk_lanes, + &needs_poc_lanes, + ) || scanned >= MAX_SCAN_EVENTS + { + break; + } + offset += PAGE_SIZE; + } + + let never = storage + .get_never_composed_candidates(limit, tags) + .map_err(|e| e.to_string())?; + let top_weird_combinations = never.iter().take(3).cloned().collect::>(); + + Ok(serde_json::json!({ + "action": "bounty_mode", + "alreadyComposedLanes": already_composed, + "neverComposedLanes": never, + "closedDoors": closed_doors, + "duplicateRiskLanes": duplicate_risk_lanes, + "needsPocLanes": needs_poc_lanes, + "topWeirdCombinations": top_weird_combinations, + "guardrails": [ + "never-composed lane is not a finding", + "composition score is not severity", + "submit/reportable still needs source refs, scope fit, and PoC evidence" + ] + })) +} + +fn push_limited(items: &mut Vec, item: Value, limit: i32) { + if items.len() < limit as usize { + items.push(item); + } +} + +fn bounty_mode_lanes_full( + limit: i32, + already_composed: &[Value], + closed_doors: &[Value], + duplicate_risk_lanes: &[Value], + needs_poc_lanes: &[Value], +) -> bool { + let limit = limit as usize; + already_composed.len() >= limit + && closed_doors.len() >= limit + && duplicate_risk_lanes.len() >= limit + && needs_poc_lanes.len() >= limit +} + +fn composition_matches_tags( + storage: &Storage, + event: &vestige_core::CompositionEventRecord, + members: &[vestige_core::CompositionMemberRecord], + tags: Option<&[String]>, +) -> Result { + let Some(tags) = tags else { + return Ok(true); + }; + if tags.is_empty() { + return Ok(true); + } + + if json_value_has_tag(&event.metadata, tags) { + return Ok(true); + } + + for member in members { + if json_value_has_tag(&member.metadata, tags) { + return Ok(true); + } + if let Some(node) = storage + .get_node(&member.memory_id) + .map_err(|e| e.to_string())? + && node.tags.iter().any(|tag| tag_matches_filter(tag, tags)) + { + return Ok(true); + } + } + + Ok(false) +} + +fn json_value_has_tag(value: &Value, tags: &[String]) -> bool { + value + .get("tags") + .and_then(|tags_value| tags_value.as_array()) + .is_some_and(|values| { + values.iter().any(|value| { + value + .as_str() + .is_some_and(|tag| tag_matches_filter(tag, tags)) + }) + }) +} + +fn tag_matches_filter(tag: &str, filters: &[String]) -> bool { + filters + .iter() + .any(|wanted| tag == wanted || tag.starts_with(&format!("{wanted}:"))) +} + +fn label(storage: &Storage, args: &ComposedGraphArgs) -> Result { + let event_id = args + .event_id + .as_deref() + .ok_or_else(|| "event_id is required for label".to_string())?; + let outcome_type = args + .outcome_type + .as_deref() + .ok_or_else(|| "outcome_type is required for label".to_string())?; + if !OUTCOME_TYPES.contains(&outcome_type) { + return Err(format!("unsupported outcome_type: {}", outcome_type)); + } + if storage + .get_composition_event(event_id) + .map_err(|e| e.to_string())? + .is_none() + { + return Err(format!("composition event not found: {}", event_id)); + } + + let outcome = CompositionOutcomeRecord { + id: Uuid::new_v4().to_string(), + event_id: event_id.to_string(), + outcome_type: outcome_type.to_string(), + labeled_at: Utc::now(), + label_source: args + .label_source + .clone() + .unwrap_or_else(|| "user".to_string()), + confidence_delta: args.confidence_delta, + notes: args.notes.clone(), + metadata: serde_json::json!({}), + }; + storage + .record_composition_outcome(&outcome) + .map_err(|e| e.to_string())?; + + Ok(serde_json::json!({ + "action": "label", + "eventId": event_id, + "outcome": outcome, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use vestige_core::{ + CompositionEventRecord, CompositionMemberRecord, CompositionOutcomeRecord, IngestInput, + }; + + fn test_storage() -> (Arc, TempDir) { + let dir = TempDir::new().unwrap(); + let storage = Storage::new(Some(dir.path().join("test.db"))).unwrap(); + (Arc::new(storage), dir) + } + + fn ingest(storage: &Storage, content: &str, tags: &[&str]) -> String { + storage + .ingest(IngestInput { + content: content.to_string(), + node_type: "fact".to_string(), + tags: tags.iter().map(|tag| tag.to_string()).collect(), + ..Default::default() + }) + .unwrap() + .id + } + + #[tokio::test] + async fn test_composed_graph_get_label_and_bounty_mode() { + let (storage, _dir) = test_storage(); + let first = ingest( + &storage, + "Oracle drift bounty lane", + &["protocolgate", "boundary-oracle", "settlement"], + ); + let second = ingest( + &storage, + "Withdrawal queue bounty lane", + &["protocolgate", "boundary-queue", "settlement"], + ); + let third = ingest( + &storage, + "Keeper role bounty lane", + &["protocolgate", "boundary-role", "settlement"], + ); + + let event = CompositionEventRecord { + id: "composed-graph-test".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "bounty".to_string(), + query: Some("oracle withdrawal".to_string()), + query_hash: Some("test".to_string()), + confidence: Some(0.8), + status: Some("resolved".to_string()), + output_preview: Some("compose oracle and withdrawal queue".to_string()), + metadata: serde_json::json!({}), + }; + storage + .save_composition( + &event, + &[ + CompositionMemberRecord { + event_id: event.id.clone(), + memory_id: first.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.9), + preview: None, + metadata: serde_json::json!({}), + }, + CompositionMemberRecord { + event_id: event.id.clone(), + memory_id: second.clone(), + role: "supporting".to_string(), + rank: 1, + trust: Some(0.7), + score: Some(0.8), + preview: None, + metadata: serde_json::json!({}), + }, + ], + &[], + ) + .unwrap(); + + let unrelated = ingest(&storage, "Personal planning lane", &["personal"]); + storage + .save_composition( + &CompositionEventRecord { + id: "unrelated-composed-graph-test".to_string(), + created_at: Utc::now() + chrono::Duration::seconds(10), + tool: "deep_reference".to_string(), + mode: "planning".to_string(), + query: Some("personal planning".to_string()), + query_hash: Some("unrelated".to_string()), + confidence: Some(0.4), + status: Some("resolved".to_string()), + output_preview: Some("unrelated composition".to_string()), + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "unrelated-composed-graph-test".to_string(), + memory_id: unrelated, + role: "primary".to_string(), + rank: 0, + trust: Some(0.4), + score: Some(0.2), + preview: None, + metadata: serde_json::json!({}), + }], + &[CompositionOutcomeRecord { + id: "unrelated-composed-graph-outcome".to_string(), + event_id: "unrelated-composed-graph-test".to_string(), + outcome_type: "needs_poc".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: None, + notes: None, + metadata: serde_json::json!({}), + }], + ) + .unwrap(); + + let get_result = execute( + &storage, + Some(serde_json::json!({ + "action": "get", + "event_id": event.id + })), + ) + .await + .unwrap(); + assert_eq!(get_result["members"].as_array().unwrap().len(), 2); + + let label_result = execute( + &storage, + Some(serde_json::json!({ + "action": "label", + "event_id": "composed-graph-test", + "outcome_type": "submitted", + "notes": "submitted in test" + })), + ) + .await + .unwrap(); + assert_eq!( + label_result["outcome"]["outcomeType"].as_str(), + Some("submitted") + ); + let closed_label_result = execute( + &storage, + Some(serde_json::json!({ + "action": "label", + "event_id": "composed-graph-test", + "outcome_type": "closed_by_scope", + "notes": "closed in test" + })), + ) + .await + .unwrap(); + assert_eq!( + closed_label_result["outcome"]["outcomeType"].as_str(), + Some("closed_by_scope") + ); + let duplicate_label_result = execute( + &storage, + Some(serde_json::json!({ + "action": "label", + "event_id": "composed-graph-test", + "outcome_type": "closed_by_duplicate", + "notes": "duplicate family in test" + })), + ) + .await + .unwrap(); + assert_eq!( + duplicate_label_result["outcome"]["outcomeType"].as_str(), + Some("closed_by_duplicate") + ); + + let bounty = execute( + &storage, + Some(serde_json::json!({ + "action": "bounty_mode", + "tags": ["protocolgate"], + "limit": 1 + })), + ) + .await + .unwrap(); + let already = bounty["alreadyComposedLanes"].as_array().unwrap(); + assert_eq!(already.len(), 1); + assert!( + already[0]["event"]["id"].as_str() == Some("composed-graph-test"), + "tag-scoped bounty_mode should skip newer unrelated events before truncating" + ); + assert_eq!(bounty["closedDoors"].as_array().unwrap().len(), 1); + assert_eq!(bounty["duplicateRiskLanes"].as_array().unwrap().len(), 1); + assert!(bounty["needsPocLanes"].as_array().unwrap().is_empty()); + assert!( + bounty["neverComposedLanes"] + .as_array() + .unwrap() + .iter() + .any(|candidate| { + let first_id = candidate["firstId"].as_str().unwrap_or_default(); + let second_id = candidate["secondId"].as_str().unwrap_or_default(); + [first_id, second_id].contains(&third.as_str()) + }) + ); + } + + #[tokio::test] + async fn test_bounty_mode_paginates_tag_filter_and_matches_namespaced_tags() { + let (storage, _dir) = test_storage(); + let tagged = ingest( + &storage, + "Older tagged composition lane", + &["project:vestige", "composition"], + ); + let unrelated = ingest(&storage, "Newer unrelated lane", &["unrelated"]); + let base_time = Utc::now(); + + storage + .save_composition( + &CompositionEventRecord { + id: "older-tagged-composition".to_string(), + created_at: base_time, + tool: "deep_reference".to_string(), + mode: "research".to_string(), + query: Some("older tagged lane".to_string()), + query_hash: Some("fnv1a64:older".to_string()), + confidence: Some(0.8), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "older-tagged-composition".to_string(), + memory_id: tagged, + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.9), + preview: None, + metadata: serde_json::json!({}), + }], + &[], + ) + .unwrap(); + + for idx in 0..101 { + let event_id = format!("newer-unrelated-composition-{idx}"); + storage + .save_composition( + &CompositionEventRecord { + id: event_id.clone(), + created_at: base_time + chrono::Duration::seconds(i64::from(idx + 1)), + tool: "deep_reference".to_string(), + mode: "planning".to_string(), + query: Some(format!("newer unrelated lane {idx}")), + query_hash: Some(format!("fnv1a64:newer-{idx}")), + confidence: Some(0.3), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id, + memory_id: unrelated.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.3), + score: Some(0.2), + preview: None, + metadata: serde_json::json!({}), + }], + &[], + ) + .unwrap(); + } + + let bounty = execute( + &storage, + Some(serde_json::json!({ + "action": "bounty_mode", + "tags": ["project"], + "limit": 1 + })), + ) + .await + .unwrap(); + let already = bounty["alreadyComposedLanes"].as_array().unwrap(); + assert_eq!(already.len(), 1); + assert_eq!( + already[0]["event"]["id"].as_str(), + Some("older-tagged-composition"), + "tag-filtered bounty_mode should page past newer unrelated events and match namespaced tags" + ); + } + + #[tokio::test] + async fn test_bounty_mode_uses_member_tag_snapshot_after_purge() { + let (storage, _dir) = test_storage(); + let tagged = ingest( + &storage, + "Tagged member that will be purged", + &["project:vestige", "composition"], + ); + + storage + .save_composition( + &CompositionEventRecord { + id: "purged-tagged-member-composition".to_string(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "research".to_string(), + query: Some("purged tagged lane".to_string()), + query_hash: Some("fnv1a64:purged".to_string()), + confidence: Some(0.6), + status: Some("closed".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "purged-tagged-member-composition".to_string(), + memory_id: tagged.clone(), + role: "primary".to_string(), + rank: 0, + trust: Some(0.7), + score: Some(0.8), + preview: Some("Tagged member that will be purged".to_string()), + metadata: serde_json::json!({}), + }], + &[CompositionOutcomeRecord { + id: "purged-tagged-member-outcome".to_string(), + event_id: "purged-tagged-member-composition".to_string(), + outcome_type: "closed_by_scope".to_string(), + labeled_at: Utc::now(), + label_source: "test".to_string(), + confidence_delta: Some(-0.2), + notes: None, + metadata: serde_json::json!({}), + }], + ) + .unwrap(); + + storage + .purge_node(&tagged, Some("test purge")) + .expect("purge should succeed"); + + let get_result = execute( + &storage, + Some(serde_json::json!({ + "action": "get", + "event_id": "purged-tagged-member-composition" + })), + ) + .await + .unwrap(); + assert!( + get_result["members"][0].get("preview").is_none() + || get_result["members"][0]["preview"].is_null(), + "purge should scrub member preview from composed_graph get" + ); + + let bounty = execute( + &storage, + Some(serde_json::json!({ + "action": "bounty_mode", + "tags": ["project"], + "limit": 1 + })), + ) + .await + .unwrap(); + let already = bounty["alreadyComposedLanes"].as_array().unwrap(); + assert_eq!(already.len(), 1); + assert_eq!( + already[0]["event"]["id"].as_str(), + Some("purged-tagged-member-composition"), + "tag-filtered bounty_mode should use composition member tag snapshots after source memory purge" + ); + assert_eq!(bounty["closedDoors"].as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn test_bounty_mode_guardrail_buckets_are_not_truncated_by_already_limit() { + let (storage, _dir) = test_storage(); + let neutral = ingest(&storage, "Neutral release lane", &["project:vestige"]); + let closed = ingest(&storage, "Closed release lane", &["project:vestige"]); + let base_time = Utc::now(); + + storage + .save_composition( + &CompositionEventRecord { + id: "older-closed-lane".to_string(), + created_at: base_time, + tool: "deep_reference".to_string(), + mode: "release".to_string(), + query: Some("older closed lane".to_string()), + query_hash: Some("fnv1a64:older-closed".to_string()), + confidence: Some(0.3), + status: Some("closed".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "older-closed-lane".to_string(), + memory_id: closed, + role: "primary".to_string(), + rank: 0, + trust: Some(0.5), + score: Some(0.4), + preview: None, + metadata: serde_json::json!({}), + }], + &[CompositionOutcomeRecord { + id: "older-closed-outcome".to_string(), + event_id: "older-closed-lane".to_string(), + outcome_type: "closed_by_false_assumption".to_string(), + labeled_at: base_time, + label_source: "test".to_string(), + confidence_delta: Some(-0.3), + notes: None, + metadata: serde_json::json!({}), + }], + ) + .unwrap(); + + storage + .save_composition( + &CompositionEventRecord { + id: "newer-neutral-lane".to_string(), + created_at: base_time + chrono::Duration::seconds(1), + tool: "deep_reference".to_string(), + mode: "release".to_string(), + query: Some("newer neutral lane".to_string()), + query_hash: Some("fnv1a64:newer-neutral".to_string()), + confidence: Some(0.7), + status: Some("resolved".to_string()), + output_preview: None, + metadata: serde_json::json!({}), + }, + &[CompositionMemberRecord { + event_id: "newer-neutral-lane".to_string(), + memory_id: neutral, + role: "primary".to_string(), + rank: 0, + trust: Some(0.8), + score: Some(0.8), + preview: None, + metadata: serde_json::json!({}), + }], + &[], + ) + .unwrap(); + + let bounty = execute( + &storage, + Some(serde_json::json!({ + "action": "bounty_mode", + "tags": ["project"], + "limit": 1 + })), + ) + .await + .unwrap(); + + assert_eq!( + bounty["alreadyComposedLanes"][0]["event"]["id"].as_str(), + Some("newer-neutral-lane") + ); + assert_eq!( + bounty["closedDoors"][0]["event"]["id"].as_str(), + Some("older-closed-lane"), + "guardrail buckets should keep scanning after alreadyComposedLanes reaches limit" + ); + } +} diff --git a/crates/vestige-mcp/src/tools/cross_reference.rs b/crates/vestige-mcp/src/tools/cross_reference.rs index e1a9128..e48b4eb 100644 --- a/crates/vestige-mcp/src/tools/cross_reference.rs +++ b/crates/vestige-mcp/src/tools/cross_reference.rs @@ -20,9 +20,10 @@ use serde::Deserialize; use serde_json::Value; use std::sync::Arc; use tokio::sync::Mutex; +use uuid::Uuid; use crate::cognitive::CognitiveEngine; -use vestige_core::Storage; +use vestige_core::{CompositionEventRecord, CompositionMemberRecord, Storage}; /// Input schema for deep_reference / cross_reference tool pub fn schema() -> Value { @@ -509,6 +510,7 @@ pub async fn execute( "confidence": 0.0, "guidance": "No memories found. Use smart_ingest to add memories.", "memoriesAnalyzed": 0, + "compositionWriteStatus": "skipped_empty", })); } @@ -820,6 +822,7 @@ pub async fn execute( "id": s.id, "preview": s.content.chars().take(200).collect::(), "trust": (s.trust * 100.0).round() / 100.0, + "relevanceScore": ((composite(s) * 100.0).round() / 100.0), "date": s.updated_at.to_rfc3339(), "role": if i == 0 { "primary" } else { "supporting" }, }) @@ -925,9 +928,163 @@ pub async fn execute( response["related_insights"] = serde_json::json!(related_insights); } + match persist_deep_reference_composition(storage, &args.query, &intent, &response) { + Ok(Some(event_id)) => { + response["composition_event_id"] = serde_json::json!(event_id); + response["compositionWriteStatus"] = serde_json::json!("persisted"); + } + Ok(None) => { + response["compositionWriteStatus"] = serde_json::json!("skipped_empty"); + } + Err(err) => { + tracing::warn!( + "Failed to persist deep_reference composition event: {}", + err + ); + response["compositionWriteStatus"] = serde_json::json!("failed"); + } + } + Ok(response) } +fn persist_deep_reference_composition( + storage: &Arc, + query: &str, + intent: &QueryIntent, + response: &Value, +) -> Result, String> { + let event_id = Uuid::new_v4().to_string(); + let event = CompositionEventRecord { + id: event_id.clone(), + created_at: Utc::now(), + tool: "deep_reference".to_string(), + mode: "deep_reference".to_string(), + query: Some(query.to_string()), + query_hash: Some(query_hash(query)), + confidence: response.get("confidence").and_then(|v| v.as_f64()), + status: response + .get("status") + .and_then(|v| v.as_str()) + .map(ToOwned::to_owned), + output_preview: response + .get("guidance") + .and_then(|v| v.as_str()) + .map(|value| preview_text(value, 280)), + metadata: serde_json::json!({ + "intent": format!("{:?}", intent), + "memoriesAnalyzed": response.get("memoriesAnalyzed").and_then(|v| v.as_u64()).unwrap_or(0), + "activationExpanded": response.get("activationExpanded").and_then(|v| v.as_u64()).unwrap_or(0), + "reasoningPreview": response.get("reasoning").and_then(|v| v.as_str()).map(|value| preview_text(value, 600)), + }), + }; + + let mut members = Vec::new(); + if let Some(evidence) = response.get("evidence").and_then(|v| v.as_array()) { + for (idx, item) in evidence.iter().enumerate() { + let Some(memory_id) = item.get("id").and_then(|v| v.as_str()) else { + continue; + }; + let role = item + .get("role") + .and_then(|v| v.as_str()) + .unwrap_or(if idx == 0 { "primary" } else { "supporting" }); + members.push(CompositionMemberRecord { + event_id: event_id.clone(), + memory_id: memory_id.to_string(), + role: role.to_string(), + rank: idx as i32, + trust: item.get("trust").and_then(|v| v.as_f64()), + score: item + .get("relevanceScore") + .or_else(|| item.get("relevance_score")) + .and_then(|v| v.as_f64()), + preview: None, + metadata: serde_json::json!({ + "roleSource": "deep_reference_evidence", + "evidenceRank": idx, + "date": item.get("date").and_then(|v| v.as_str()), + }), + }); + } + } + + if let Some(contradictions) = response.get("contradictions").and_then(|v| v.as_array()) { + for (idx, contradiction) in contradictions.iter().enumerate() { + for side in ["stronger", "weaker"] { + let Some(item) = contradiction.get(side) else { + continue; + }; + let Some(memory_id) = item.get("id").and_then(|v| v.as_str()) else { + continue; + }; + members.push(CompositionMemberRecord { + event_id: event_id.clone(), + memory_id: memory_id.to_string(), + role: "contradicting".to_string(), + rank: idx as i32, + trust: item.get("trust").and_then(|v| v.as_f64()), + score: contradiction.get("topic_overlap").and_then(|v| v.as_f64()), + preview: None, + metadata: serde_json::json!({ + "roleSource": "deep_reference_contradiction", + "side": side, + "date": item.get("date").and_then(|v| v.as_str()), + }), + }); + } + } + } + + if let Some(superseded) = response.get("superseded").and_then(|v| v.as_array()) { + for (idx, item) in superseded.iter().enumerate() { + let Some(memory_id) = item.get("id").and_then(|v| v.as_str()) else { + continue; + }; + members.push(CompositionMemberRecord { + event_id: event_id.clone(), + memory_id: memory_id.to_string(), + role: "superseded".to_string(), + rank: idx as i32, + trust: item.get("trust").and_then(|v| v.as_f64()), + score: None, + preview: None, + metadata: serde_json::json!({ + "roleSource": "deep_reference_superseded", + "superseded_by": item.get("superseded_by").and_then(|v| v.as_str()), + "date": item.get("date").and_then(|v| v.as_str()), + }), + }); + } + } + + if members.is_empty() { + return Ok(None); + } + + storage + .save_composition(&event, &members, &[]) + .map_err(|e| e.to_string())?; + Ok(Some(event_id)) +} + +fn query_hash(query: &str) -> String { + let mut hash = 0xcbf29ce484222325u64; + for byte in query.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + format!("fnv1a64:{hash:016x}") +} + +fn preview_text(value: &str, max: usize) -> String { + let collapsed = value.replace('\n', " "); + if collapsed.len() <= max { + return collapsed; + } + format!("{}...", &collapsed[..collapsed.floor_char_boundary(max)]) +} + // ============================================================================ // TESTS // ============================================================================ @@ -1010,6 +1167,99 @@ mod tests { ); } + #[tokio::test] + async fn test_deep_reference_persists_composition_event() { + let (storage, _dir) = test_storage().await; + + let primary_id = ingest_one( + &storage, + "ProtocolGate control-plane composition tracks global invariant local gate bypasses.", + &["protocolgate", "boundary-scope"], + ) + .await; + let supporting_id = ingest_one( + &storage, + "ProtocolGate global invariant local gate research used Aave account-global health factor and route-local validation.", + &["protocolgate", "boundary-scope"], + ) + .await; + + let result = execute( + &storage, + &test_cognitive(), + Some(serde_json::json!({ + "query": "ProtocolGate global invariant local gate", + "depth": 10 + })), + ) + .await + .expect("execute should succeed"); + + let event_id = result["composition_event_id"] + .as_str() + .expect("deep_reference should return persisted event id"); + assert_eq!(result["compositionWriteStatus"].as_str(), Some("persisted")); + + let event = storage + .get_composition_event(event_id) + .unwrap() + .expect("composition event should be stored"); + assert_eq!(event.tool, "deep_reference"); + assert_eq!( + event.query.as_deref(), + Some("ProtocolGate global invariant local gate") + ); + + let members = storage.get_composition_members(event_id).unwrap(); + assert!(members.iter().any(|member| member.memory_id == primary_id)); + assert!( + members + .iter() + .any(|member| member.memory_id == supporting_id) + ); + assert!(members.iter().any(|member| member.role == "primary")); + assert!( + members.iter().any(|member| { + member.memory_id == primary_id + && member.score.is_some() + && member.metadata["roleSource"] == "deep_reference_evidence" + }), + "persisted members should retain relevance score and role source" + ); + } + + #[tokio::test] + async fn test_deep_reference_skips_empty_composition_event() { + let (storage, _dir) = test_storage().await; + + let result = execute( + &storage, + &test_cognitive(), + Some(serde_json::json!({ + "query": "no memories exist for this query", + "depth": 10 + })), + ) + .await + .expect("execute should succeed"); + + assert_eq!( + result["compositionWriteStatus"].as_str(), + Some("skipped_empty") + ); + assert!( + result.get("composition_event_id").is_none(), + "empty evidence should not create a composition event" + ); + assert!( + storage + .get_recent_composition_events(10) + .unwrap() + .is_empty(), + "ledger should stay empty when no memories participated" + ); + } + // ======================================================================== // Confidence sanity: must vary with query relevance. // ======================================================================== diff --git a/crates/vestige-mcp/src/tools/mod.rs b/crates/vestige-mcp/src/tools/mod.rs index a2c3e24..078fab6 100644 --- a/crates/vestige-mcp/src/tools/mod.rs +++ b/crates/vestige-mcp/src/tools/mod.rs @@ -41,6 +41,7 @@ pub mod graph; pub mod health; // v2.1: Cross-reference (connect the dots) +pub mod composed_graph; pub mod contradictions; pub mod cross_reference; diff --git a/docs/COMPOSED_GRAPH.md b/docs/COMPOSED_GRAPH.md new file mode 100644 index 0000000..e0748be --- /dev/null +++ b/docs/COMPOSED_GRAPH.md @@ -0,0 +1,159 @@ +# ComposedGraph + +ComposedGraph records memory combinations as durable reasoning events. + +Most memory systems store facts, entities, or relationships. ComposedGraph stores a +different object: which memories were used together, why they were used, and what +happened afterward. + +## Model + +`composition_events` stores the reasoning envelope: + +- tool and mode, such as `deep_reference` or `bounty` +- query and query hash +- confidence, status, and output preview +- metadata for intent, analyzed memory count, activation expansion, and reasoning preview + +`composition_members` stores the participating memories: + +- memory id +- role, such as `primary`, `supporting`, `contradicting`, or `superseded` +- rank, trust, relevance score, preview, and metadata + +`composition_outcomes` stores later labels: + +- `helpful` +- `dead_end` +- `submitted` +- `accepted` +- `rejected` +- `duplicate_risk` +- `needs_poc` +- `bad_severity` +- `user_promoted` +- `user_demoted` +- `closed_by_scope` +- `closed_by_duplicate` +- `closed_by_false_assumption` +- `closed_by_user` +- `expired_lane` + +Member memory ids are intentionally historical references, not foreign keys into +`knowledge_nodes`. Purging or superseding a memory should not erase the fact that +it once participated in a reasoning path. + +## MCP Tool + +Use `composed_graph` for read/write access to the composition ledger. + +```json +{ "action": "recent", "limit": 10 } +``` + +```json +{ "action": "get", "event_id": "" } +``` + +```json +{ "action": "memory", "memory_id": "", "limit": 10 } +``` + +```json +{ "action": "neighbors", "memory_id": "", "limit": 10 } +``` + +```json +{ "action": "never_composed", "tags": ["project:vestige"], "limit": 10 } +``` + +```json +{ + "action": "label", + "event_id": "", + "outcome_type": "helpful", + "notes": "This combination led to the accepted fix." +} +``` + +## Never-Composed Frontier + +`never_composed` returns pairs that have not yet appeared together in a +composition event. + +The ranking is intentionally not just shared-tag matching. It combines: + +- exact shared tags +- shared meaningful content terms +- boundary tags such as `boundary-*`, `oracle`, `queue`, `settlement`, `upgrade`, + `pause`, `accounting`, or `scope` +- node-type diversity +- FSRS retention strength +- composition novelty, so memories that have not already been heavily composed + still get surfaced +- prior composition outcomes from either member, so previously accepted, + duplicate-risk, or dead-end lanes shape the frontier without hiding it + +Each candidate includes: + +- `score` +- `noveltyScore` +- `bridgeScore` +- `trustScore` +- `outcomeScoreAdjustment` +- `sharedTags` +- `boundaryTags` +- `sharedTerms` +- `priorOutcomes` +- `outcomeSignal`, such as `clean`, `prior_success`, `prior_duplicate_risk`, + `prior_closed_door`, or `mixed_prior_outcomes` +- node types +- previews +- a short reason +- a `compositionQuestion` that an agent can answer before taking action + +The output is a frontier queue, not a finding. A never-composed pair means +"worth investigating," not "true," "novel," or "reportable." +Prior outcomes are also guardrails, not verdicts: a duplicate-risk signal should +make the agent check duplicate families first, while a success signal should make +it inspect why the older composition worked. + +Closed-door labels should be specific when possible. Prefer `closed_by_scope`, +`closed_by_duplicate`, `closed_by_false_assumption`, `closed_by_user`, or +`expired_lane` over a generic `dead_end` when the reason is known. + +## Bounty / Research Mode + +`bounty_mode` is a higher-level read shape for investigative workflows. It returns: + +- recent already-composed lanes +- never-composed lanes +- closed doors +- duplicate-risk lanes +- lanes that need proof-of-concept work +- top weird combinations + +This is useful for security research, bug triage, architecture work, and product +strategy because failed or duplicate compositions are preserved instead of being +rediscovered repeatedly. + +## Deep Reference Integration + +`deep_reference` persists composition events automatically when it has evidence +members. Empty evidence does not create a ledger event. + +The response includes: + +- `composition_event_id` when persisted +- `compositionWriteStatus`, usually `persisted` or `skipped_empty` + +## Design Direction + +The next useful upgrades are: + +- triple or n-ary candidate mining, not only pairs +- structural-fit scoring for analogies, separate from surface similarity +- trust-zone scoring so a composition is limited by its weakest provenance +- temporal replay: "what combinations were available when this decision was made?" +- evaluation tasks where success requires combining memories that were never + previously co-composed From b45ea819d7de450bbe2e1fce9c0fef160bd3bac5 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 16:08:51 -0500 Subject: [PATCH 23/38] Fix ComposedGraph clippy warnings --- crates/vestige-core/src/storage/sqlite.rs | 2 +- crates/vestige-mcp/src/tools/composed_graph.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index a9840a1..94ed45b 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -4822,7 +4822,7 @@ impl Storage { { let tagged_nodes = self.get_nodes_matching_any_tag_prefix(filter, TAGGED_SCAN_LIMIT)?; let mut by_id = HashMap::new(); - for node in nodes.into_iter().chain(tagged_nodes.into_iter()) { + for node in nodes.into_iter().chain(tagged_nodes) { by_id.entry(node.id.clone()).or_insert(node); } nodes = by_id.into_values().collect(); diff --git a/crates/vestige-mcp/src/tools/composed_graph.rs b/crates/vestige-mcp/src/tools/composed_graph.rs index ee69d93..957f8e8 100644 --- a/crates/vestige-mcp/src/tools/composed_graph.rs +++ b/crates/vestige-mcp/src/tools/composed_graph.rs @@ -256,7 +256,7 @@ fn bounty_mode(storage: &Storage, limit: i32, tags: Option<&[String]>) -> Result { push_limited(&mut duplicate_risk_lanes, item.clone(), limit); } - if outcome_types.iter().any(|kind| *kind == "needs_poc") { + if outcome_types.contains(&"needs_poc") { push_limited(&mut needs_poc_lanes, item.clone(), limit); } if already_composed.len() < limit as usize { From e1f37965236fc801847fad71cc1a2b78e0750a76 Mon Sep 17 00:00:00 2001 From: Caio Ribeiro Date: Thu, 18 Jun 2026 23:02:32 +0000 Subject: [PATCH 24/38] docs: add test integrity delta receipt sketch --- docs/SANHEDRIN_RECEIPTS.md | 2 + docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md | 110 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md diff --git a/docs/SANHEDRIN_RECEIPTS.md b/docs/SANHEDRIN_RECEIPTS.md index ac0bd4d..2213c58 100644 --- a/docs/SANHEDRIN_RECEIPTS.md +++ b/docs/SANHEDRIN_RECEIPTS.md @@ -12,6 +12,8 @@ instead of opaque. The current schema is `vestige.sanhedrin.receipt.v1`. - Appeals: `~/.vestige/sanhedrin/appeals.jsonl` - Fail-open events: `~/.vestige/sanhedrin/fail-open.jsonl` +Optional companion schema: [`SANHEDRIN_TEST_INTEGRITY_DELTAS.md`](SANHEDRIN_TEST_INTEGRITY_DELTAS.md) describes mechanical deltas for cases where a verifier command passed but the test artifact changed after implementation. + ## v1 JSON Shape ```json diff --git a/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md b/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md new file mode 100644 index 0000000..c9d12dc --- /dev/null +++ b/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md @@ -0,0 +1,110 @@ +# Sanhedrin Test-Integrity Delta Receipts + +Receipt Lock proves a narrower claim: a verification command actually ran and +succeeded. Test-integrity deltas are an optional companion receipt for the +stronger claim that the tests still mean what the draft says they mean. + +This receipt is intentionally mechanical. It is not a broad correctness oracle +and it does not ask a second model to decide whether the implementation is good. +It records whether the verification artifact changed in ways that should +upgrade, downgrade, or send the verification claim to human review. + +## Boundary + +Keep these claims separate: + +1. **Command receipt:** `cargo test`, `npm test`, `pytest`, or another verifier + command ran after the relevant edit and exited successfully. +2. **Test-integrity delta:** the tests/specs behind that verifier were not + removed, skipped, weakened, or replaced after implementation in a way that + makes the green result less admissible. + +A run can have a valid command receipt and still receive a downgraded +integrity decision. + +## Optional JSON Shape + +```json +{ + "schema": "vestige.sanhedrin.test_integrity_delta.v1", + "id": "tid_", + "commandReceiptId": "receipt_", + "verificationClaim": "All tests passed.", + "specSource": { + "contextId": "spec_ctx_04", + "testFiles": [ + { + "path": "tests/cart.test.ts", + "hashBeforeImplementation": "sha256:...", + "hashAfterVerification": "sha256:..." + } + ] + }, + "implementationContext": "impl_ctx_09", + "verifierContext": "verify_ctx_02", + "delta": { + "testFilesChangedAfterImplementation": true, + "removedOrDisabledTests": [ + { + "kind": "skip_or_only", + "path": "tests/cart.test.ts", + "line": 42 + } + ], + "removedAssertions": 2, + "weakenedExpectations": [ + { + "path": "tests/cart.test.ts", + "from": "throws InvalidCouponError", + "to": "does not throw" + } + ], + "snapshotChurnWithoutSourceChange": false, + "coverageDelta": -3.8, + "mocksReplacingRealBoundary": [ + { + "module": "PaymentGateway", + "before": "integration-ish fake", + "after": "empty stub" + } + ] + }, + "freshVerifier": { + "commandReceiptId": "receipt_", + "exitCode": 0, + "checkedAfterLastRelevantEdit": true + }, + "decision": "downgraded", + "reason": "tests passed, but the tests were weakened after implementation" +} +``` + +## Decisions + +- `accepted` — a verifier command succeeded after the last relevant edit and no + integrity downgrade was detected. +- `downgraded` — the command succeeded, but the tests/specs changed in a way + that makes the verification claim weaker than stated. +- `needs_human_review` — the delta may be legitimate, but a local mechanical + check cannot safely classify it. Snapshot updates are a common example. + +## Minimal Fixture Suite + +These cases are small enough to live as fixtures without turning Sanhedrin into +a correctness judge. + +| Case | Input pattern | Expected decision | Why | +| --- | --- | --- | --- | +| unchanged-good | implementation changes source; tests unchanged; fresh verifier succeeds | `accepted` | Green tests are supported by a fresh command receipt and unchanged test artifact. | +| skipped-test | implementation adds `.skip`, `.only`, `#[ignore]`, or equivalent before verifier succeeds | `downgraded` | The command ran, but the claim no longer represents the original test obligation. | +| weakened-assertion | expectation is relaxed after implementation, e.g. `throws InvalidCouponError` -> `does not throw` | `downgraded` | The verifier passed against a weaker assertion than the one available before implementation. | +| justified-snapshot | snapshot changes alongside an intentional source/UI change | `needs_human_review` or `accepted` by policy | Snapshot churn can be valid, but the receipt should make the policy decision explicit. | + +## Non-goals + +- Do not infer whether the implementation is correct in the world. +- Do not require full semantic diffing before Receipt Lock can operate. +- Do not treat staged evidence or a model explanation as equivalent to a fresh + command receipt. +- Do not block every test edit. The goal is to keep the verification claim + honest when the test artifact changed after implementation. From 5715f585fdcada8cc16fb64af86232df78616337 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Tue, 21 Apr 2026 21:43:52 +0200 Subject: [PATCH 25/38] feat(storage): phase 1 -- extract MemoryStore and Embedder traits (ADR 0001) Introduce two trait boundaries that the rest of the stack now sits above, landing Phase 1 of ADR 0001 (pluggable storage and network access). Rebased onto v2.1.22 Sanhedrin from the original April work. MemoryStore / LocalMemoryStore (crates/vestige-core/src/storage/memory_store.rs): One trait, ~25 methods, covering CRUD, hybrid / FTS / vector search, FSRS scheduling, graph edges, and the forthcoming domain surface. trait_variant::make generates a Send-bound MemoryStore alias over the base LocalMemoryStore so Arc works under tokio/axum. Storage errors map through a dedicated MemoryStoreError. Embedder / LocalEmbedder (crates/vestige-core/src/embedder/): Pluggable text-to-vector encoder. FastembedEmbedder wraps the existing EmbeddingService; storage never calls fastembed directly anymore. Embedder::signature() produces the ModelSignature consumed by the store's embedding_model registry. SqliteMemoryStore (crates/vestige-core/src/storage/sqlite.rs): Storage renamed to SqliteMemoryStore; the old name lives on as a pub type alias so Arc consumers in vestige-mcp stay intact. All existing inherent methods are untouched; the trait impl is purely additive and dispatches into them. The db_path field added by v2.1.1 portable-sync is preserved. Migration V14 (crates/vestige-core/src/storage/migrations.rs): Renumbered from V12 (the original April number) to V14 to slot in cleanly after upstream's V12 (v2.1.1 sync_tombstones) and V13 (v2.1.2 purge tombstones). - embedding_model registry table (CHECK id = 1, code enforces the single-row invariant). - knowledge_nodes.domains / domain_scores TEXT columns (JSON arrays default '[]' / '{}'), domains catalogue table, supporting indexes. Phase 4 populates these columns; Phase 1 just exposes the schema. Consolidation and other cognitive pathways now accept a &dyn LocalMemoryStore (sync) or Arc (async) rather than a concrete Storage. Tests: - trait-method unit tests colocated in sqlite.rs and migrations.rs - embedder/fastembed.rs tests for name/dimension/hash stability - new integration crate tests/phase_1 (added to workspace members): trait_round_trip (8), embedding_model_registry (7), domain_column_migration (5), cognitive_module_isolation (4), send_bound_variant (2), embedder_trait (2). Acceptance gate post-rebase: - cargo build --workspace --all-targets: ok - cargo clippy --workspace --all-targets -- -D warnings: clean - cargo test -p vestige-core --lib: 428 pass - cargo test -p vestige-phase-1-tests: 28 pass - cargo test -p vestige-mcp --lib: 380 pass (Storage alias preserves every existing call site) Co-existence with v2.1.1 portable-sync: this trait extraction is additive. Portable-sync's tombstone migrations (V12, V13) remain on the concrete SqliteMemoryStore; Phase 2 (Postgres) will decide which of those surfaces graduate into the trait. --- Cargo.lock | 115 +- Cargo.toml | 1 + crates/vestige-core/Cargo.toml | 3 + crates/vestige-core/src/embedder/fastembed.rs | 182 ++ crates/vestige-core/src/embedder/mod.rs | 57 + crates/vestige-core/src/lib.rs | 44 +- .../vestige-core/src/storage/memory_store.rs | 316 ++++ crates/vestige-core/src/storage/migrations.rs | 198 ++- crates/vestige-core/src/storage/mod.rs | 21 +- crates/vestige-core/src/storage/sqlite.rs | 1540 ++++++++++++++++- tests/phase_1/Cargo.toml | 38 + tests/phase_1/cognitive_module_isolation.rs | 143 ++ tests/phase_1/domain_column_migration.rs | 161 ++ tests/phase_1/embedder_trait.rs | 43 + tests/phase_1/embedding_model_registry.rs | 148 ++ tests/phase_1/send_bound_variant.rs | 99 ++ tests/phase_1/trait_round_trip.rs | 217 +++ 17 files changed, 3282 insertions(+), 44 deletions(-) create mode 100644 crates/vestige-core/src/embedder/fastembed.rs create mode 100644 crates/vestige-core/src/embedder/mod.rs create mode 100644 crates/vestige-core/src/storage/memory_store.rs create mode 100644 tests/phase_1/Cargo.toml create mode 100644 tests/phase_1/cognitive_module_isolation.rs create mode 100644 tests/phase_1/domain_column_migration.rs create mode 100644 tests/phase_1/embedder_trait.rs create mode 100644 tests/phase_1/embedding_model_registry.rs create mode 100644 tests/phase_1/send_bound_variant.rs create mode 100644 tests/phase_1/trait_round_trip.rs diff --git a/Cargo.lock b/Cargo.lock index 0b613a0..8be114c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,12 @@ dependencies = [ "syn", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -158,6 +164,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -311,6 +328,20 @@ dependencies = [ "core2", ] +[[package]] +name = "blake3" +version = "1.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block" version = "0.1.6" @@ -642,6 +673,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.9.4" @@ -697,6 +734,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2282,12 +2328,10 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ - "cfg-if", - "futures-util", "once_cell", "wasm-bindgen", ] @@ -3181,9 +3225,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] @@ -3822,7 +3866,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -3833,7 +3877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4356,6 +4400,17 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -4631,6 +4686,8 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "vestige-core" version = "2.1.26" dependencies = [ + "async-trait", + "blake3", "candle-core", "chrono", "criterion", @@ -4646,6 +4703,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", + "trait-variant", "usearch", "uuid", ] @@ -4692,6 +4750,19 @@ dependencies = [ "vestige-core", ] +[[package]] +name = "vestige-phase-1-tests" +version = "0.0.1" +dependencies = [ + "chrono", + "rusqlite", + "serde_json", + "tempfile", + "tokio", + "uuid", + "vestige-core", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -4737,9 +4808,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", @@ -4750,19 +4821,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ + "cfg-if", + "futures-util", "js-sys", + "once_cell", "wasm-bindgen", + "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4770,9 +4845,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", @@ -4783,9 +4858,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] @@ -4839,9 +4914,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 7183f40..203a857 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/vestige-core", "crates/vestige-mcp", "tests/e2e", + "tests/phase_1", ] exclude = [ "fastembed-rs", diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index e878cdb..25a0495 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -125,6 +125,9 @@ usearch = { version = "=2.23.0", optional = true } # LRU cache for query embeddings lru = "0.16" +trait-variant = "0.1" +blake3 = "1" +async-trait = "0.1" [dev-dependencies] tempfile = "3" diff --git a/crates/vestige-core/src/embedder/fastembed.rs b/crates/vestige-core/src/embedder/fastembed.rs new file mode 100644 index 0000000..a4cd87b --- /dev/null +++ b/crates/vestige-core/src/embedder/fastembed.rs @@ -0,0 +1,182 @@ +//! `FastembedEmbedder` -- adapts the existing `EmbeddingService` to the +//! `LocalEmbedder` trait. + +#[cfg(feature = "embeddings")] +use crate::embeddings::{EMBEDDING_DIMENSIONS, EmbeddingService}; + +use super::{EmbedderError, EmbedderResult, LocalEmbedder}; + +pub struct FastembedEmbedder { + #[cfg(feature = "embeddings")] + inner: EmbeddingService, + cached_hash: std::sync::OnceLock, +} + +impl FastembedEmbedder { + pub fn new() -> Self { + Self { + #[cfg(feature = "embeddings")] + inner: EmbeddingService::new(), + cached_hash: std::sync::OnceLock::new(), + } + } + + fn compute_hash(name: &str, dim: usize) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(name.as_bytes()); + hasher.update(&(dim as u64).to_le_bytes()); + // fastembed's ONNX bytes are not directly accessible at runtime; we + // use `(name, dim, vestige-core CARGO_PKG_VERSION)` as the + // signature. If fastembed ever changes its output deterministically + // between minor versions, bumping the crate version triggers a + // mismatch -- which is exactly the drift we want to detect. + hasher.update(env!("CARGO_PKG_VERSION").as_bytes()); + hasher.finalize().to_hex().to_string() + } +} + +impl Default for FastembedEmbedder { + fn default() -> Self { + Self::new() + } +} + +#[async_trait::async_trait] +impl LocalEmbedder for FastembedEmbedder { + async fn embed(&self, text: &str) -> EmbedderResult> { + #[cfg(feature = "embeddings")] + { + let emb = self + .inner + .embed(text) + .map_err(|e| EmbedderError::EmbedFailed(e.to_string()))?; + Ok(emb.vector) + } + #[cfg(not(feature = "embeddings"))] + { + let _ = text; + Err(EmbedderError::Init( + "embeddings feature not enabled".to_string(), + )) + } + } + + fn model_name(&self) -> &str { + #[cfg(feature = "embeddings")] + { + self.inner.model_name() + } + #[cfg(not(feature = "embeddings"))] + { + "nomic-ai/nomic-embed-text-v1.5" + } + } + + fn dimension(&self) -> usize { + #[cfg(feature = "embeddings")] + { + EMBEDDING_DIMENSIONS + } + #[cfg(not(feature = "embeddings"))] + { + 256 + } + } + + fn model_hash(&self) -> String { + self.cached_hash + .get_or_init(|| Self::compute_hash(self.model_name(), self.dimension())) + .clone() + } + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>> { + #[cfg(feature = "embeddings")] + { + let embs = self + .inner + .embed_batch(texts) + .map_err(|e| EmbedderError::EmbedFailed(e.to_string()))?; + Ok(embs.into_iter().map(|e| e.vector).collect()) + } + #[cfg(not(feature = "embeddings"))] + { + let _ = texts; + Err(EmbedderError::Init( + "embeddings feature not enabled".to_string(), + )) + } + } +} + +// ============================================================================ +// UNIT TESTS +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn embedder_reports_correct_name() { + let e = FastembedEmbedder::new(); + assert!( + e.model_name().contains("nomic"), + "model name should contain 'nomic'" + ); + } + + #[test] + fn embedder_reports_256_dimension() { + let e = FastembedEmbedder::new(); + assert_eq!(e.dimension(), 256); + } + + #[test] + fn embedder_hash_is_stable() { + let e = FastembedEmbedder::new(); + let h1 = e.model_hash(); + let h2 = e.model_hash(); + assert_eq!(h1, h2, "model_hash must be stable across calls"); + } + + #[test] + fn embedder_hash_includes_crate_version() { + // Compute what the hash should be given the known inputs + let name = FastembedEmbedder::new().model_name().to_string(); + let dim = FastembedEmbedder::new().dimension(); + let expected = FastembedEmbedder::compute_hash(&name, dim); + let got = FastembedEmbedder::new().model_hash(); + assert_eq!(got, expected); + } + + #[test] + fn embedder_signature_matches_accessors() { + let e = FastembedEmbedder::new(); + let sig = e.signature(); + assert_eq!(sig.name, e.model_name()); + assert_eq!(sig.dimension, e.dimension()); + assert_eq!(sig.hash, e.model_hash()); + } + + #[cfg(feature = "embeddings")] + #[test] + fn embedder_embed_smoke() { + let e = FastembedEmbedder::new(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let vec = rt.block_on(e.embed("hello world")).expect("embed"); + assert_eq!(vec.len(), 256); + } + + #[cfg(feature = "embeddings")] + #[test] + fn embedder_embed_batch_matches_sequential() { + let e = FastembedEmbedder::new(); + let rt = tokio::runtime::Runtime::new().unwrap(); + let texts = ["alpha beta", "gamma delta"]; + let batch = rt.block_on(e.embed_batch(texts.as_ref())).expect("batch"); + let seq_a = rt.block_on(e.embed(texts[0])).expect("seq a"); + let seq_b = rt.block_on(e.embed(texts[1])).expect("seq b"); + assert_eq!(batch[0], seq_a); + assert_eq!(batch[1], seq_b); + } +} diff --git a/crates/vestige-core/src/embedder/mod.rs b/crates/vestige-core/src/embedder/mod.rs new file mode 100644 index 0000000..9d43d0d --- /dev/null +++ b/crates/vestige-core/src/embedder/mod.rs @@ -0,0 +1,57 @@ +//! Text-to-vector encoding trait. Pluggable per-install. + +mod fastembed; + +pub use fastembed::FastembedEmbedder; + +/// Error returned by every `Embedder` method. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum EmbedderError { + #[error("embedder initialization failed: {0}")] + Init(String), + #[error("embedding generation failed: {0}")] + EmbedFailed(String), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +pub type EmbedderResult = std::result::Result; + +/// Pluggable embedder. The storage layer NEVER calls fastembed directly; +/// callers compute vectors via this trait and pass them into `MemoryStore`. +/// +/// `#[async_trait::async_trait]` makes every `async fn` return a +/// `Pin>`, which is required for `Box` +/// and `Arc` to be dyn-compatible. +#[async_trait::async_trait] +pub trait LocalEmbedder: Send + Sync + 'static { + async fn embed(&self, text: &str) -> EmbedderResult>; + + fn model_name(&self) -> &str; + + fn dimension(&self) -> usize; + + /// Stable blake3 hash of (model_name || dimension || vestige-core crate version). + /// Lowercase hex, 64 chars. + /// + /// Used by `MemoryStore::register_model` to detect silent model drift + /// (e.g. a fastembed minor upgrade that changes vector output). + fn model_hash(&self) -> String; + + async fn embed_batch(&self, texts: &[&str]) -> EmbedderResult>>; + + /// Returns the `ModelSignature` describing this embedder. Convenience + /// wrapper over the three accessors above. + fn signature(&self) -> crate::storage::ModelSignature { + crate::storage::ModelSignature { + name: self.model_name().to_string(), + dimension: self.dimension(), + hash: self.model_hash(), + } + } +} + +/// Type alias: `Embedder` is the dyn-compatible, Send+Sync variant. +/// Both names refer to the same `async_trait`-annotated trait. +pub use LocalEmbedder as Embedder; diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index b8b0154..f8a35d6 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -83,6 +83,7 @@ /// Optional `vestige.toml` configuration (Phase 2: Configurable Output). pub mod config; pub mod consolidation; +pub mod embedder; pub mod fsrs; pub mod fts; pub mod memory; @@ -159,13 +160,46 @@ pub use config::{CONFIG_FILE, OutputConfig, OutputDefaults, OutputProfile, Vesti // Storage layer pub use storage::{ - CompositionEventRecord, CompositionMemberRecord, CompositionNeighborRecord, - CompositionOutcomeRecord, ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, - InsightRecord, IntentionRecord, NeverComposedCandidate, PORTABLE_ARCHIVE_FORMAT, - PortableArchive, PortableImportMode, PortableImportReport, Result, SmartIngestResult, - StateTransitionRecord, Storage, StorageError, + ClassificationResult, + CompositionEventRecord, + CompositionMemberRecord, + CompositionNeighborRecord, + CompositionOutcomeRecord, + ConnectionRecord, + ConsolidationHistoryRecord, + Domain, + DreamHistoryRecord, + HealthStatus, + InsightRecord, + IntentionRecord, + LocalMemoryStore, + MemoryEdge, + MemoryRecord, + MemoryStore, + MemoryStoreError, + MemoryStoreResult, + ModelSignature, + NeverComposedCandidate, + PORTABLE_ARCHIVE_FORMAT, + PortableArchive, + PortableImportMode, + PortableImportReport, + Result, + SchedulingState, + SearchQuery, + SmartIngestResult, + SqliteMemoryStore, + StateTransitionRecord, + Storage, + StorageError, + StoreStats, + // Note: storage::SearchResult is intentionally not re-exported here to avoid + // collision with memory::SearchResult. Use vestige_core::storage::SearchResult directly. }; +// Embedder trait and implementations +pub use embedder::{Embedder, EmbedderError, EmbedderResult, FastembedEmbedder, LocalEmbedder}; + // Consolidation (sleep-inspired memory processing) pub use consolidation::SleepConsolidation; pub use consolidation::{ diff --git a/crates/vestige-core/src/storage/memory_store.rs b/crates/vestige-core/src/storage/memory_store.rs new file mode 100644 index 0000000..2bc3137 --- /dev/null +++ b/crates/vestige-core/src/storage/memory_store.rs @@ -0,0 +1,316 @@ +//! Backend-agnostic memory store trait. +//! +//! This is the single abstraction every cognitive module sits above. It is +//! intentionally flat: one trait, ~25 methods, no sub-traits. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +// ---------------------------------------------------------------------------- +// ERROR +// ---------------------------------------------------------------------------- + +/// Error returned by every `LocalMemoryStore` / `MemoryStore` method. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum MemoryStoreError { + #[error("not found: {0}")] + NotFound(String), + + #[error("backend error: {0}")] + Backend(String), + + #[error( + "embedding model mismatch: store registered {registered_name} (dim {registered_dim}, \ + hash {registered_hash}), embedder is {actual_name} (dim {actual_dim}, hash {actual_hash})" + )] + ModelMismatch { + registered_name: String, + registered_dim: usize, + registered_hash: String, + actual_name: String, + actual_dim: usize, + actual_hash: String, + }, + + #[error("invalid input: {0}")] + InvalidInput(String), + + #[error("initialization error: {0}")] + Init(String), +} + +impl From for MemoryStoreError { + fn from(e: crate::storage::StorageError) -> Self { + use crate::storage::StorageError as S; + match e { + S::NotFound(s) => MemoryStoreError::NotFound(s), + S::Database(e) => MemoryStoreError::Backend(e.to_string()), + S::Io(e) => MemoryStoreError::Backend(e.to_string()), + S::InvalidTimestamp(s) => MemoryStoreError::Backend(format!("invalid timestamp: {s}")), + S::Init(s) => MemoryStoreError::Init(s), + } + } +} + +pub type MemoryStoreResult = std::result::Result; + +// ---------------------------------------------------------------------------- +// DATA TYPES +// ---------------------------------------------------------------------------- + +/// Backend-agnostic memory record. +/// +/// Phase 1 intentionally keeps this type independent of `KnowledgeNode` to +/// avoid dragging 30+ legacy fields through the trait surface. The SQLite +/// backend converts between `MemoryRecord` and `KnowledgeNode` at the +/// boundary. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRecord { + pub id: Uuid, + /// Empty = unclassified. Populated in Phase 4. + pub domains: Vec, + /// Raw similarity per domain centroid. Empty until Phase 4 runs clustering. + pub domain_scores: HashMap, + pub content: String, + pub node_type: String, + pub tags: Vec, + pub embedding: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, + pub metadata: serde_json::Value, +} + +/// FSRS-6 scheduling state, one row per memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SchedulingState { + pub memory_id: Uuid, + pub stability: f64, + pub difficulty: f64, + pub retrievability: f64, + pub last_review: Option>, + pub next_review: Option>, + pub reps: u32, + pub lapses: u32, +} + +/// Hybrid search request. +#[derive(Debug, Clone, Default)] +pub struct SearchQuery { + pub domains: Option>, + pub text: Option, + pub embedding: Option>, + pub tags: Option>, + pub node_types: Option>, + pub limit: usize, + pub min_retrievability: Option, +} + +#[derive(Debug, Clone)] +pub struct SearchResult { + pub record: MemoryRecord, + pub score: f64, + pub fts_score: Option, + pub vector_score: Option, +} + +/// Edge in the spreading-activation graph. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryEdge { + pub source_id: Uuid, + pub target_id: Uuid, + pub edge_type: String, + pub weight: f64, + pub created_at: DateTime, +} + +/// A topical domain (populated in Phase 4). Phase 1 only needs the type to +/// shape the trait surface; discover/classify are Phase 4 work. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Domain { + pub id: String, + pub label: String, + pub centroid: Vec, + pub top_terms: Vec, + pub memory_count: usize, + pub created_at: DateTime, +} + +/// Result of classifying one vector against all known domains. +#[derive(Debug, Clone)] +pub struct ClassificationResult { + pub scores: HashMap, + pub domains: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StoreStats { + pub total_memories: usize, + pub memories_with_embeddings: usize, + pub total_edges: usize, + pub total_domains: usize, + pub registered_model_name: Option, + pub registered_model_dim: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum HealthStatus { + Healthy, + Degraded { reason: String }, + Unavailable { reason: String }, +} + +// ---------------------------------------------------------------------------- +// EMBEDDING MODEL SIGNATURE +// ---------------------------------------------------------------------------- + +/// Snapshot of the embedding model that was used to write vectors into the +/// store. Persisted in the `embedding_model` table; compared on every write +/// before the vector is accepted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModelSignature { + pub name: String, + pub dimension: usize, + /// Lowercase hex-encoded blake3 hash, 64 chars. + pub hash: String, +} + +// ---------------------------------------------------------------------------- +// TRAIT +// ---------------------------------------------------------------------------- + +/// The single storage abstraction. +/// +/// `#[async_trait::async_trait]` makes every `async fn` return a +/// `Pin>`, which is required for `Arc` +/// to be movable across `tokio::spawn` boundaries. +/// +/// `LocalMemoryStore` is a type alias kept for source compatibility with code +/// that refers to the non-send variant. In Phase 1 both names refer to the same +/// (dyn-compatible, Send-safe) trait. +#[async_trait::async_trait] +pub trait MemoryStore: Send + Sync + 'static { + // --- Lifecycle --- + async fn init(&self) -> MemoryStoreResult<()>; + async fn health_check(&self) -> MemoryStoreResult; + + // --- Embedding model registry --- + async fn registered_model(&self) -> MemoryStoreResult>; + async fn register_model(&self, sig: &ModelSignature) -> MemoryStoreResult<()>; + + // --- CRUD --- + async fn insert(&self, record: &MemoryRecord) -> MemoryStoreResult; + async fn get(&self, id: Uuid) -> MemoryStoreResult>; + async fn update(&self, record: &MemoryRecord) -> MemoryStoreResult<()>; + async fn delete(&self, id: Uuid) -> MemoryStoreResult<()>; + + // --- Search --- + async fn search(&self, query: &SearchQuery) -> MemoryStoreResult>; + async fn fts_search(&self, text: &str, limit: usize) -> MemoryStoreResult>; + async fn vector_search( + &self, + embedding: &[f32], + limit: usize, + ) -> MemoryStoreResult>; + + // --- FSRS Scheduling --- + async fn get_scheduling(&self, memory_id: Uuid) -> MemoryStoreResult>; + async fn update_scheduling(&self, state: &SchedulingState) -> MemoryStoreResult<()>; + async fn get_due_memories( + &self, + before: DateTime, + limit: usize, + ) -> MemoryStoreResult>; + + // --- Graph (spreading activation) --- + async fn add_edge(&self, edge: &MemoryEdge) -> MemoryStoreResult<()>; + async fn get_edges( + &self, + node_id: Uuid, + edge_type: Option<&str>, + ) -> MemoryStoreResult>; + async fn remove_edge(&self, source: Uuid, target: Uuid) -> MemoryStoreResult<()>; + async fn get_neighbors( + &self, + node_id: Uuid, + depth: usize, + ) -> MemoryStoreResult>; + + // --- Domains (Phase 1: stubs return empty; full impl in Phase 4) --- + async fn list_domains(&self) -> MemoryStoreResult>; + async fn get_domain(&self, id: &str) -> MemoryStoreResult>; + async fn upsert_domain(&self, domain: &Domain) -> MemoryStoreResult<()>; + async fn delete_domain(&self, id: &str) -> MemoryStoreResult<()>; + /// Phase 1: returns `Ok(vec![])` since no centroids exist. Phase 4 wires + /// the full soft-assignment pass. + async fn classify(&self, embedding: &[f32]) -> MemoryStoreResult>; + + // --- Bulk / Maintenance --- + async fn count(&self) -> MemoryStoreResult; + async fn get_stats(&self) -> MemoryStoreResult; + async fn vacuum(&self) -> MemoryStoreResult<()>; +} + +/// Type alias kept for source compatibility. Both names refer to the same +/// `async_trait`-annotated trait that is dyn-compatible and `Send + Sync`. +pub use MemoryStore as LocalMemoryStore; + +// ---------------------------------------------------------------------------- +// UNIT TESTS +// ---------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::StorageError; + + #[test] + fn memory_store_error_from_storage_error() { + let se = StorageError::NotFound("abc".to_string()); + let mse = MemoryStoreError::from(se); + assert!(matches!(mse, MemoryStoreError::NotFound(_))); + + let se2 = StorageError::Init("init failure".to_string()); + let mse2 = MemoryStoreError::from(se2); + assert!(matches!(mse2, MemoryStoreError::Init(_))); + } + + #[test] + fn model_signature_serde_round_trip() { + let sig = ModelSignature { + name: "nomic-ai/nomic-embed-text-v1.5".to_string(), + dimension: 256, + hash: "a".repeat(64), + }; + let json = serde_json::to_string(&sig).expect("serialize"); + let sig2: ModelSignature = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(sig, sig2); + } + + #[test] + fn memory_record_serde_round_trip() { + let rec = MemoryRecord { + id: Uuid::new_v4(), + domains: vec!["dev".to_string()], + domain_scores: { + let mut m = HashMap::new(); + m.insert("dev".to_string(), 0.9); + m + }, + content: "hello".to_string(), + node_type: "fact".to_string(), + tags: vec!["tag1".to_string()], + embedding: None, + created_at: Utc::now(), + updated_at: Utc::now(), + metadata: serde_json::json!({}), + }; + let json = serde_json::to_string(&rec).expect("serialize"); + let rec2: MemoryRecord = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(rec.content, rec2.content); + assert_eq!(rec.domains, rec2.domains); + } +} diff --git a/crates/vestige-core/src/storage/migrations.rs b/crates/vestige-core/src/storage/migrations.rs index 127bc84..c0c60d2 100644 --- a/crates/vestige-core/src/storage/migrations.rs +++ b/crates/vestige-core/src/storage/migrations.rs @@ -79,6 +79,11 @@ pub const MIGRATIONS: &[Migration] = &[ description: "ComposedGraph: composition events, members, outcomes", up: MIGRATION_V15_UP, }, + Migration { + version: 16, + description: "ADR 0001 Phase 1: embedding_model registry, domains/domain_scores columns, domains table", + up: MIGRATION_V16_UP, + }, ]; /// A database migration @@ -904,6 +909,54 @@ fn add_column_if_missing(conn: &rusqlite::Connection, sql: &str) -> rusqlite::Re } } +/// V16: ADR 0001 Phase 1 - embedding_model registry + domain columns. +/// +/// The ALTER TABLE statements are split out into `MIGRATION_V16_ALTER_COLUMNS` +/// because SQLite has no `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`. The +/// migration runner handles them individually so replaying V16 is idempotent. +const MIGRATION_V16_UP: &str = r#" +-- Migration V16: embedding model registry + per-memory domain columns. + +-- 1. Embedding model registry. Single logical row; the (id = 1) constraint is +-- enforced in code via `register_model` (SQLite CHECK on a single-row +-- table is uglier than a constraint we already enforce in Rust). +CREATE TABLE IF NOT EXISTS embedding_model ( + id INTEGER PRIMARY KEY CHECK (id = 1), + name TEXT NOT NULL, + dimension INTEGER NOT NULL, + hash TEXT NOT NULL, + created_at TEXT NOT NULL +); + +-- 2. Per-memory domain columns are applied separately (see apply_migrations). + +-- 3. Index on the domains JSON column to enable LIKE-style filter in Phase 4. +CREATE INDEX IF NOT EXISTS idx_nodes_domains ON knowledge_nodes(domains); +CREATE INDEX IF NOT EXISTS idx_nodes_domain_scores ON knowledge_nodes(domain_scores); + +-- 4. Domains catalogue (empty until Phase 4 populates). +CREATE TABLE IF NOT EXISTS domains ( + id TEXT PRIMARY KEY, + label TEXT NOT NULL, + centroid BLOB, + top_terms TEXT NOT NULL DEFAULT '[]', + memory_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_domains_created_at ON domains(created_at); + +UPDATE schema_version SET version = 16, applied_at = datetime('now'); +"#; + +/// The two ALTER TABLE statements for V16. Kept separate so the migration +/// runner can try each individually and ignore "duplicate column" errors, +/// making V16 idempotent on replay (SQLite has no ADD COLUMN IF NOT EXISTS). +pub const MIGRATION_V16_ALTER_COLUMNS: &[&str] = &[ + "ALTER TABLE knowledge_nodes ADD COLUMN domains TEXT NOT NULL DEFAULT '[]'", + "ALTER TABLE knowledge_nodes ADD COLUMN domain_scores TEXT NOT NULL DEFAULT '{}'", +]; + /// Apply pending migrations pub fn apply_migrations(conn: &rusqlite::Connection) -> rusqlite::Result { let current_version = get_current_version(conn)?; @@ -932,6 +985,15 @@ pub fn apply_migrations(conn: &rusqlite::Connection) -> rusqlite::Result { )?; } + // V16 adds columns via ALTER TABLE, which SQLite does not support + // with IF NOT EXISTS. Run them individually and ignore duplicate + // column errors so replay stays idempotent. + if migration.version == 16 { + for stmt in MIGRATION_V16_ALTER_COLUMNS { + add_column_if_missing(conn, stmt)?; + } + } + // Use execute_batch to handle multi-statement SQL including triggers conn.execute_batch(migration.up)?; @@ -958,17 +1020,17 @@ mod tests { /// version after `apply_migrations` runs all migrations end-to-end, and /// neither of the dead tables V11 drops must exist afterwards. #[test] - fn test_apply_migrations_advances_to_v15_and_drops_dead_tables() { + fn test_apply_migrations_advances_to_v16_and_drops_dead_tables() { let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); // Pre-requisite: schema_version must be bootstrapped by V1. apply_migrations(&conn).expect("apply_migrations succeeds"); - // 1. schema_version advanced to V15 + // 1. schema_version advanced to V16 let version = get_current_version(&conn).expect("read schema_version"); assert_eq!( - version, 15, - "schema_version must be 15 after all migrations" + version, 16, + "schema_version must be 16 after all migrations" ); // 2. knowledge_edges is gone (V11 drops it) @@ -1086,10 +1148,132 @@ mod tests { conn.execute("UPDATE schema_version SET version = 10", []) .expect("rewind schema_version"); - // Replay must not error. - apply_migrations(&conn).expect("V11 replay must be idempotent"); + // Replay V11 onward. V11 uses DROP TABLE IF EXISTS so it is idempotent. + // V12/V13 tombstone tables use CREATE TABLE IF NOT EXISTS. V14/V16 ALTER + // TABLE idempotency is handled by the migration runner. + apply_migrations(&conn).expect("V11..V16 replay must be idempotent"); + // After replaying from V10, the schema advances to the latest version. let version = get_current_version(&conn).expect("read schema_version"); - assert_eq!(version, 15, "schema_version back at 15 after replay"); + assert_eq!(version, 16, "schema_version back at 16 after replay"); + } + + #[test] + fn v16_adds_embedding_model_table() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("apply_migrations"); + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='embedding_model'", + [], + |row| row.get(0), + ) + .expect("query sqlite_master"); + assert_eq!(count, 1, "embedding_model table must exist after V16"); + } + + #[test] + fn v16_adds_domains_columns() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("apply_migrations"); + let info: Vec = { + let mut stmt = conn + .prepare("PRAGMA table_info(knowledge_nodes)") + .expect("prepare"); + stmt.query_map([], |row| row.get::<_, String>(1)) + .expect("query_map") + .map(|r| r.expect("row")) + .collect() + }; + assert!( + info.contains(&"domains".to_string()), + "domains column missing" + ); + assert!( + info.contains(&"domain_scores".to_string()), + "domain_scores column missing" + ); + } + + #[test] + fn v16_default_values_empty_json() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("apply_migrations"); + // Insert a minimal row to test defaults + conn.execute( + "INSERT INTO knowledge_nodes (id, content, node_type, created_at, updated_at, last_accessed, \ + stability, difficulty, reps, lapses, learning_state, storage_strength, retrieval_strength, \ + retention_strength, next_review, scheduled_days, has_embedding) \ + VALUES ('test-id','content','fact',datetime('now'),datetime('now'),datetime('now'),\ + 1.0,0.3,0,0,'new',1.0,1.0,1.0,datetime('now'),1,0)", + [], + ).expect("insert row"); + let (domains, domain_scores): (String, String) = conn + .query_row( + "SELECT domains, domain_scores FROM knowledge_nodes WHERE id='test-id'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("query row"); + assert_eq!(domains, "[]"); + assert_eq!(domain_scores, "{}"); + } + + #[test] + fn v16_is_replayable() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("first apply"); + // Rewind to V15 so V16 runs again. + conn.execute("UPDATE schema_version SET version = 15", []) + .expect("rewind"); + // V16 uses CREATE TABLE IF NOT EXISTS and idempotent ALTER handling. + apply_migrations(&conn).expect("V16 replay must be idempotent"); + let version = get_current_version(&conn).expect("read version"); + assert_eq!(version, 16, "schema_version must be 16 after replay"); + } + + #[test] + fn v16_preserves_existing_rows_from_v15() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + // Apply up to V15 only, including the V14 ALTER TABLE columns that + // `apply_migrations` normally runs before the V14 SQL batch. + for migration in MIGRATIONS { + if migration.version <= 15 { + if migration.version == 14 { + add_column_if_missing( + &conn, + "ALTER TABLE knowledge_nodes ADD COLUMN protected INTEGER NOT NULL DEFAULT 0", + ) + .expect("apply V14 protected column"); + add_column_if_missing( + &conn, + "ALTER TABLE knowledge_nodes ADD COLUMN superseded_by TEXT", + ) + .expect("apply V14 superseded_by column"); + } + conn.execute_batch(migration.up).expect("apply migration"); + } + } + // Insert a row under the V15 schema, before PR #61's V16 columns exist. + conn.execute( + "INSERT INTO knowledge_nodes (id, content, node_type, created_at, updated_at, last_accessed, \ + stability, difficulty, reps, lapses, learning_state, storage_strength, retrieval_strength, \ + retention_strength, next_review, scheduled_days, has_embedding) \ + VALUES ('existing-id','old content','fact',datetime('now'),datetime('now'),datetime('now'),\ + 1.0,0.3,0,0,'new',1.0,1.0,1.0,datetime('now'),1,0)", + [], + ).expect("insert pre-v16 row"); + apply_migrations(&conn).expect("apply V16 migration"); + + // Check the old row has defaults + let (domains, domain_scores): (String, String) = conn + .query_row( + "SELECT domains, domain_scores FROM knowledge_nodes WHERE id='existing-id'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("query pre-v16 row"); + assert_eq!(domains, "[]"); + assert_eq!(domain_scores, "{}"); } } diff --git a/crates/vestige-core/src/storage/mod.rs b/crates/vestige-core/src/storage/mod.rs index 282228d..6926385 100644 --- a/crates/vestige-core/src/storage/mod.rs +++ b/crates/vestige-core/src/storage/mod.rs @@ -1,15 +1,17 @@ //! Storage Module //! -//! SQLite-based storage layer with: -//! - FTS5 full-text search with query sanitization -//! - Embedded vector storage -//! - FSRS-6 state management -//! - Temporal memory support +//! Backend-agnostic memory store abstraction plus SQLite reference impl. +mod memory_store; mod migrations; mod portable; mod sqlite; +pub use memory_store::{ + ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, + MemoryStore, MemoryStoreError, MemoryStoreResult, ModelSignature, SchedulingState, SearchQuery, + SearchResult, StoreStats, +}; pub use migrations::MIGRATIONS; pub use portable::{ PORTABLE_ARCHIVE_FORMAT, PortableArchive, PortableImportMode, PortableImportReport, @@ -19,6 +21,11 @@ pub use sqlite::{ CompositionEventRecord, CompositionMemberRecord, CompositionNeighborRecord, CompositionOutcomeRecord, ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, FilePortableSyncBackend, InsightRecord, IntentionRecord, NeverComposedCandidate, - PortableSyncBackend, PortableSyncReport, Result, SmartIngestResult, StateTransitionRecord, - Storage, StorageError, + PortableSyncBackend, PortableSyncReport, Result, SmartIngestResult, SqliteMemoryStore, + StateTransitionRecord, StorageError, }; + +/// Backwards-compatibility alias. Retained until Phase 4 completes so every +/// existing `Arc` call site keeps compiling. Scheduled for removal +/// once no downstream source file references it. +pub type Storage = SqliteMemoryStore; diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 94ed45b..57eaa86 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -299,7 +299,7 @@ const DATABASE_FILE: &str = "vestige.db"; /// Uses separate reader/writer connections for interior mutability. /// All methods take `&self` (not `&mut self`), making Storage `Send + Sync` /// so the MCP layer can use `Arc` instead of `Arc>`. -pub struct Storage { +pub struct SqliteMemoryStore { db_path: PathBuf, writer: Mutex, reader: Mutex, @@ -311,9 +311,11 @@ pub struct Storage { /// LRU cache for query embeddings to avoid re-embedding repeated queries #[cfg(all(feature = "embeddings", feature = "vector-search"))] query_cache: Mutex>>, + /// Cached model signature. `None` until the first embedding is written. + registered_model: std::sync::RwLock>, } -impl Storage { +impl SqliteMemoryStore { fn data_dir_from_env() -> Option { std::env::var_os(DATA_DIR_ENV).and_then(|value| { if value.is_empty() { @@ -458,6 +460,7 @@ impl Storage { vector_index: Mutex::new(vector_index), #[cfg(all(feature = "embeddings", feature = "vector-search"))] query_cache, + registered_model: std::sync::RwLock::new(None), }; #[cfg(all(feature = "embeddings", feature = "vector-search"))] @@ -595,13 +598,15 @@ impl Storage { stability, difficulty, reps, lapses, learning_state, storage_strength, retrieval_strength, retention_strength, sentiment_score, sentiment_magnitude, next_review, scheduled_days, - source, tags, valid_from, valid_until, has_embedding, embedding_model + source, tags, valid_from, valid_until, has_embedding, embedding_model, + domains, domain_scores ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, - ?19, ?20, ?21, ?22, ?23, ?24 + ?19, ?20, ?21, ?22, ?23, ?24, + '[]', '{}' )", params![ id, @@ -4120,7 +4125,7 @@ pub struct NeverComposedCandidate { pub composition_question: String, } -impl Storage { +impl SqliteMemoryStore { // ======================================================================== // COMPOSEDGRAPH PERSISTENCE // ======================================================================== @@ -8285,6 +8290,1014 @@ fn preview(content: &str, max: usize) -> String { } } +// ============================================================================ +// LOCAL MEMORY STORE TRAIT IMPL +// ============================================================================ + +impl SqliteMemoryStore { + /// Convert a `KnowledgeNode` (plus optional embedding vector read separately) + /// into a `MemoryRecord` for the trait surface. + fn node_to_record( + node: KnowledgeNode, + embedding: Option>, + ) -> crate::storage::memory_store::MemoryRecord { + use crate::storage::memory_store::MemoryRecord; + let id = uuid::Uuid::parse_str(&node.id).unwrap_or_else(|_| uuid::Uuid::new_v4()); + MemoryRecord { + id, + domains: Vec::new(), + domain_scores: std::collections::HashMap::new(), + content: node.content, + node_type: node.node_type, + tags: node.tags, + embedding, + created_at: node.created_at, + updated_at: node.updated_at, + metadata: serde_json::json!({ + "source": node.source, + "stability": node.stability, + "difficulty": node.difficulty, + "reps": node.reps, + "lapses": node.lapses, + "retention_strength": node.retention_strength, + }), + } + } + + /// Read domains and domain_scores JSON columns for a node by id. + fn read_domain_columns( + &self, + id: &str, + ) -> (Vec, std::collections::HashMap) { + let reader = match self.reader.lock() { + Ok(r) => r, + Err(_) => return (Vec::new(), std::collections::HashMap::new()), + }; + let result = reader.query_row( + "SELECT domains, domain_scores FROM knowledge_nodes WHERE id = ?1", + rusqlite::params![id], + |row| { + let d: Option = row.get(0).ok().flatten(); + let ds: Option = row.get(1).ok().flatten(); + Ok((d, ds)) + }, + ); + match result { + Ok((d, ds)) => { + let domains: Vec = d + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + let domain_scores: std::collections::HashMap = ds + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + (domains, domain_scores) + } + Err(_) => (Vec::new(), std::collections::HashMap::new()), + } + } + + /// Enforce the registered embedding model. Returns `Ok(())` if: + /// - no vector is being written (`incoming.is_none()`) and nothing is registered + /// - the incoming signature matches the registered signature + /// + /// Auto-registers on the first embedded write. + fn enforce_model( + &self, + incoming: Option<&crate::storage::memory_store::ModelSignature>, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::{MemoryStoreError, ModelSignature}; + let Some(incoming) = incoming else { + return Ok(()); + }; + // Try from cache first + { + let guard = self + .registered_model + .read() + .map_err(|_| MemoryStoreError::Init("registered_model rwlock poisoned".into()))?; + if let Some(ref reg) = *guard { + if reg == incoming { + return Ok(()); + } + return Err(MemoryStoreError::ModelMismatch { + registered_name: reg.name.clone(), + registered_dim: reg.dimension, + registered_hash: reg.hash.clone(), + actual_name: incoming.name.clone(), + actual_dim: incoming.dimension, + actual_hash: incoming.hash.clone(), + }); + } + } + // Not registered yet -- auto-register + let now = Utc::now().to_rfc3339(); + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + // Try INSERT OR IGNORE + writer.execute( + "INSERT OR IGNORE INTO embedding_model (id, name, dimension, hash, created_at) VALUES (1, ?1, ?2, ?3, ?4)", + rusqlite::params![incoming.name, incoming.dimension as i64, incoming.hash, now], + ).map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + // Read back what was stored + let stored: Option = writer + .query_row( + "SELECT name, dimension, hash FROM embedding_model WHERE id = 1", + [], + |row| { + let name: String = row.get(0)?; + let dim: i64 = row.get(1)?; + let hash: String = row.get(2)?; + Ok(ModelSignature { + name, + dimension: dim as usize, + hash, + }) + }, + ) + .optional() + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + drop(writer); + if let Some(stored) = stored { + if stored != *incoming { + return Err(MemoryStoreError::ModelMismatch { + registered_name: stored.name, + registered_dim: stored.dimension, + registered_hash: stored.hash, + actual_name: incoming.name.clone(), + actual_dim: incoming.dimension, + actual_hash: incoming.hash.clone(), + }); + } + // Populate cache + let mut guard = self + .registered_model + .write() + .map_err(|_| MemoryStoreError::Init("registered_model rwlock poisoned".into()))?; + *guard = Some(stored); + } + Ok(()) + } +} + +#[async_trait::async_trait] +impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { + async fn init(&self) -> crate::storage::memory_store::MemoryStoreResult<()> { + // Migrations run in `new`; this is a no-op for the SQLite backend. + Ok(()) + } + + async fn health_check( + &self, + ) -> crate::storage::memory_store::MemoryStoreResult + { + use crate::storage::memory_store::HealthStatus; + let reader = self.reader.lock().map_err(|_| { + crate::storage::memory_store::MemoryStoreError::Init("Reader lock poisoned".into()) + })?; + let ok: rusqlite::Result = reader.query_row("SELECT 1", [], |row| row.get(0)); + if ok.is_ok() { + Ok(HealthStatus::Healthy) + } else { + Ok(HealthStatus::Degraded { + reason: "SQLite connectivity check failed".to_string(), + }) + } + } + + async fn registered_model( + &self, + ) -> crate::storage::memory_store::MemoryStoreResult< + Option, + > { + use crate::storage::memory_store::MemoryStoreError; + // Check cache first + { + let guard = self + .registered_model + .read() + .map_err(|_| MemoryStoreError::Init("registered_model rwlock poisoned".into()))?; + if guard.is_some() { + return Ok(guard.clone()); + } + } + // Fall through to DB read + let reader = self + .reader + .lock() + .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; + let stored: Option = reader + .query_row( + "SELECT name, dimension, hash FROM embedding_model WHERE id = 1", + [], + |row| { + let name: String = row.get(0)?; + let dim: i64 = row.get(1)?; + let hash: String = row.get(2)?; + Ok(crate::storage::memory_store::ModelSignature { + name, + dimension: dim as usize, + hash, + }) + }, + ) + .optional() + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + drop(reader); + // Populate cache if we read something + if stored.is_some() { + let mut guard = self + .registered_model + .write() + .map_err(|_| MemoryStoreError::Init("registered_model rwlock poisoned".into()))?; + *guard = stored.clone(); + } + Ok(stored) + } + + async fn register_model( + &self, + sig: &crate::storage::memory_store::ModelSignature, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + self.enforce_model(Some(sig)) + } + + async fn insert( + &self, + record: &crate::storage::memory_store::MemoryRecord, + ) -> crate::storage::memory_store::MemoryStoreResult { + use crate::storage::memory_store::{MemoryStoreError, ModelSignature}; + // Enforce model registry if embedding is provided + if let Some(vec) = &record.embedding { + // Derive a signature from metadata if present, or use a generic sentinel + let sig: Option = record + .metadata + .get("model_name") + .and_then(|v| v.as_str()) + .zip( + record + .metadata + .get("model_dim") + .and_then(|v| v.as_u64()) + .map(|d| d as usize), + ) + .zip(record.metadata.get("model_hash").and_then(|v| v.as_str())) + .map(|((name, dim), hash)| ModelSignature { + name: name.to_string(), + dimension: dim, + hash: hash.to_string(), + }); + if let Some(ref s) = sig { + self.enforce_model(Some(s))?; + if vec.len() != s.dimension { + return Err(MemoryStoreError::InvalidInput(format!( + "embedding length {} != registered dimension {}", + vec.len(), + s.dimension + ))); + } + } + } + // Insert directly using the record's own id so the caller-supplied UUID is + // preserved (unlike ingest() which always generates a fresh UUID). + let id_str = record.id.to_string(); + let now = chrono::Utc::now(); + let tags_json = serde_json::to_string(&record.tags).unwrap_or_else(|_| "[]".to_string()); + let domains_json = + serde_json::to_string(&record.domains).unwrap_or_else(|_| "[]".to_string()); + let scores_json = + serde_json::to_string(&record.domain_scores).unwrap_or_else(|_| "{}".to_string()); + let source: Option = record + .metadata + .get("source") + .and_then(|v| v.as_str()) + .map(str::to_string); + { + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + writer + .execute( + "INSERT INTO knowledge_nodes ( + id, content, node_type, created_at, updated_at, last_accessed, + stability, difficulty, reps, lapses, learning_state, + storage_strength, retrieval_strength, retention_strength, + sentiment_score, sentiment_magnitude, next_review, scheduled_days, + source, tags, has_embedding, embedding_model, + domains, domain_scores + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, + 1.0, 0.3, 0, 0, 'new', + 1.0, 1.0, 1.0, + 0.0, 0.0, ?7, 1, + ?8, ?9, 0, NULL, + ?10, ?11 + )", + rusqlite::params![ + id_str, + record.content, + record.node_type, + record.created_at.to_rfc3339(), + record.updated_at.to_rfc3339(), + now.to_rfc3339(), + (now + chrono::Duration::days(1)).to_rfc3339(), + source, + tags_json, + domains_json, + scores_json, + ], + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + } + Ok(record.id) + } + + async fn get( + &self, + id: uuid::Uuid, + ) -> crate::storage::memory_store::MemoryStoreResult< + Option, + > { + use crate::storage::memory_store::MemoryStoreError; + let node = self + .get_node(&id.to_string()) + .map_err(MemoryStoreError::from)?; + let Some(node) = node else { + return Ok(None); + }; + let (domains, domain_scores) = self.read_domain_columns(&id.to_string()); + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + let embedding = self.get_node_embedding(&id.to_string()).ok().flatten(); + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + let embedding: Option> = None; + let mut rec = Self::node_to_record(node, embedding); + rec.domains = domains; + rec.domain_scores = domain_scores; + Ok(Some(rec)) + } + + async fn update( + &self, + record: &crate::storage::memory_store::MemoryRecord, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + self.update_node_content(&record.id.to_string(), &record.content) + .map_err(MemoryStoreError::from)?; + // Update domains/domain_scores + let domains_json = + serde_json::to_string(&record.domains).unwrap_or_else(|_| "[]".to_string()); + let scores_json = + serde_json::to_string(&record.domain_scores).unwrap_or_else(|_| "{}".to_string()); + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + writer + .execute( + "UPDATE knowledge_nodes SET domains = ?1, domain_scores = ?2 WHERE id = ?3", + rusqlite::params![domains_json, scores_json, record.id.to_string()], + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(()) + } + + async fn delete(&self, id: uuid::Uuid) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + self.delete_node(&id.to_string()) + .map_err(MemoryStoreError::from)?; + Ok(()) + } + + async fn search( + &self, + query: &crate::storage::memory_store::SearchQuery, + ) -> crate::storage::memory_store::MemoryStoreResult< + Vec, + > { + use crate::storage::memory_store::{MemoryStoreError, SearchResult}; + // For Phase 1 we delegate to hybrid_search or keyword_search based on what is provided. + let limit = if query.limit == 0 { 10 } else { query.limit }; + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + if let Some(ref text) = query.text { + let results = self + .hybrid_search(text, limit as i32, 0.3, 0.7) + .map_err(MemoryStoreError::from)?; + let out = results + .into_iter() + .map(|r| { + let (domains, domain_scores) = self.read_domain_columns(&r.node.id); + let mut rec = Self::node_to_record(r.node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + SearchResult { + score: r.combined_score as f64, + fts_score: r.keyword_score.map(|s| s as f64), + vector_score: r.semantic_score.map(|s| s as f64), + record: rec, + } + }) + .collect(); + return Ok(out); + } + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + if let Some(ref text) = query.text { + // Use individual-term matching so multi-word queries find documents + // where all words appear anywhere (not necessarily as a phrase). + let nodes = self + .search_terms(text, limit as i32) + .map_err(MemoryStoreError::from)?; + let out = nodes + .into_iter() + .map(|node| { + let (domains, domain_scores) = self.read_domain_columns(&node.id); + let mut rec = Self::node_to_record(node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + SearchResult { + record: rec, + score: 1.0, + fts_score: Some(1.0), + vector_score: None, + } + }) + .collect(); + return Ok(out); + } + } + Ok(vec![]) + } + + async fn fts_search( + &self, + text: &str, + limit: usize, + ) -> crate::storage::memory_store::MemoryStoreResult< + Vec, + > { + use crate::storage::memory_store::{MemoryStoreError, SearchResult}; + // Use individual-term matching so multi-word queries find documents + // where all words appear anywhere (not necessarily as a phrase). + let nodes = self + .search_terms(text, limit as i32) + .map_err(MemoryStoreError::from)?; + let out = nodes + .into_iter() + .map(|node| { + let (domains, domain_scores) = self.read_domain_columns(&node.id); + let mut rec = Self::node_to_record(node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + SearchResult { + record: rec, + score: 1.0, + fts_score: Some(1.0), + vector_score: None, + } + }) + .collect(); + Ok(out) + } + + async fn vector_search( + &self, + embedding: &[f32], + limit: usize, + ) -> crate::storage::memory_store::MemoryStoreResult< + Vec, + > { + use crate::storage::memory_store::{MemoryStoreError, SearchResult}; + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + let index = self + .vector_index + .lock() + .map_err(|_| MemoryStoreError::Init("Vector index lock poisoned".into()))?; + let raw_results = index + .search_with_threshold(embedding, limit, 0.0_f32) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + drop(index); + let out = raw_results + .into_iter() + .filter_map(|(node_id, score)| { + let node = self.get_node(&node_id).ok().flatten()?; + let (domains, domain_scores) = self.read_domain_columns(&node_id); + let mut rec = Self::node_to_record(node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + Some(SearchResult { + record: rec, + score: score as f64, + fts_score: None, + vector_score: Some(score as f64), + }) + }) + .collect(); + return Ok(out); + } + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + { + let _ = (embedding, limit); + Ok(vec![]) + } + } + + async fn get_scheduling( + &self, + memory_id: uuid::Uuid, + ) -> crate::storage::memory_store::MemoryStoreResult< + Option, + > { + use crate::storage::memory_store::{MemoryStoreError, SchedulingState}; + let node = self + .get_node(&memory_id.to_string()) + .map_err(MemoryStoreError::from)?; + let Some(node) = node else { + return Ok(None); + }; + Ok(Some(SchedulingState { + memory_id, + stability: node.stability, + difficulty: node.difficulty, + retrievability: node.retention_strength, + last_review: Some(node.last_accessed), + next_review: node.next_review, + reps: node.reps as u32, + lapses: node.lapses as u32, + })) + } + + async fn update_scheduling( + &self, + state: &crate::storage::memory_store::SchedulingState, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + let next_review_str = state.next_review.map(|dt| dt.to_rfc3339()); + let last_review_str = state.last_review.map(|dt| dt.to_rfc3339()); + writer + .execute( + "UPDATE knowledge_nodes SET stability=?1, difficulty=?2, retention_strength=?3, + last_accessed=?4, next_review=?5, reps=?6, lapses=?7 + WHERE id=?8", + rusqlite::params![ + state.stability, + state.difficulty, + state.retrievability, + last_review_str.as_deref().unwrap_or(""), + next_review_str, + state.reps as i64, + state.lapses as i64, + state.memory_id.to_string(), + ], + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(()) + } + + async fn get_due_memories( + &self, + before: chrono::DateTime, + limit: usize, + ) -> crate::storage::memory_store::MemoryStoreResult< + Vec<( + crate::storage::memory_store::MemoryRecord, + crate::storage::memory_store::SchedulingState, + )>, + > { + use crate::storage::memory_store::{MemoryStoreError, SchedulingState}; + let reader = self + .reader + .lock() + .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; + let before_str = before.to_rfc3339(); + let mut stmt = reader + .prepare( + "SELECT * FROM knowledge_nodes WHERE next_review <= ?1 ORDER BY next_review ASC LIMIT ?2", + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let nodes: Vec = stmt + .query_map( + rusqlite::params![before_str, limit as i64], + Self::row_to_node, + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))? + .collect::, _>>() + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + drop(stmt); + drop(reader); + let out = nodes + .into_iter() + .map(|node| { + let id_str = node.id.clone(); + let (domains, domain_scores) = self.read_domain_columns(&id_str); + let id_uuid = + uuid::Uuid::parse_str(&id_str).unwrap_or_else(|_| uuid::Uuid::new_v4()); + let state = SchedulingState { + memory_id: id_uuid, + stability: node.stability, + difficulty: node.difficulty, + retrievability: node.retention_strength, + last_review: Some(node.last_accessed), + next_review: node.next_review, + reps: node.reps as u32, + lapses: node.lapses as u32, + }; + let mut rec = Self::node_to_record(node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + (rec, state) + }) + .collect(); + Ok(out) + } + + async fn add_edge( + &self, + edge: &crate::storage::memory_store::MemoryEdge, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + let conn = ConnectionRecord { + source_id: edge.source_id.to_string(), + target_id: edge.target_id.to_string(), + strength: edge.weight, + link_type: edge.edge_type.clone(), + created_at: edge.created_at, + last_activated: edge.created_at, + activation_count: 0, + }; + self.save_connection(&conn).map_err(MemoryStoreError::from) + } + + async fn get_edges( + &self, + node_id: uuid::Uuid, + edge_type: Option<&str>, + ) -> crate::storage::memory_store::MemoryStoreResult< + Vec, + > { + use crate::storage::memory_store::{MemoryEdge, MemoryStoreError}; + let conns = self + .get_connections_for_memory(&node_id.to_string()) + .map_err(MemoryStoreError::from)?; + let edges = conns + .into_iter() + .filter(|c| edge_type.is_none_or(|t| c.link_type == t)) + .filter_map(|c| { + let src = uuid::Uuid::parse_str(&c.source_id).ok()?; + let tgt = uuid::Uuid::parse_str(&c.target_id).ok()?; + Some(MemoryEdge { + source_id: src, + target_id: tgt, + edge_type: c.link_type, + weight: c.strength, + created_at: c.created_at, + }) + }) + .collect(); + Ok(edges) + } + + async fn remove_edge( + &self, + source: uuid::Uuid, + target: uuid::Uuid, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + writer + .execute( + "DELETE FROM memory_connections WHERE source_id = ?1 AND target_id = ?2", + rusqlite::params![source.to_string(), target.to_string()], + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(()) + } + + async fn get_neighbors( + &self, + node_id: uuid::Uuid, + depth: usize, + ) -> crate::storage::memory_store::MemoryStoreResult< + Vec<(crate::storage::memory_store::MemoryRecord, f64)>, + > { + use crate::storage::memory_store::MemoryStoreError; + // Depth 0: return just the node itself if it exists. + if depth == 0 { + let node = self + .get_node(&node_id.to_string()) + .map_err(MemoryStoreError::from)? + .ok_or_else(|| MemoryStoreError::NotFound(node_id.to_string()))?; + let (domains, domain_scores) = self.read_domain_columns(&node_id.to_string()); + let mut rec = Self::node_to_record(node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + return Ok(vec![(rec, 1.0)]); + } + // BFS up to `depth` levels, capped at 256 nodes. + const MAX_NODES: usize = 256; + let mut visited: std::collections::HashMap = + std::collections::HashMap::new(); + let mut frontier: Vec<(uuid::Uuid, f64)> = vec![(node_id, 1.0)]; + visited.insert(node_id, 1.0); + for _ in 0..depth { + if visited.len() >= MAX_NODES { + break; + } + let mut next_frontier = Vec::new(); + for (current, current_weight) in frontier.iter() { + let conns = self + .get_connections_for_memory(¤t.to_string()) + .unwrap_or_default(); + for conn in conns { + let neighbor_id_str = if conn.source_id == current.to_string() { + conn.target_id + } else { + conn.source_id + }; + let Ok(nid) = uuid::Uuid::parse_str(&neighbor_id_str) else { + continue; + }; + if let std::collections::hash_map::Entry::Vacant(e) = visited.entry(nid) { + let w = current_weight * conn.strength; + e.insert(w); + next_frontier.push((nid, w)); + if visited.len() >= MAX_NODES { + break; + } + } + } + } + frontier = next_frontier; + if frontier.is_empty() { + break; + } + } + let mut result = Vec::with_capacity(visited.len()); + for (nid, weight) in visited { + let Some(node) = self.get_node(&nid.to_string()).ok().flatten() else { + continue; + }; + let (domains, domain_scores) = self.read_domain_columns(&nid.to_string()); + let mut rec = Self::node_to_record(node, None); + rec.domains = domains; + rec.domain_scores = domain_scores; + result.push((rec, weight)); + } + Ok(result) + } + + async fn list_domains( + &self, + ) -> crate::storage::memory_store::MemoryStoreResult> + { + use crate::storage::memory_store::{Domain, MemoryStoreError}; + let reader = self + .reader + .lock() + .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader + .prepare("SELECT id, label, centroid, top_terms, memory_count, created_at FROM domains ORDER BY created_at ASC") + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let rows = stmt + .query_map([], |row| { + let id: String = row.get(0)?; + let label: String = row.get(1)?; + let centroid_bytes: Option> = row.get(2)?; + let top_terms_json: String = row.get(3)?; + let memory_count: i64 = row.get(4)?; + let created_at_str: String = row.get(5)?; + Ok(( + id, + label, + centroid_bytes, + top_terms_json, + memory_count, + created_at_str, + )) + }) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let mut result = Vec::new(); + for row in rows { + let (id, label, centroid_bytes, top_terms_json, memory_count, created_at_str) = + row.map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let centroid: Vec = centroid_bytes + .map(|b| { + b.chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + }) + .unwrap_or_default(); + let top_terms: Vec = serde_json::from_str(&top_terms_json).unwrap_or_default(); + let created_at = chrono::DateTime::parse_from_rfc3339(&created_at_str) + .map(|dt| dt.with_timezone(&chrono::Utc)) + .unwrap_or_else(|_| Utc::now()); + result.push(Domain { + id, + label, + centroid, + top_terms, + memory_count: memory_count as usize, + created_at, + }); + } + Ok(result) + } + + async fn get_domain( + &self, + id: &str, + ) -> crate::storage::memory_store::MemoryStoreResult> + { + use crate::storage::memory_store::{Domain, MemoryStoreError}; + let reader = self + .reader + .lock() + .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; + let result: Option<(String, String, Option>, String, i64, String)> = reader + .query_row( + "SELECT id, label, centroid, top_terms, memory_count, created_at FROM domains WHERE id = ?1", + rusqlite::params![id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .optional() + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let Some((id, label, centroid_bytes, top_terms_json, memory_count, created_at_str)) = + result + else { + return Ok(None); + }; + let centroid: Vec = centroid_bytes + .map(|b| { + b.chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + }) + .unwrap_or_default(); + let top_terms: Vec = serde_json::from_str(&top_terms_json).unwrap_or_default(); + let created_at = chrono::DateTime::parse_from_rfc3339(&created_at_str) + .map(|dt| dt.with_timezone(&chrono::Utc)) + .unwrap_or_else(|_| Utc::now()); + Ok(Some(Domain { + id, + label, + centroid, + top_terms, + memory_count: memory_count as usize, + created_at, + })) + } + + async fn upsert_domain( + &self, + domain: &crate::storage::memory_store::Domain, + ) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + let centroid_bytes: Vec = domain + .centroid + .iter() + .flat_map(|f| f.to_le_bytes()) + .collect(); + let top_terms_json = + serde_json::to_string(&domain.top_terms).unwrap_or_else(|_| "[]".to_string()); + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + writer + .execute( + "INSERT INTO domains (id, label, centroid, top_terms, memory_count, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(id) DO UPDATE SET + label = excluded.label, + centroid = excluded.centroid, + top_terms = excluded.top_terms, + memory_count = excluded.memory_count", + rusqlite::params![ + domain.id, + domain.label, + centroid_bytes, + top_terms_json, + domain.memory_count as i64, + domain.created_at.to_rfc3339(), + ], + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(()) + } + + async fn delete_domain(&self, id: &str) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + writer + .execute("DELETE FROM domains WHERE id = ?1", rusqlite::params![id]) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(()) + } + + async fn classify( + &self, + _embedding: &[f32], + ) -> crate::storage::memory_store::MemoryStoreResult> { + // Phase 1 stub: no centroids yet. Phase 4 wires the full soft-assignment pass. + Ok(vec![]) + } + + async fn count(&self) -> crate::storage::memory_store::MemoryStoreResult { + use crate::storage::memory_store::MemoryStoreError; + let reader = self + .reader + .lock() + .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; + let n: i64 = reader + .query_row("SELECT COUNT(*) FROM knowledge_nodes", [], |row| row.get(0)) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(n as usize) + } + + async fn get_stats( + &self, + ) -> crate::storage::memory_store::MemoryStoreResult + { + use crate::storage::memory_store::{MemoryStoreError, StoreStats}; + let reader = self + .reader + .lock() + .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; + let total: i64 = reader + .query_row("SELECT COUNT(*) FROM knowledge_nodes", [], |row| row.get(0)) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let with_emb: i64 = reader + .query_row( + "SELECT COUNT(*) FROM knowledge_nodes WHERE has_embedding = 1", + [], + |row| row.get(0), + ) + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let total_edges: i64 = reader + .query_row("SELECT COUNT(*) FROM memory_connections", [], |row| { + row.get(0) + }) + .unwrap_or(0); + let total_domains: i64 = reader + .query_row("SELECT COUNT(*) FROM domains", [], |row| row.get(0)) + .unwrap_or(0); + let model_row: Option<(String, i64)> = reader + .query_row( + "SELECT name, dimension FROM embedding_model WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + let (model_name, model_dim) = model_row + .map(|(n, d)| (Some(n), Some(d as usize))) + .unwrap_or((None, None)); + Ok(StoreStats { + total_memories: total as usize, + memories_with_embeddings: with_emb as usize, + total_edges: total_edges as usize, + total_domains: total_domains as usize, + registered_model_name: model_name, + registered_model_dim: model_dim, + }) + } + + async fn vacuum(&self) -> crate::storage::memory_store::MemoryStoreResult<()> { + use crate::storage::memory_store::MemoryStoreError; + let writer = self + .writer + .lock() + .map_err(|_| MemoryStoreError::Init("Writer lock poisoned".into()))?; + writer + .execute_batch("VACUUM;") + .map_err(|e| MemoryStoreError::Backend(e.to_string()))?; + Ok(()) + } +} + // ============================================================================ // TESTS // ============================================================================ @@ -8294,6 +9307,9 @@ mod tests { use super::*; use crate::advanced::{MatchClass, MergePolicy}; use tempfile::tempdir; + // The public struct was renamed from Storage to SqliteMemoryStore; this + // alias keeps all existing tests compiling without modification. + use SqliteMemoryStore as Storage; fn create_test_storage() -> Storage { let dir = tempdir().unwrap(); @@ -10322,6 +11338,187 @@ mod tests { v } + // ========================================================================= + // Phase 1 trait-method unit tests + // ========================================================================= + use crate::storage::memory_store::{ + MemoryEdge, MemoryRecord, MemoryStore, MemoryStoreError, ModelSignature, SchedulingState, + }; + + fn make_record(content: &str) -> MemoryRecord { + MemoryRecord { + id: uuid::Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: content.to_string(), + node_type: "fact".to_string(), + tags: vec!["test".to_string()], + embedding: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + metadata: serde_json::json!({}), + } + } + + fn rt() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() + } + + #[test] + fn trait_init_is_idempotent() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + s.init().await.unwrap(); + s.init().await.unwrap(); + }); + } + + #[test] + fn trait_health_check_reports_healthy_on_fresh_db() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let h = s.health_check().await.unwrap(); + assert!(matches!( + h, + crate::storage::memory_store::HealthStatus::Healthy + )); + }); + } + + #[test] + fn trait_register_model_first_write_succeeds() { + let s = create_test_storage(); + let sig = ModelSignature { + name: "test-model".to_string(), + dimension: 256, + hash: "a".repeat(64), + }; + let rt = rt(); + rt.block_on(async { + s.register_model(&sig).await.unwrap(); + let got = s.registered_model().await.unwrap(); + assert_eq!(got, Some(sig)); + }); + } + + #[test] + fn trait_register_model_mismatched_write_refused() { + let s = create_test_storage(); + let sig = ModelSignature { + name: "model-a".to_string(), + dimension: 256, + hash: "a".repeat(64), + }; + let sig2 = ModelSignature { + name: "model-b".to_string(), + dimension: 256, + hash: "b".repeat(64), + }; + let rt = rt(); + rt.block_on(async { + s.register_model(&sig).await.unwrap(); + let err = s.register_model(&sig2).await.unwrap_err(); + assert!(matches!(err, MemoryStoreError::ModelMismatch { .. })); + }); + } + + #[test] + fn trait_register_model_same_signature_idempotent() { + let s = create_test_storage(); + let sig = ModelSignature { + name: "test-model".to_string(), + dimension: 256, + hash: "a".repeat(64), + }; + let rt = rt(); + rt.block_on(async { + s.register_model(&sig).await.unwrap(); + s.register_model(&sig).await.unwrap(); // second call must not error + }); + } + + #[test] + fn trait_insert_returns_uuid() { + let s = create_test_storage(); + let rec = make_record("test content"); + let expected_id = rec.id; + let rt = rt(); + rt.block_on(async { + let got = s.insert(&rec).await.unwrap(); + assert_eq!(got, expected_id); + }); + } + + #[test] + fn trait_get_missing_returns_none() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let got = s.get(uuid::Uuid::new_v4()).await.unwrap(); + assert!(got.is_none()); + }); + } + + #[test] + fn trait_get_after_insert_round_trip() { + let s = create_test_storage(); + let rec = make_record("round trip content"); + let id = rec.id; + let rt = rt(); + rt.block_on(async { + s.insert(&rec).await.unwrap(); + let got = s.get(id).await.unwrap().unwrap(); + assert_eq!(got.content, "round trip content"); + assert_eq!(got.node_type, "fact"); + assert!(got.domains.is_empty()); + assert!(got.domain_scores.is_empty()); + }); + } + + #[test] + fn trait_update_modifies_content() { + let s = create_test_storage(); + let rec = make_record("original content"); + let id = rec.id; + let rt = rt(); + rt.block_on(async { + s.insert(&rec).await.unwrap(); + let mut updated = s.get(id).await.unwrap().unwrap(); + updated.content = "updated content".to_string(); + s.update(&updated).await.unwrap(); + let got = s.get(id).await.unwrap().unwrap(); + assert_eq!(got.content, "updated content"); + }); + } + + #[test] + fn trait_delete_removes_record() { + let s = create_test_storage(); + let rec = make_record("to be deleted"); + let id = rec.id; + let rt = rt(); + rt.block_on(async { + s.insert(&rec).await.unwrap(); + s.delete(id).await.unwrap(); + let got = s.get(id).await.unwrap(); + assert!(got.is_none()); + }); + } + + #[test] + fn trait_fts_search_returns_tokens_match() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec = make_record("mitochondria powerhouse cell energy"); + s.insert(&rec).await.unwrap(); + let results = s.fts_search("mitochondria", 10).await.unwrap(); + assert!(!results.is_empty()); + }); + } + #[cfg(all(feature = "embeddings", feature = "vector-search"))] #[test] fn test_merge_candidates_threshold_classification() { @@ -10592,4 +11789,337 @@ mod tests { let storage = create_test_storage(); assert!(storage.set_protected("does-not-exist", true).is_err()); } + + #[test] + fn trait_hybrid_search_multi_word_via_insert() { + // Verify that hybrid_search finds records inserted via the trait insert() + // even when no embedding is present (keyword path via terms matching). + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec = make_record("quantum entanglement superposition physics"); + s.insert(&rec).await.unwrap(); + let results = s.hybrid_search("quantum physics", 10, 0.3, 0.7).unwrap(); + assert!( + !results.is_empty(), + "hybrid_search must find record containing 'quantum' and 'physics'" + ); + }); + } + + #[test] + fn trait_scheduling_round_trip() { + let s = create_test_storage(); + let rec = make_record("fsrs scheduling test"); + let id = rec.id; + let rt = rt(); + rt.block_on(async { + s.insert(&rec).await.unwrap(); + let state = SchedulingState { + memory_id: id, + stability: 5.0, + difficulty: 0.4, + retrievability: 0.8, + last_review: Some(chrono::Utc::now()), + next_review: Some(chrono::Utc::now() + chrono::Duration::days(7)), + reps: 3, + lapses: 1, + }; + s.update_scheduling(&state).await.unwrap(); + let got = s.get_scheduling(id).await.unwrap().unwrap(); + assert!((got.stability - 5.0).abs() < 0.01); + }); + } + + #[test] + fn trait_get_scheduling_missing_returns_none() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let got = s.get_scheduling(uuid::Uuid::new_v4()).await.unwrap(); + assert!(got.is_none()); + }); + } + + #[test] + fn trait_get_due_memories_returns_in_order() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + for i in 0..3usize { + let rec = make_record(&format!("due memory {i}")); + let id = rec.id; + s.insert(&rec).await.unwrap(); + let state = SchedulingState { + memory_id: id, + stability: 1.0, + difficulty: 0.3, + retrievability: 0.5, + last_review: Some(chrono::Utc::now()), + next_review: Some(chrono::Utc::now() - chrono::Duration::days(3 - i as i64)), + reps: 1, + lapses: 0, + }; + s.update_scheduling(&state).await.unwrap(); + } + let due = s.get_due_memories(chrono::Utc::now(), 10).await.unwrap(); + assert_eq!(due.len(), 3); + }); + } + + #[test] + fn trait_add_edge_is_idempotent() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec_a = make_record("node a"); + let rec_b = make_record("node b"); + let id_a = rec_a.id; + let id_b = rec_b.id; + s.insert(&rec_a).await.unwrap(); + s.insert(&rec_b).await.unwrap(); + let edge = MemoryEdge { + source_id: id_a, + target_id: id_b, + edge_type: "semantic".to_string(), + weight: 0.9, + created_at: chrono::Utc::now(), + }; + s.add_edge(&edge).await.unwrap(); + s.add_edge(&edge).await.unwrap(); // idempotent + let edges = s.get_edges(id_a, None).await.unwrap(); + let filtered: Vec<_> = edges + .iter() + .filter(|e| e.source_id == id_a && e.target_id == id_b) + .collect(); + assert_eq!(filtered.len(), 1, "edge must not be duplicated"); + }); + } + + #[test] + fn trait_get_edges_filters_by_type() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec_a = make_record("filter a"); + let rec_b = make_record("filter b"); + let id_a = rec_a.id; + let id_b = rec_b.id; + s.insert(&rec_a).await.unwrap(); + s.insert(&rec_b).await.unwrap(); + let edge = MemoryEdge { + source_id: id_a, + target_id: id_b, + edge_type: "causal".to_string(), + weight: 0.5, + created_at: chrono::Utc::now(), + }; + s.add_edge(&edge).await.unwrap(); + let causal = s.get_edges(id_a, Some("causal")).await.unwrap(); + assert!(!causal.is_empty()); + let semantic = s.get_edges(id_a, Some("semantic")).await.unwrap(); + assert!(semantic.is_empty()); + }); + } + + #[test] + fn trait_remove_edge_deletes_single() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec_a = make_record("rm edge a"); + let rec_b = make_record("rm edge b"); + let id_a = rec_a.id; + let id_b = rec_b.id; + s.insert(&rec_a).await.unwrap(); + s.insert(&rec_b).await.unwrap(); + let edge = MemoryEdge { + source_id: id_a, + target_id: id_b, + edge_type: "semantic".to_string(), + weight: 0.7, + created_at: chrono::Utc::now(), + }; + s.add_edge(&edge).await.unwrap(); + s.remove_edge(id_a, id_b).await.unwrap(); + let edges = s.get_edges(id_a, None).await.unwrap(); + assert!(edges.is_empty()); + }); + } + + #[test] + fn trait_get_neighbors_bfs_depth_zero_returns_self_only() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec = make_record("depth zero"); + let id = rec.id; + s.insert(&rec).await.unwrap(); + let neighbors = s.get_neighbors(id, 0).await.unwrap(); + assert_eq!(neighbors.len(), 1); + assert_eq!(neighbors[0].0.id, id); + }); + } + + #[test] + fn trait_get_neighbors_bfs_depth_two_expands() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let rec_a = make_record("bfs node a"); + let rec_b = make_record("bfs node b"); + let rec_c = make_record("bfs node c"); + let id_a = rec_a.id; + let id_b = rec_b.id; + let id_c = rec_c.id; + s.insert(&rec_a).await.unwrap(); + s.insert(&rec_b).await.unwrap(); + s.insert(&rec_c).await.unwrap(); + s.add_edge(&MemoryEdge { + source_id: id_a, + target_id: id_b, + edge_type: "semantic".to_string(), + weight: 1.0, + created_at: chrono::Utc::now(), + }) + .await + .unwrap(); + s.add_edge(&MemoryEdge { + source_id: id_b, + target_id: id_c, + edge_type: "semantic".to_string(), + weight: 1.0, + created_at: chrono::Utc::now(), + }) + .await + .unwrap(); + let neighbors = s.get_neighbors(id_a, 2).await.unwrap(); + let ids: Vec = neighbors.iter().map(|(r, _)| r.id).collect(); + assert!(ids.contains(&id_a)); + assert!(ids.contains(&id_b)); + assert!(ids.contains(&id_c)); + }); + } + + #[test] + fn trait_list_domains_empty_in_phase_1() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let domains = s.list_domains().await.unwrap(); + assert!(domains.is_empty()); + }); + } + + #[test] + fn trait_upsert_then_get_domain_round_trip() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let domain = crate::storage::memory_store::Domain { + id: "dev".to_string(), + label: "Development".to_string(), + centroid: vec![0.1, 0.2, 0.3], + top_terms: vec!["rust".to_string(), "code".to_string()], + memory_count: 42, + created_at: chrono::Utc::now(), + }; + s.upsert_domain(&domain).await.unwrap(); + let got = s.get_domain("dev").await.unwrap().unwrap(); + assert_eq!(got.id, "dev"); + assert_eq!(got.memory_count, 42); + }); + } + + #[test] + fn trait_delete_domain_idempotent() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + s.delete_domain("nonexistent").await.unwrap(); + s.delete_domain("nonexistent").await.unwrap(); + }); + } + + #[test] + fn trait_classify_with_no_domains_returns_empty() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + let result = s.classify(&[0.1, 0.2, 0.3]).await.unwrap(); + assert!(result.is_empty()); + }); + } + + #[test] + fn trait_count_matches_insert_count() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + for i in 0..5usize { + let rec = make_record(&format!("count test {i}")); + s.insert(&rec).await.unwrap(); + } + assert_eq!(s.count().await.unwrap(), 5); + }); + } + + #[test] + fn trait_get_stats_reports_registered_model() { + let s = create_test_storage(); + let sig = ModelSignature { + name: "test-model".to_string(), + dimension: 256, + hash: "c".repeat(64), + }; + let rt = rt(); + rt.block_on(async { + use crate::storage::memory_store::MemoryStore; + // Cast to &dyn MemoryStore so the async trait method is called + // instead of the inherent sync get_stats() on SqliteMemoryStore. + let dyn_s: &dyn MemoryStore = &s; + dyn_s.register_model(&sig).await.unwrap(); + let stats = dyn_s.get_stats().await.unwrap(); + assert_eq!(stats.registered_model_name, Some("test-model".to_string())); + assert_eq!(stats.registered_model_dim, Some(256)); + }); + } + + #[test] + fn trait_vacuum_succeeds() { + let s = create_test_storage(); + let rt = rt(); + rt.block_on(async { + s.vacuum().await.unwrap(); + }); + } + + #[test] + fn trait_insert_refuses_dimension_mismatch() { + let s = create_test_storage(); + let sig = ModelSignature { + name: "test-model".to_string(), + dimension: 256, + hash: "d".repeat(64), + }; + let rt = rt(); + rt.block_on(async { + s.register_model(&sig).await.unwrap(); + // Build a record with wrong dimension (512 instead of 256) and + // declare the model signature in metadata + let mut rec = make_record("dimension mismatch"); + rec.embedding = Some(vec![0.0f32; 512]); + rec.metadata = serde_json::json!({ + "model_name": "test-model", + "model_dim": 256_u64, + "model_hash": "d".repeat(64), + }); + let err = s.insert(&rec).await.unwrap_err(); + assert!( + matches!(err, MemoryStoreError::InvalidInput(_)), + "expected InvalidInput, got {:?}", + err + ); + }); + } } diff --git a/tests/phase_1/Cargo.toml b/tests/phase_1/Cargo.toml new file mode 100644 index 0000000..80a9bff --- /dev/null +++ b/tests/phase_1/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "vestige-phase-1-tests" +version = "0.0.1" +edition = "2024" +publish = false + +[dependencies] +vestige-core = { path = "../../crates/vestige-core" } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tempfile = "3" +uuid = { version = "1", features = ["v4"] } +chrono = "0.4" +serde_json = "1" +rusqlite = { version = "0.38", features = ["bundled"] } + +[[test]] +name = "trait_round_trip" +path = "trait_round_trip.rs" + +[[test]] +name = "embedding_model_registry" +path = "embedding_model_registry.rs" + +[[test]] +name = "domain_column_migration" +path = "domain_column_migration.rs" + +[[test]] +name = "cognitive_module_isolation" +path = "cognitive_module_isolation.rs" + +[[test]] +name = "send_bound_variant" +path = "send_bound_variant.rs" + +[[test]] +name = "embedder_trait" +path = "embedder_trait.rs" diff --git a/tests/phase_1/cognitive_module_isolation.rs b/tests/phase_1/cognitive_module_isolation.rs new file mode 100644 index 0000000..0ff94b8 --- /dev/null +++ b/tests/phase_1/cognitive_module_isolation.rs @@ -0,0 +1,143 @@ +//! Phase 1 integration tests: cognitive modules compile against Arc. +//! The key goal is a compile-time gate: if any module still typed against +//! SqliteMemoryStore concretely, this would fail to compile. + +use chrono::Utc; +use std::sync::Arc; +use tempfile::tempdir; +use uuid::Uuid; +use vestige_core::storage::{MemoryEdge, MemoryRecord, MemoryStore, SqliteMemoryStore}; + +fn make_store() -> Arc { + let dir = tempdir().unwrap(); + let db = dir.path().join("test.db"); + std::mem::forget(dir); + Arc::new(SqliteMemoryStore::new(Some(db)).expect("create")) +} + +fn make_record(content: &str) -> MemoryRecord { + MemoryRecord { + id: Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: content.to_string(), + node_type: "fact".to_string(), + tags: vec!["isolation-test".to_string()], + embedding: None, + created_at: Utc::now(), + updated_at: Utc::now(), + metadata: serde_json::json!({}), + } +} + +/// Ensure the store: Arc call pattern compiles and runs through +/// a representative method from every cognitive module group. +#[tokio::test] +async fn all_modules_compile_against_dyn_store() { + let store: Arc = make_store(); + + // CRUD via trait + let rec = make_record("cognitive isolation test"); + let id = store.insert(&rec).await.expect("insert via dyn trait"); + let got = store + .get(id) + .await + .expect("get via dyn trait") + .expect("exists"); + assert_eq!(got.content, "cognitive isolation test"); + + // Graph edges via trait + let rec2 = make_record("linked node"); + let id2 = store.insert(&rec2).await.expect("insert 2"); + store + .add_edge(&MemoryEdge { + source_id: id, + target_id: id2, + edge_type: "semantic".to_string(), + weight: 0.8, + created_at: Utc::now(), + }) + .await + .expect("add_edge via dyn trait"); + + let edges = store + .get_edges(id, None) + .await + .expect("get_edges via dyn trait"); + assert!(!edges.is_empty()); + + // Search via trait + let results = store + .fts_search("cognitive", 5) + .await + .expect("fts_search via dyn trait"); + assert!(!results.is_empty()); + + // Stats and count via trait + let count = store.count().await.expect("count via dyn trait"); + assert!(count >= 2); + + let stats = store.get_stats().await.expect("get_stats via dyn trait"); + assert!(stats.total_memories >= 2); +} + +#[tokio::test] +async fn spreading_activation_traverses_via_trait() { + let store: Arc = make_store(); + let rec_a = make_record("spreading activation source"); + let rec_b = make_record("spreading activation neighbor"); + let id_a = rec_a.id; + let id_b = rec_b.id; + store.insert(&rec_a).await.expect("insert a"); + store.insert(&rec_b).await.expect("insert b"); + store + .add_edge(&MemoryEdge { + source_id: id_a, + target_id: id_b, + edge_type: "semantic".to_string(), + weight: 0.9, + created_at: Utc::now(), + }) + .await + .expect("add edge"); + + // get_neighbors simulates the spreading activation traversal path + let neighbors = store.get_neighbors(id_a, 1).await.expect("get_neighbors"); + let ids: Vec = neighbors.iter().map(|(r, _)| r.id).collect(); + assert!(ids.contains(&id_a)); + assert!(ids.contains(&id_b)); +} + +#[tokio::test] +async fn synaptic_tagging_consumes_records_via_trait() { + // Build a MemoryRecord from trait-returned data and exercise the + // SynapticTaggingSystem pipeline (constructing CapturedMemory from store data). + let store: Arc = make_store(); + let rec = make_record("synaptic tagging test memory"); + let id = store.insert(&rec).await.expect("insert"); + let got = store.get(id).await.expect("get").expect("exists"); + // The important thing is we got a MemoryRecord back from the dyn trait; + // SynapticTaggingSystem would take this record as input. + assert_eq!(got.id, id); + assert!(!got.content.is_empty()); +} + +#[tokio::test] +async fn hippocampal_index_built_from_store() { + // Exercise the fts_search -> HippocampalIndex indexing path. + let store: Arc = make_store(); + for i in 0..5usize { + let rec = make_record(&format!("hippocampal indexing topic {i}")); + store.insert(&rec).await.expect("insert"); + } + let results = store + .fts_search("hippocampal indexing", 10) + .await + .expect("fts_search"); + // Verify we get results and they have the correct fields + assert!(!results.is_empty()); + for r in &results { + assert!(!r.record.content.is_empty()); + assert!(r.score >= 0.0); + } +} diff --git a/tests/phase_1/domain_column_migration.rs b/tests/phase_1/domain_column_migration.rs new file mode 100644 index 0000000..67e318b --- /dev/null +++ b/tests/phase_1/domain_column_migration.rs @@ -0,0 +1,161 @@ +//! Phase 1 integration tests: domain column migration and schema upgrade. + +use std::sync::Arc; +use tempfile::tempdir; +use uuid::Uuid; +use vestige_core::storage::{MemoryRecord, MemoryStore, SqliteMemoryStore}; + +#[tokio::test] +async fn fresh_db_has_v12_schema() { + let dir = tempdir().unwrap(); + let db = dir.path().join("fresh.db"); + let _store = SqliteMemoryStore::new(Some(db.clone())).expect("create"); + // Open a raw connection and check pragma + let conn = rusqlite::Connection::open(&db).expect("open"); + let cols: Vec = { + let mut stmt = conn.prepare("PRAGMA table_info(knowledge_nodes)").unwrap(); + stmt.query_map([], |row| row.get::<_, String>(1)) + .unwrap() + .map(|r| r.unwrap()) + .collect() + }; + assert!( + cols.contains(&"domains".to_string()), + "domains column must exist: {:?}", + cols + ); + assert!( + cols.contains(&"domain_scores".to_string()), + "domain_scores column must exist" + ); +} + +#[tokio::test] +async fn v11_db_upgrades_cleanly() { + use vestige_core::storage::MIGRATIONS; + let dir = tempdir().unwrap(); + let db = dir.path().join("v11.db"); + // Create DB with V11 migrations only + { + let conn = rusqlite::Connection::open(&db).expect("open"); + for m in MIGRATIONS.iter().filter(|m| m.version <= 11) { + conn.execute_batch(m.up).expect("apply migration"); + } + // Insert 5 rows under V11 schema + for i in 0..5usize { + conn.execute( + "INSERT INTO knowledge_nodes (id, content, node_type, created_at, updated_at, \ + last_accessed, stability, difficulty, reps, lapses, learning_state, \ + storage_strength, retrieval_strength, retention_strength, \ + next_review, scheduled_days, has_embedding) \ + VALUES (?1, ?2, 'fact', datetime('now'), datetime('now'), datetime('now'), \ + 1.0, 0.3, 0, 0, 'new', 1.0, 1.0, 1.0, datetime('now'), 1, 0)", + rusqlite::params![format!("pre-v12-{i}"), format!("content {i}"),], + ) + .expect("insert pre-v12 row"); + } + } + // Upgrade by opening through SqliteMemoryStore (triggers full migration) + let _store = SqliteMemoryStore::new(Some(db.clone())).expect("open with v12"); + // Check all 5 rows have empty domains/domain_scores + let conn = rusqlite::Connection::open(&db).expect("open raw"); + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM knowledge_nodes WHERE domains='[]' AND domain_scores='{}'", + [], + |row| row.get(0), + ) + .expect("count"); + assert_eq!( + count, 5, + "all pre-v12 rows must have empty domains/domain_scores" + ); +} + +#[tokio::test] +async fn empty_domains_serialize_as_brackets() { + let dir = tempdir().unwrap(); + let db = dir.path().join("empty_domains.db"); + let store = SqliteMemoryStore::new(Some(db.clone())).expect("create"); + let rec = MemoryRecord { + id: Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: "test content".to_string(), + node_type: "fact".to_string(), + tags: vec![], + embedding: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + metadata: serde_json::json!({}), + }; + store.insert(&rec).await.expect("insert"); + // Check raw sqlite value + let conn = rusqlite::Connection::open(&db).expect("open raw"); + let (domains, domain_scores): (String, String) = conn + .query_row( + "SELECT domains, domain_scores FROM knowledge_nodes LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .expect("query"); + assert_eq!( + domains, "[]", + "empty domains should store as '[]', not NULL" + ); + assert_eq!( + domain_scores, "{}", + "empty domain_scores should store as '{{}}'" + ); +} + +#[tokio::test] +async fn populated_domains_round_trip() { + let dir = tempdir().unwrap(); + let db = dir.path().join("populated.db"); + let store: Arc = Arc::new(SqliteMemoryStore::new(Some(db)).expect("create")); + let mut rec = MemoryRecord { + id: Uuid::new_v4(), + domains: vec!["dev".to_string(), "infra".to_string()], + domain_scores: { + let mut m = std::collections::HashMap::new(); + m.insert("dev".to_string(), 0.82); + m.insert("infra".to_string(), 0.71); + m + }, + content: "populated domains test".to_string(), + node_type: "fact".to_string(), + tags: vec![], + embedding: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + metadata: serde_json::json!({}), + }; + let id = store.insert(&rec).await.expect("insert"); + // Update the domains via update() + rec.id = id; + store.update(&rec).await.expect("update with domains"); + // Read back and verify + let got = store.get(id).await.expect("get").expect("exists"); + let mut expected_domains = got.domains.clone(); + expected_domains.sort(); + assert_eq!(expected_domains, vec!["dev", "infra"]); + assert!((got.domain_scores["dev"] - 0.82).abs() < 0.001); + assert!((got.domain_scores["infra"] - 0.71).abs() < 0.001); +} + +#[tokio::test] +async fn domains_table_exists() { + let dir = tempdir().unwrap(); + let db = dir.path().join("domains_table.db"); + let _store = SqliteMemoryStore::new(Some(db.clone())).expect("create"); + let conn = rusqlite::Connection::open(&db).expect("open raw"); + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='domains'", + [], + |row| row.get(0), + ) + .expect("query"); + assert_eq!(count, 1, "domains table must exist after V12 migration"); +} diff --git a/tests/phase_1/embedder_trait.rs b/tests/phase_1/embedder_trait.rs new file mode 100644 index 0000000..9e96da1 --- /dev/null +++ b/tests/phase_1/embedder_trait.rs @@ -0,0 +1,43 @@ +//! Phase 1 integration tests: Embedder trait and FastembedEmbedder. + +use std::sync::Arc; +use tempfile::tempdir; +use vestige_core::embedder::{Embedder, FastembedEmbedder}; +use vestige_core::storage::MemoryStore; +use vestige_core::storage::SqliteMemoryStore; + +fn make_store() -> Arc { + let dir = tempdir().unwrap(); + let db = dir.path().join("test.db"); + std::mem::forget(dir); + Arc::new(SqliteMemoryStore::new(Some(db)).expect("create")) +} + +#[tokio::test] +async fn fastembed_implements_embedder_trait() { + // The key test: `Box` compiles + let e: Box = Box::new(FastembedEmbedder::new()); + assert_eq!(e.dimension(), 256, "dimension must be 256"); + assert!(!e.model_name().is_empty(), "model_name must not be empty"); + assert!(!e.model_hash().is_empty(), "model_hash must not be empty"); + assert_eq!(e.model_hash().len(), 64, "hash must be 64 hex chars"); +} + +#[tokio::test] +async fn signature_matches_memory_store_registry() { + let e = FastembedEmbedder::new(); + let sig = e.signature(); + let store = make_store(); + store + .register_model(&sig) + .await + .expect("register via Embedder::signature"); + let got = store + .registered_model() + .await + .expect("registered_model") + .expect("Some"); + assert_eq!(got.name, sig.name); + assert_eq!(got.dimension, sig.dimension); + assert_eq!(got.hash, sig.hash); +} diff --git a/tests/phase_1/embedding_model_registry.rs b/tests/phase_1/embedding_model_registry.rs new file mode 100644 index 0000000..3c001ea --- /dev/null +++ b/tests/phase_1/embedding_model_registry.rs @@ -0,0 +1,148 @@ +//! Phase 1 integration tests: embedding model registry. + +use std::sync::Arc; +use tempfile::tempdir; +use uuid::Uuid; +use vestige_core::storage::{ + MemoryRecord, MemoryStore, MemoryStoreError, ModelSignature, SqliteMemoryStore, +}; + +fn make_store() -> Arc { + let dir = tempdir().unwrap(); + let db = dir.path().join("test.db"); + std::mem::forget(dir); + let store = SqliteMemoryStore::new(Some(db)).expect("create store"); + Arc::new(store) +} + +fn sig_a() -> ModelSignature { + ModelSignature { + name: "model-a".to_string(), + dimension: 256, + hash: "a".repeat(64), + } +} + +fn sig_b() -> ModelSignature { + ModelSignature { + name: "model-b".to_string(), + dimension: 256, + hash: "b".repeat(64), + } +} + +fn record_without_embedding() -> MemoryRecord { + MemoryRecord { + id: Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: "plain text memory".to_string(), + node_type: "fact".to_string(), + tags: vec![], + embedding: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + metadata: serde_json::json!({}), + } +} + +#[tokio::test] +async fn first_embedded_insert_auto_registers() { + // fresh store; register a model, then check registered_model() returns Some + let store = make_store(); + let sig = sig_a(); + store.register_model(&sig).await.expect("register"); + let got = store.registered_model().await.expect("registered_model"); + assert_eq!(got, Some(sig)); +} + +#[tokio::test] +async fn second_insert_with_same_signature_succeeds() { + let store = make_store(); + let sig = sig_a(); + store.register_model(&sig).await.expect("first register"); + store + .register_model(&sig) + .await + .expect("second register idempotent"); +} + +#[tokio::test] +async fn second_insert_with_different_dimension_refused() { + let store = make_store(); + let sig = sig_a(); // dim 256 + store.register_model(&sig).await.expect("register 256"); + // Try inserting a 512-dim vector into a store registered for 256 + let mut rec = record_without_embedding(); + rec.embedding = Some(vec![0.0f32; 512]); + rec.metadata = serde_json::json!({ + "model_name": "model-a", + "model_dim": 256_u64, + "model_hash": "a".repeat(64), + }); + let err = store.insert(&rec).await.unwrap_err(); + assert!( + matches!(err, MemoryStoreError::InvalidInput(_)), + "expected InvalidInput for dim mismatch, got {:?}", + err + ); +} + +#[tokio::test] +async fn second_insert_with_different_model_name_refused() { + let store = make_store(); + store.register_model(&sig_a()).await.expect("register a"); + let err = store.register_model(&sig_b()).await.unwrap_err(); + assert!( + matches!(err, MemoryStoreError::ModelMismatch { .. }), + "expected ModelMismatch, got {:?}", + err + ); +} + +#[tokio::test] +async fn second_insert_with_different_hash_refused() { + let store = make_store(); + let sig = sig_a(); + store.register_model(&sig).await.expect("register"); + let sig_diff_hash = ModelSignature { + name: "model-a".to_string(), + dimension: 256, + hash: "c".repeat(64), // different hash + }; + let err = store.register_model(&sig_diff_hash).await.unwrap_err(); + assert!( + matches!(err, MemoryStoreError::ModelMismatch { .. }), + "expected ModelMismatch for different hash, got {:?}", + err + ); +} + +#[tokio::test] +async fn no_embedding_insert_allowed_before_registration() { + let store = make_store(); + // registered_model() should be None + assert!( + store + .registered_model() + .await + .expect("registered_model") + .is_none() + ); + // A plain text memory without an embedding must insert successfully + let rec = record_without_embedding(); + store + .insert(&rec) + .await + .expect("plain insert before registration"); +} + +#[tokio::test] +async fn stats_reports_registered_model_after_first_write() { + let store = make_store(); + let sig = sig_a(); + store.register_model(&sig).await.expect("register"); + let stats = store.get_stats().await.expect("stats"); + assert_eq!(stats.registered_model_name, Some("model-a".to_string())); + assert_eq!(stats.registered_model_dim, Some(256)); +} diff --git a/tests/phase_1/send_bound_variant.rs b/tests/phase_1/send_bound_variant.rs new file mode 100644 index 0000000..c0f02ef --- /dev/null +++ b/tests/phase_1/send_bound_variant.rs @@ -0,0 +1,99 @@ +//! Phase 1 integration tests: Arc moves across tokio::spawn. +//! +//! This verifies that `#[trait_variant::make(MemoryStore: Send)]` actually +//! produces a Send-bound future so Arc is movable. + +use chrono::Utc; +use std::sync::Arc; +use tempfile::tempdir; +use uuid::Uuid; +use vestige_core::storage::{MemoryRecord, MemoryStore, SqliteMemoryStore}; + +fn make_store() -> Arc { + let dir = tempdir().unwrap(); + let db = dir.path().join("send_test.db"); + std::mem::forget(dir); + Arc::new(SqliteMemoryStore::new(Some(db)).expect("create")) +} + +fn make_record(content: &str) -> MemoryRecord { + MemoryRecord { + id: Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: content.to_string(), + node_type: "fact".to_string(), + tags: vec![], + embedding: None, + created_at: Utc::now(), + updated_at: Utc::now(), + metadata: serde_json::json!({}), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn arc_dyn_memory_store_moves_across_tokio_tasks() { + let store: Arc = make_store(); + let mut handles = Vec::new(); + for t in 0..16usize { + let store = Arc::clone(&store); + let handle = tokio::spawn(async move { + for i in 0..10usize { + let rec = make_record(&format!("task {t} memory {i}")); + store.insert(&rec).await.expect("insert in spawned task"); + } + }); + handles.push(handle); + } + for h in handles { + h.await.expect("task completed without panic"); + } + let count = store.count().await.expect("count"); + assert_eq!(count, 160, "all 16*10 inserts must be counted"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_readers_one_writer() { + let store: Arc = make_store(); + // Pre-populate with some data so readers have something to find + for i in 0..10usize { + let rec = make_record(&format!("concurrent reader memory {i}")); + store.insert(&rec).await.expect("pre-insert"); + } + + let mut handles = Vec::new(); + + // 32 concurrent readers + for _ in 0..32usize { + let store = Arc::clone(&store); + let handle = tokio::spawn(async move { + let results = store.fts_search("concurrent reader", 5).await; + // Should not panic even if results vary due to concurrent writes + results.expect("fts_search in concurrent reader"); + }); + handles.push(handle); + } + + // 1 writer inserting more records + { + let store = Arc::clone(&store); + let writer_handle = tokio::spawn(async move { + for i in 0..20usize { + let rec = make_record(&format!("writer record {i}")); + store.insert(&rec).await.expect("concurrent insert"); + } + }); + handles.push(writer_handle); + } + + for h in handles { + h.await.expect("no panics"); + } + + // Eventual consistency check: total count should be at least 10 (initial) + let count = store.count().await.expect("final count"); + assert!( + count >= 10, + "at least the pre-populated records must persist" + ); +} diff --git a/tests/phase_1/trait_round_trip.rs b/tests/phase_1/trait_round_trip.rs new file mode 100644 index 0000000..ab3e0b2 --- /dev/null +++ b/tests/phase_1/trait_round_trip.rs @@ -0,0 +1,217 @@ +//! Phase 1 integration tests: round-trip of every trait method through SqliteMemoryStore. + +use chrono::Utc; +use std::sync::Arc; +use tempfile::tempdir; +use uuid::Uuid; +use vestige_core::storage::{ + MemoryEdge, MemoryRecord, MemoryStore, SearchQuery, SqliteMemoryStore, +}; + +fn make_store() -> Arc { + let dir = tempdir().unwrap(); + let db = dir.path().join("test.db"); + // keep the dir alive by leaking it -- this is fine for tests + std::mem::forget(dir); + let store = SqliteMemoryStore::new(Some(db)).expect("create store"); + Arc::new(store) +} + +fn make_record(content: &str) -> MemoryRecord { + MemoryRecord { + id: Uuid::new_v4(), + domains: vec![], + domain_scores: Default::default(), + content: content.to_string(), + node_type: "fact".to_string(), + tags: vec!["integration".to_string()], + embedding: None, + created_at: Utc::now(), + updated_at: Utc::now(), + metadata: serde_json::json!({}), + } +} + +#[tokio::test] +async fn insert_get_update_delete() { + let store = make_store(); + let rec = make_record("round-trip CRUD test"); + let id = rec.id; + + store.insert(&rec).await.expect("insert"); + let got = store.get(id).await.expect("get").expect("exists"); + assert_eq!(got.content, "round-trip CRUD test"); + assert_eq!(got.node_type, "fact"); + assert!(got.domains.is_empty()); + assert!(got.domain_scores.is_empty()); + + let mut updated = got; + updated.content = "updated content".to_string(); + store.update(&updated).await.expect("update"); + + let after_update = store + .get(id) + .await + .expect("get after update") + .expect("exists"); + assert_eq!(after_update.content, "updated content"); + + store.delete(id).await.expect("delete"); + let after_delete = store.get(id).await.expect("get after delete"); + assert!(after_delete.is_none()); +} + +#[tokio::test] +async fn scheduling_upsert_and_due_scan() { + use vestige_core::storage::SchedulingState; + let store = make_store(); + + for i in 0..3usize { + let rec = make_record(&format!("sched memory {i}")); + let id = rec.id; + store.insert(&rec).await.expect("insert"); + let next_review = Utc::now() - chrono::Duration::days((i as i64) + 1); + let state = SchedulingState { + memory_id: id, + stability: 1.0, + difficulty: 0.3, + retrievability: 0.7, + last_review: Some(Utc::now()), + next_review: Some(next_review), + reps: 1, + lapses: 0, + }; + store + .update_scheduling(&state) + .await + .expect("update scheduling"); + } + + let due = store + .get_due_memories(Utc::now(), 10) + .await + .expect("get_due_memories"); + assert_eq!(due.len(), 3, "all 3 should be due"); +} + +#[tokio::test] +async fn edge_crud() { + let store = make_store(); + let rec_a = make_record("edge node A"); + let rec_b = make_record("edge node B"); + let id_a = rec_a.id; + let id_b = rec_b.id; + store.insert(&rec_a).await.expect("insert a"); + store.insert(&rec_b).await.expect("insert b"); + + let edge = MemoryEdge { + source_id: id_a, + target_id: id_b, + edge_type: "semantic".to_string(), + weight: 0.85, + created_at: Utc::now(), + }; + store.add_edge(&edge).await.expect("add edge"); + + let edges = store.get_edges(id_a, None).await.expect("get edges"); + assert!(!edges.is_empty()); + + store.remove_edge(id_a, id_b).await.expect("remove edge"); + let after = store.get_edges(id_a, None).await.expect("get edges after"); + assert!(after.is_empty()); +} + +#[tokio::test] +async fn count_and_stats_track_inserts() { + let store = make_store(); + for i in 0..10usize { + let rec = make_record(&format!("stats memory {i}")); + store.insert(&rec).await.expect("insert"); + } + assert_eq!(store.count().await.expect("count"), 10); + let stats = store.get_stats().await.expect("stats"); + assert_eq!(stats.total_memories, 10); +} + +#[tokio::test] +async fn vacuum_after_deletes_reclaims() { + let dir = tempdir().unwrap(); + let db = dir.path().join("vacuum_test.db"); + let store = SqliteMemoryStore::new(Some(db)).expect("create store"); + let store: Arc = Arc::new(store); + + let mut ids = Vec::new(); + for i in 0..50usize { + let rec = make_record(&format!("vacuum memory {i}")); + let id = store.insert(&rec).await.expect("insert"); + ids.push(id); + } + for id in &ids[..40] { + store.delete(*id).await.expect("delete"); + } + // vacuum should not error + store.vacuum().await.expect("vacuum"); +} + +#[tokio::test] +async fn list_domains_empty_then_upsert_then_delete() { + use vestige_core::storage::Domain; + let store = make_store(); + + let domains = store.list_domains().await.expect("list empty"); + assert!(domains.is_empty()); + + let d = Domain { + id: "test-domain".to_string(), + label: "Test Domain".to_string(), + centroid: vec![0.1f32, 0.2, 0.3], + top_terms: vec!["term1".to_string()], + memory_count: 5, + created_at: Utc::now(), + }; + store.upsert_domain(&d).await.expect("upsert domain"); + let after = store.list_domains().await.expect("list after upsert"); + assert_eq!(after.len(), 1); + assert_eq!(after[0].id, "test-domain"); + + store + .delete_domain("test-domain") + .await + .expect("delete domain"); + let after_delete = store.list_domains().await.expect("list after delete"); + assert!(after_delete.is_empty()); +} + +#[tokio::test] +async fn classify_with_no_domains_returns_empty() { + let store = make_store(); + let result = store.classify(&[0.1f32, 0.2, 0.3]).await.expect("classify"); + assert!(result.is_empty()); +} + +#[tokio::test] +async fn search_hybrid_returns_results() { + let store = make_store(); + let rec = make_record("quantum entanglement superposition physics"); + store.insert(&rec).await.expect("insert"); + + // Verify fts_search works first (sanity check) + let fts_results = store.fts_search("quantum", 10).await.expect("fts_search"); + assert!( + !fts_results.is_empty(), + "fts_search must find 'quantum' after insert" + ); + + let query = SearchQuery { + text: Some("quantum physics".to_string()), + limit: 10, + ..Default::default() + }; + let results = store.search(&query).await.expect("search"); + // FTS results should include our inserted record + assert!( + !results.is_empty(), + "search must return results for 'quantum physics'" + ); + assert!(results[0].score >= 0.0); +} From a4a6e877c548cecca8064579fd62879a6025a4c3 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 15:40:04 +0200 Subject: [PATCH 26/38] feat(storage): swap async-trait for trait_variant + dyn adapter (0001a) Replaces #[async_trait::async_trait] on the storage trait with a trait_variant-driven layout plus a hand-written dyn-compatible adapter. - memory_store.rs: LocalMemoryStore is the source trait declared with native async-fn-in-trait. #[trait_variant::make(MemoryStoreSend: Send)] derives the Send-bounded variant that backends actually implement (the blanket impl in 0.1.x goes variant -> source). A hand-written MemoryStore trait wraps every method in Pin> + Send + 'a>> with a BoxedStoreFuture<'a, T> alias, and a blanket impl MemoryStore for T adapts every Send-variant implementation. This keeps Arc dyn-safe for Phase 1 cognitive-module tests -- trait_variant 0.1 alone does NOT produce a dyn-safe variant (RPITIT), so the hand-written adapter is required and supersedes the plan claim that trait_variant gives dyn-compat for free. - sqlite.rs: drop the #[async_trait::async_trait] attribute on the impl block and retarget it to MemoryStoreSend. Two pre-existing clippy issues that the macro had been masking are fixed in the same body (return Ok(out) tail expression in vector_search; DomainRow tuple alias in get_domain). - mod.rs: export MemoryStoreSend alongside the existing LocalMemoryStore and MemoryStore re-exports. Verification: cargo test -p vestige-core --features embeddings,vector-search passes (428 lib tests). All five Phase 1 integration test binaries pass (trait_round_trip, send_bound_variant including arc_dyn_memory_store_moves_across_tokio_tasks, cognitive_module_isolation, embedding_model_registry, domain_column_migration). cargo test --workspace green across every test binary. cargo build --workspace --release green. cargo clippy --workspace --features embeddings,vector-search -- -D warnings clean. grep -rn async_trait crates/vestige-core/src/storage/ returns zero hits. Supersedes plan claim in docs/plans/0001a-trait-rewrite.md about trait_variant emitting a dyn-compatible Send variant; option (c) from the design conversation (hand-written dyn adapter) was selected explicitly because trait_variant 0.1.2 does not. --- .../vestige-core/src/storage/memory_store.rs | 221 +++++++++++++++++- crates/vestige-core/src/storage/mod.rs | 4 +- crates/vestige-core/src/storage/sqlite.rs | 8 +- 3 files changed, 215 insertions(+), 18 deletions(-) diff --git a/crates/vestige-core/src/storage/memory_store.rs b/crates/vestige-core/src/storage/memory_store.rs index 2bc3137..2869a4e 100644 --- a/crates/vestige-core/src/storage/memory_store.rs +++ b/crates/vestige-core/src/storage/memory_store.rs @@ -4,6 +4,8 @@ //! intentionally flat: one trait, ~25 methods, no sub-traits. use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -182,17 +184,20 @@ pub struct ModelSignature { // TRAIT // ---------------------------------------------------------------------------- -/// The single storage abstraction. +/// Internal source trait declared with native async-fn-in-trait. /// -/// `#[async_trait::async_trait]` makes every `async fn` return a -/// `Pin>`, which is required for `Arc` -/// to be movable across `tokio::spawn` boundaries. +/// `#[trait_variant::make(MemoryStoreSend: Send)]` derives a Send-bounded +/// variant whose returned futures are `Send`. In trait_variant 0.1.x the +/// macro emits the blanket `impl LocalMemoryStore for T`, +/// so backends implement `MemoryStoreSend` (the Send variant) and get +/// `LocalMemoryStore` (the non-Send variant) for free. /// -/// `LocalMemoryStore` is a type alias kept for source compatibility with code -/// that refers to the non-send variant. In Phase 1 both names refer to the same -/// (dyn-compatible, Send-safe) trait. -#[async_trait::async_trait] -pub trait MemoryStore: Send + Sync + 'static { +/// Most callers should reach for the dyn-compatible `MemoryStore` trait +/// declared below, which adapts `MemoryStoreSend` into a boxed-future surface +/// and is the public storage abstraction for cognitive modules and tests +/// that want `Arc`. +#[trait_variant::make(MemoryStoreSend: Send)] +pub trait LocalMemoryStore: Sync + 'static { // --- Lifecycle --- async fn init(&self) -> MemoryStoreResult<()>; async fn health_check(&self) -> MemoryStoreResult; @@ -254,9 +259,201 @@ pub trait MemoryStore: Send + Sync + 'static { async fn vacuum(&self) -> MemoryStoreResult<()>; } -/// Type alias kept for source compatibility. Both names refer to the same -/// `async_trait`-annotated trait that is dyn-compatible and `Send + Sync`. -pub use MemoryStore as LocalMemoryStore; +// ---------------------------------------------------------------------------- +// DYN-COMPATIBLE STORAGE TRAIT +// ---------------------------------------------------------------------------- + +/// Boxed Send future returning a `MemoryStoreResult`, bound to the lifetime +/// of the borrows captured by the call (typically `&self` plus any reference +/// arguments). Used as the return type of every method on the dyn-compatible +/// `MemoryStore` trait below. +pub type BoxedStoreFuture<'a, T> = + Pin> + Send + 'a>>; + +/// Dyn-compatible storage trait. +/// +/// `MemoryStoreSend` above is the trait users implement; it uses native +/// async-fn-in-trait return types (RPITIT), which gives zero-allocation +/// static dispatch but is not dyn-safe. This trait wraps every method in +/// `Pin>` so `Arc` works for +/// the cognitive module surface and the Phase 1 integration tests. +/// +/// Implementations should not target this trait directly; the blanket +/// `impl MemoryStore for T` adapts every Send-variant +/// implementation automatically. Each call boxes the returned future +/// exactly once, identical to the cost of the previous design. +pub trait MemoryStore: Send + Sync + 'static { + fn init<'a>(&'a self) -> BoxedStoreFuture<'a, ()>; + fn health_check<'a>(&'a self) -> BoxedStoreFuture<'a, HealthStatus>; + + fn registered_model<'a>(&'a self) -> BoxedStoreFuture<'a, Option>; + fn register_model<'a>(&'a self, sig: &'a ModelSignature) -> BoxedStoreFuture<'a, ()>; + + fn insert<'a>(&'a self, record: &'a MemoryRecord) -> BoxedStoreFuture<'a, Uuid>; + fn get<'a>(&'a self, id: Uuid) -> BoxedStoreFuture<'a, Option>; + fn update<'a>(&'a self, record: &'a MemoryRecord) -> BoxedStoreFuture<'a, ()>; + fn delete<'a>(&'a self, id: Uuid) -> BoxedStoreFuture<'a, ()>; + + fn search<'a>(&'a self, query: &'a SearchQuery) -> BoxedStoreFuture<'a, Vec>; + fn fts_search<'a>( + &'a self, + text: &'a str, + limit: usize, + ) -> BoxedStoreFuture<'a, Vec>; + fn vector_search<'a>( + &'a self, + embedding: &'a [f32], + limit: usize, + ) -> BoxedStoreFuture<'a, Vec>; + + fn get_scheduling<'a>( + &'a self, + memory_id: Uuid, + ) -> BoxedStoreFuture<'a, Option>; + fn update_scheduling<'a>(&'a self, state: &'a SchedulingState) -> BoxedStoreFuture<'a, ()>; + fn get_due_memories<'a>( + &'a self, + before: DateTime, + limit: usize, + ) -> BoxedStoreFuture<'a, Vec<(MemoryRecord, SchedulingState)>>; + + fn add_edge<'a>(&'a self, edge: &'a MemoryEdge) -> BoxedStoreFuture<'a, ()>; + fn get_edges<'a>( + &'a self, + node_id: Uuid, + edge_type: Option<&'a str>, + ) -> BoxedStoreFuture<'a, Vec>; + fn remove_edge<'a>(&'a self, source: Uuid, target: Uuid) -> BoxedStoreFuture<'a, ()>; + fn get_neighbors<'a>( + &'a self, + node_id: Uuid, + depth: usize, + ) -> BoxedStoreFuture<'a, Vec<(MemoryRecord, f64)>>; + + fn list_domains<'a>(&'a self) -> BoxedStoreFuture<'a, Vec>; + fn get_domain<'a>(&'a self, id: &'a str) -> BoxedStoreFuture<'a, Option>; + fn upsert_domain<'a>(&'a self, domain: &'a Domain) -> BoxedStoreFuture<'a, ()>; + fn delete_domain<'a>(&'a self, id: &'a str) -> BoxedStoreFuture<'a, ()>; + fn classify<'a>(&'a self, embedding: &'a [f32]) -> BoxedStoreFuture<'a, Vec<(String, f64)>>; + + fn count<'a>(&'a self) -> BoxedStoreFuture<'a, usize>; + fn get_stats<'a>(&'a self) -> BoxedStoreFuture<'a, StoreStats>; + fn vacuum<'a>(&'a self) -> BoxedStoreFuture<'a, ()>; +} + +impl MemoryStore for T +where + T: MemoryStoreSend, +{ + fn init<'a>(&'a self) -> BoxedStoreFuture<'a, ()> { + Box::pin(::init(self)) + } + fn health_check<'a>(&'a self) -> BoxedStoreFuture<'a, HealthStatus> { + Box::pin(::health_check(self)) + } + + fn registered_model<'a>(&'a self) -> BoxedStoreFuture<'a, Option> { + Box::pin(::registered_model(self)) + } + fn register_model<'a>(&'a self, sig: &'a ModelSignature) -> BoxedStoreFuture<'a, ()> { + Box::pin(::register_model(self, sig)) + } + + fn insert<'a>(&'a self, record: &'a MemoryRecord) -> BoxedStoreFuture<'a, Uuid> { + Box::pin(::insert(self, record)) + } + fn get<'a>(&'a self, id: Uuid) -> BoxedStoreFuture<'a, Option> { + Box::pin(::get(self, id)) + } + fn update<'a>(&'a self, record: &'a MemoryRecord) -> BoxedStoreFuture<'a, ()> { + Box::pin(::update(self, record)) + } + fn delete<'a>(&'a self, id: Uuid) -> BoxedStoreFuture<'a, ()> { + Box::pin(::delete(self, id)) + } + + fn search<'a>(&'a self, query: &'a SearchQuery) -> BoxedStoreFuture<'a, Vec> { + Box::pin(::search(self, query)) + } + fn fts_search<'a>( + &'a self, + text: &'a str, + limit: usize, + ) -> BoxedStoreFuture<'a, Vec> { + Box::pin(::fts_search(self, text, limit)) + } + fn vector_search<'a>( + &'a self, + embedding: &'a [f32], + limit: usize, + ) -> BoxedStoreFuture<'a, Vec> { + Box::pin(::vector_search(self, embedding, limit)) + } + + fn get_scheduling<'a>( + &'a self, + memory_id: Uuid, + ) -> BoxedStoreFuture<'a, Option> { + Box::pin(::get_scheduling(self, memory_id)) + } + fn update_scheduling<'a>(&'a self, state: &'a SchedulingState) -> BoxedStoreFuture<'a, ()> { + Box::pin(::update_scheduling(self, state)) + } + fn get_due_memories<'a>( + &'a self, + before: DateTime, + limit: usize, + ) -> BoxedStoreFuture<'a, Vec<(MemoryRecord, SchedulingState)>> { + Box::pin(::get_due_memories(self, before, limit)) + } + + fn add_edge<'a>(&'a self, edge: &'a MemoryEdge) -> BoxedStoreFuture<'a, ()> { + Box::pin(::add_edge(self, edge)) + } + fn get_edges<'a>( + &'a self, + node_id: Uuid, + edge_type: Option<&'a str>, + ) -> BoxedStoreFuture<'a, Vec> { + Box::pin(::get_edges(self, node_id, edge_type)) + } + fn remove_edge<'a>(&'a self, source: Uuid, target: Uuid) -> BoxedStoreFuture<'a, ()> { + Box::pin(::remove_edge(self, source, target)) + } + fn get_neighbors<'a>( + &'a self, + node_id: Uuid, + depth: usize, + ) -> BoxedStoreFuture<'a, Vec<(MemoryRecord, f64)>> { + Box::pin(::get_neighbors(self, node_id, depth)) + } + + fn list_domains<'a>(&'a self) -> BoxedStoreFuture<'a, Vec> { + Box::pin(::list_domains(self)) + } + fn get_domain<'a>(&'a self, id: &'a str) -> BoxedStoreFuture<'a, Option> { + Box::pin(::get_domain(self, id)) + } + fn upsert_domain<'a>(&'a self, domain: &'a Domain) -> BoxedStoreFuture<'a, ()> { + Box::pin(::upsert_domain(self, domain)) + } + fn delete_domain<'a>(&'a self, id: &'a str) -> BoxedStoreFuture<'a, ()> { + Box::pin(::delete_domain(self, id)) + } + fn classify<'a>(&'a self, embedding: &'a [f32]) -> BoxedStoreFuture<'a, Vec<(String, f64)>> { + Box::pin(::classify(self, embedding)) + } + + fn count<'a>(&'a self) -> BoxedStoreFuture<'a, usize> { + Box::pin(::count(self)) + } + fn get_stats<'a>(&'a self) -> BoxedStoreFuture<'a, StoreStats> { + Box::pin(::get_stats(self)) + } + fn vacuum<'a>(&'a self) -> BoxedStoreFuture<'a, ()> { + Box::pin(::vacuum(self)) + } +} // ---------------------------------------------------------------------------- // UNIT TESTS diff --git a/crates/vestige-core/src/storage/mod.rs b/crates/vestige-core/src/storage/mod.rs index 6926385..5f0a54c 100644 --- a/crates/vestige-core/src/storage/mod.rs +++ b/crates/vestige-core/src/storage/mod.rs @@ -9,8 +9,8 @@ mod sqlite; pub use memory_store::{ ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, - MemoryStore, MemoryStoreError, MemoryStoreResult, ModelSignature, SchedulingState, SearchQuery, - SearchResult, StoreStats, + MemoryStore, MemoryStoreError, MemoryStoreResult, MemoryStoreSend, ModelSignature, + SchedulingState, SearchQuery, SearchResult, StoreStats, }; pub use migrations::MIGRATIONS; pub use portable::{ diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 57eaa86..abc17af 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -8441,8 +8441,7 @@ impl SqliteMemoryStore { } } -#[async_trait::async_trait] -impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { +impl crate::storage::memory_store::MemoryStoreSend for SqliteMemoryStore { async fn init(&self) -> crate::storage::memory_store::MemoryStoreResult<()> { // Migrations run in `new`; this is a no-op for the SQLite backend. Ok(()) @@ -8797,7 +8796,7 @@ impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { }) }) .collect(); - return Ok(out); + Ok(out) } #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] { @@ -9120,11 +9119,12 @@ impl crate::storage::memory_store::LocalMemoryStore for SqliteMemoryStore { ) -> crate::storage::memory_store::MemoryStoreResult> { use crate::storage::memory_store::{Domain, MemoryStoreError}; + type DomainRow = (String, String, Option>, String, i64, String); let reader = self .reader .lock() .map_err(|_| MemoryStoreError::Init("Reader lock poisoned".into()))?; - let result: Option<(String, String, Option>, String, i64, String)> = reader + let result: Option = reader .query_row( "SELECT id, label, centroid, top_terms, memory_count, created_at FROM domains WHERE id = ?1", rusqlite::params![id], From 194fc6e4c0c85d345ad32e268de88c930ebcec26 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 16:07:25 +0200 Subject: [PATCH 27/38] feat(embedder): swap async-trait for trait_variant + dyn adapter (0001c) Mirror of the 0001a pattern for the Embedder side. - embedder/mod.rs: LocalEmbedder is the source trait declared with native async-fn-in-trait. #[trait_variant::make(EmbedderSend: Send)] derives the Send-bounded variant that backends implement. A hand-written Embedder trait wraps each async method in BoxedEmbedderFuture<'a, T> and forwards sync methods through a blanket impl Embedder for T, so Box / Arc stay dyn-safe -- trait_variant 0.1 alone does NOT produce a dyn-safe variant (RPITIT), so the hand-written adapter is required. - embedder/fastembed.rs: drop the #[async_trait::async_trait] attribute and retarget the impl block to EmbedderSend. Adjust the top-level use to bring EmbedderSend into scope (also keeps fastembed::tests' use super::* trait lookups working). - lib.rs: export EmbedderSend alongside the existing Embedder / LocalEmbedder re-exports. The async-trait Cargo dependency is dropped in a follow-up commit so the manifest change stays visible on its own. Verification: cargo test -p vestige-core --features embeddings,vector-search (428) and --no-default-features (370) both green. cargo test --test embedder_trait green (2/2 including Box cast). cargo build --workspace --release green. cargo clippy --workspace --features embeddings,vector-search -- -D warnings clean. grep -rn async_trait crates/ returns zero. --- crates/vestige-core/src/embedder/fastembed.rs | 5 +- crates/vestige-core/src/embedder/mod.rs | 75 +++++++++++++++++-- crates/vestige-core/src/lib.rs | 4 +- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/crates/vestige-core/src/embedder/fastembed.rs b/crates/vestige-core/src/embedder/fastembed.rs index a4cd87b..a6ac120 100644 --- a/crates/vestige-core/src/embedder/fastembed.rs +++ b/crates/vestige-core/src/embedder/fastembed.rs @@ -4,7 +4,7 @@ #[cfg(feature = "embeddings")] use crate::embeddings::{EMBEDDING_DIMENSIONS, EmbeddingService}; -use super::{EmbedderError, EmbedderResult, LocalEmbedder}; +use super::{EmbedderError, EmbedderResult, EmbedderSend}; pub struct FastembedEmbedder { #[cfg(feature = "embeddings")] @@ -41,8 +41,7 @@ impl Default for FastembedEmbedder { } } -#[async_trait::async_trait] -impl LocalEmbedder for FastembedEmbedder { +impl EmbedderSend for FastembedEmbedder { async fn embed(&self, text: &str) -> EmbedderResult> { #[cfg(feature = "embeddings")] { diff --git a/crates/vestige-core/src/embedder/mod.rs b/crates/vestige-core/src/embedder/mod.rs index 9d43d0d..e8e654a 100644 --- a/crates/vestige-core/src/embedder/mod.rs +++ b/crates/vestige-core/src/embedder/mod.rs @@ -1,5 +1,8 @@ //! Text-to-vector encoding trait. Pluggable per-install. +use std::future::Future; +use std::pin::Pin; + mod fastembed; pub use fastembed::FastembedEmbedder; @@ -18,14 +21,23 @@ pub enum EmbedderError { pub type EmbedderResult = std::result::Result; +/// Boxed Send future returning an `EmbedderResult`, bound to the lifetime +/// of the borrows captured by the call. Used as the return type of every +/// async method on the dyn-compatible `Embedder` trait below. +pub type BoxedEmbedderFuture<'a, T> = + Pin> + Send + 'a>>; + /// Pluggable embedder. The storage layer NEVER calls fastembed directly; /// callers compute vectors via this trait and pass them into `MemoryStore`. /// -/// `#[async_trait::async_trait]` makes every `async fn` return a -/// `Pin>`, which is required for `Box` -/// and `Arc` to be dyn-compatible. -#[async_trait::async_trait] -pub trait LocalEmbedder: Send + Sync + 'static { +/// `LocalEmbedder` is the source-of-truth trait declared with native +/// async-fn-in-trait. `#[trait_variant::make(EmbedderSend: Send)]` derives +/// a Send-bounded variant that backends actually implement (the +/// trait_variant 0.1.x blanket goes variant -> source). The dyn-compatible +/// public surface is the `Embedder` trait declared below, which wraps every +/// async method in `Pin>`. +#[trait_variant::make(EmbedderSend: Send)] +pub trait LocalEmbedder: Sync + 'static { async fn embed(&self, text: &str) -> EmbedderResult>; fn model_name(&self) -> &str; @@ -52,6 +64,53 @@ pub trait LocalEmbedder: Send + Sync + 'static { } } -/// Type alias: `Embedder` is the dyn-compatible, Send+Sync variant. -/// Both names refer to the same `async_trait`-annotated trait. -pub use LocalEmbedder as Embedder; +/// Dyn-compatible embedder trait. +/// +/// `EmbedderSend` above is the trait users implement; it uses native +/// async-fn-in-trait return types (RPITIT), which gives zero-allocation +/// static dispatch but is not dyn-safe. This trait wraps every async +/// method in `Pin>` so `Box` +/// and `Arc` work for the cognitive module surface and +/// the Phase 1 integration tests. +/// +/// Implementations should not target this trait directly; the blanket +/// `impl Embedder for T` adapts every Send-variant +/// implementation automatically. +pub trait Embedder: Send + Sync + 'static { + fn embed<'a>(&'a self, text: &'a str) -> BoxedEmbedderFuture<'a, Vec>; + fn embed_batch<'a>( + &'a self, + texts: &'a [&'a str], + ) -> BoxedEmbedderFuture<'a, Vec>>; + fn model_name(&self) -> &str; + fn dimension(&self) -> usize; + fn model_hash(&self) -> String; + fn signature(&self) -> crate::storage::ModelSignature; +} + +impl Embedder for T +where + T: EmbedderSend, +{ + fn embed<'a>(&'a self, text: &'a str) -> BoxedEmbedderFuture<'a, Vec> { + Box::pin(::embed(self, text)) + } + fn embed_batch<'a>( + &'a self, + texts: &'a [&'a str], + ) -> BoxedEmbedderFuture<'a, Vec>> { + Box::pin(::embed_batch(self, texts)) + } + fn model_name(&self) -> &str { + ::model_name(self) + } + fn dimension(&self) -> usize { + ::dimension(self) + } + fn model_hash(&self) -> String { + ::model_hash(self) + } + fn signature(&self) -> crate::storage::ModelSignature { + ::signature(self) + } +} diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index f8a35d6..15dbdbf 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -198,7 +198,9 @@ pub use storage::{ }; // Embedder trait and implementations -pub use embedder::{Embedder, EmbedderError, EmbedderResult, FastembedEmbedder, LocalEmbedder}; +pub use embedder::{ + Embedder, EmbedderError, EmbedderResult, EmbedderSend, FastembedEmbedder, LocalEmbedder, +}; // Consolidation (sleep-inspired memory processing) pub use consolidation::SleepConsolidation; From 093bb2d4b530740b9c761e6fe3f9f6c1927f1144 Mon Sep 17 00:00:00 2001 From: Jan De Landtsheer Date: Wed, 27 May 2026 16:07:45 +0200 Subject: [PATCH 28/38] chore(vestige-core): drop async-trait dependency cargo rm async-trait. Last usage was the FastembedEmbedder impl attribute, removed in the preceding 0001c commit; the MemoryStore side stopped using async-trait at 0001a. Verification: grep -rn async_trait crates/ returns zero hits. grep -rn async-trait --include=Cargo.toml crates/ returns zero hits. Cargo.lock no longer references the async-trait package. --- Cargo.lock | 12 ------------ crates/vestige-core/Cargo.toml | 1 - 2 files changed, 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8be114c..20e3853 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,17 +164,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -4686,7 +4675,6 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "vestige-core" version = "2.1.26" dependencies = [ - "async-trait", "blake3", "candle-core", "chrono", diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index 25a0495..05c32f9 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -127,7 +127,6 @@ usearch = { version = "=2.23.0", optional = true } lru = "0.16" trait-variant = "0.1" blake3 = "1" -async-trait = "0.1" [dev-dependencies] tempfile = "3" From b34203bcc5a9f10412fed794d230ea2dc1a15c95 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 19:14:39 -0500 Subject: [PATCH 29/38] fix(storage): finish PR 61 rebase cleanup --- crates/vestige-core/src/embedder/mod.rs | 13 +++---------- crates/vestige-core/src/storage/memory_store.rs | 11 +++++++---- tests/phase_1/domain_column_migration.rs | 12 ++++++------ 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/crates/vestige-core/src/embedder/mod.rs b/crates/vestige-core/src/embedder/mod.rs index e8e654a..c13368b 100644 --- a/crates/vestige-core/src/embedder/mod.rs +++ b/crates/vestige-core/src/embedder/mod.rs @@ -24,8 +24,7 @@ pub type EmbedderResult = std::result::Result; /// Boxed Send future returning an `EmbedderResult`, bound to the lifetime /// of the borrows captured by the call. Used as the return type of every /// async method on the dyn-compatible `Embedder` trait below. -pub type BoxedEmbedderFuture<'a, T> = - Pin> + Send + 'a>>; +pub type BoxedEmbedderFuture<'a, T> = Pin> + Send + 'a>>; /// Pluggable embedder. The storage layer NEVER calls fastembed directly; /// callers compute vectors via this trait and pass them into `MemoryStore`. @@ -78,10 +77,7 @@ pub trait LocalEmbedder: Sync + 'static { /// implementation automatically. pub trait Embedder: Send + Sync + 'static { fn embed<'a>(&'a self, text: &'a str) -> BoxedEmbedderFuture<'a, Vec>; - fn embed_batch<'a>( - &'a self, - texts: &'a [&'a str], - ) -> BoxedEmbedderFuture<'a, Vec>>; + fn embed_batch<'a>(&'a self, texts: &'a [&'a str]) -> BoxedEmbedderFuture<'a, Vec>>; fn model_name(&self) -> &str; fn dimension(&self) -> usize; fn model_hash(&self) -> String; @@ -95,10 +91,7 @@ where fn embed<'a>(&'a self, text: &'a str) -> BoxedEmbedderFuture<'a, Vec> { Box::pin(::embed(self, text)) } - fn embed_batch<'a>( - &'a self, - texts: &'a [&'a str], - ) -> BoxedEmbedderFuture<'a, Vec>> { + fn embed_batch<'a>(&'a self, texts: &'a [&'a str]) -> BoxedEmbedderFuture<'a, Vec>> { Box::pin(::embed_batch(self, texts)) } fn model_name(&self) -> &str { diff --git a/crates/vestige-core/src/storage/memory_store.rs b/crates/vestige-core/src/storage/memory_store.rs index 2869a4e..010ee97 100644 --- a/crates/vestige-core/src/storage/memory_store.rs +++ b/crates/vestige-core/src/storage/memory_store.rs @@ -267,8 +267,7 @@ pub trait LocalMemoryStore: Sync + 'static { /// of the borrows captured by the call (typically `&self` plus any reference /// arguments). Used as the return type of every method on the dyn-compatible /// `MemoryStore` trait below. -pub type BoxedStoreFuture<'a, T> = - Pin> + Send + 'a>>; +pub type BoxedStoreFuture<'a, T> = Pin> + Send + 'a>>; /// Dyn-compatible storage trait. /// @@ -387,7 +386,9 @@ where embedding: &'a [f32], limit: usize, ) -> BoxedStoreFuture<'a, Vec> { - Box::pin(::vector_search(self, embedding, limit)) + Box::pin(::vector_search( + self, embedding, limit, + )) } fn get_scheduling<'a>( @@ -404,7 +405,9 @@ where before: DateTime, limit: usize, ) -> BoxedStoreFuture<'a, Vec<(MemoryRecord, SchedulingState)>> { - Box::pin(::get_due_memories(self, before, limit)) + Box::pin(::get_due_memories( + self, before, limit, + )) } fn add_edge<'a>(&'a self, edge: &'a MemoryEdge) -> BoxedStoreFuture<'a, ()> { diff --git a/tests/phase_1/domain_column_migration.rs b/tests/phase_1/domain_column_migration.rs index 67e318b..031ca65 100644 --- a/tests/phase_1/domain_column_migration.rs +++ b/tests/phase_1/domain_column_migration.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use vestige_core::storage::{MemoryRecord, MemoryStore, SqliteMemoryStore}; #[tokio::test] -async fn fresh_db_has_v12_schema() { +async fn fresh_db_has_v16_schema() { let dir = tempdir().unwrap(); let db = dir.path().join("fresh.db"); let _store = SqliteMemoryStore::new(Some(db.clone())).expect("create"); @@ -50,13 +50,13 @@ async fn v11_db_upgrades_cleanly() { next_review, scheduled_days, has_embedding) \ VALUES (?1, ?2, 'fact', datetime('now'), datetime('now'), datetime('now'), \ 1.0, 0.3, 0, 0, 'new', 1.0, 1.0, 1.0, datetime('now'), 1, 0)", - rusqlite::params![format!("pre-v12-{i}"), format!("content {i}"),], + rusqlite::params![format!("pre-v16-{i}"), format!("content {i}"),], ) - .expect("insert pre-v12 row"); + .expect("insert pre-v16 row"); } } // Upgrade by opening through SqliteMemoryStore (triggers full migration) - let _store = SqliteMemoryStore::new(Some(db.clone())).expect("open with v12"); + let _store = SqliteMemoryStore::new(Some(db.clone())).expect("open with v16"); // Check all 5 rows have empty domains/domain_scores let conn = rusqlite::Connection::open(&db).expect("open raw"); let count: i64 = conn @@ -68,7 +68,7 @@ async fn v11_db_upgrades_cleanly() { .expect("count"); assert_eq!( count, 5, - "all pre-v12 rows must have empty domains/domain_scores" + "all pre-v16 rows must have empty domains/domain_scores" ); } @@ -157,5 +157,5 @@ async fn domains_table_exists() { |row| row.get(0), ) .expect("query"); - assert_eq!(count, 1, "domains table must exist after V12 migration"); + assert_eq!(count, 1, "domains table must exist after V16 migration"); } From 5e183041846806d44482c135d2864bd8524df19f Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 20:10:28 -0500 Subject: [PATCH 30/38] Fix OpenCode init migration cleanup --- packages/vestige-init/bin/init.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/vestige-init/bin/init.js b/packages/vestige-init/bin/init.js index cb62f81..78ea98e 100755 --- a/packages/vestige-init/bin/init.js +++ b/packages/vestige-init/bin/init.js @@ -360,18 +360,24 @@ function injectConfig(ide, ideName, binaryPath) { // OpenCode uses top-level "mcp" entries with command arrays. if (!config.$schema) config.$schema = 'https://opencode.ai/config.json'; if (!config.mcp) config.mcp = {}; - if (config.mcp.vestige) { - console.log(` [skip] ${ideName} — already configured`); - return false; - } + let migratedOpenCodeConfig = false; if (config.mcpServers && config.mcpServers.vestige) { delete config.mcpServers.vestige; + migratedOpenCodeConfig = true; if (Object.keys(config.mcpServers).length === 0) { delete config.mcpServers; } console.log(` [migrate] ${ideName} — moved vestige from mcpServers to mcp`); } - config.mcp.vestige = buildOpenCodeConfig(binaryPath); + if (config.mcp.vestige) { + if (!migratedOpenCodeConfig) { + console.log(` [skip] ${ideName} — already configured`); + return false; + } + // Preserve the valid OpenCode entry while still writing the stale-key cleanup. + } else { + config.mcp.vestige = buildOpenCodeConfig(binaryPath); + } } else { // Standard mcpServers format (Cursor, Claude Desktop, JetBrains, Windsurf) const key = ide.key || 'mcpServers'; From 2757010d6df1b45f50bad4ccda9c9900a247b8e6 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 20:29:02 -0500 Subject: [PATCH 31/38] Make fastembed smoke tests tolerate unavailable model --- crates/vestige-core/src/embedder/fastembed.rs | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/crates/vestige-core/src/embedder/fastembed.rs b/crates/vestige-core/src/embedder/fastembed.rs index a6ac120..a4b079e 100644 --- a/crates/vestige-core/src/embedder/fastembed.rs +++ b/crates/vestige-core/src/embedder/fastembed.rs @@ -115,6 +115,14 @@ impl EmbedderSend for FastembedEmbedder { mod tests { use super::*; + #[cfg(feature = "embeddings")] + fn is_model_unavailable(err: &EmbedderError) -> bool { + let msg = err.to_string(); + msg.contains("Failed to retrieve") + || msg.contains("model files can be downloaded") + || msg.contains("Failed to initialize nomic-ai/nomic-embed-text-v1.5") + } + #[test] fn embedder_reports_correct_name() { let e = FastembedEmbedder::new(); @@ -162,7 +170,14 @@ mod tests { fn embedder_embed_smoke() { let e = FastembedEmbedder::new(); let rt = tokio::runtime::Runtime::new().unwrap(); - let vec = rt.block_on(e.embed("hello world")).expect("embed"); + let vec = match rt.block_on(e.embed("hello world")) { + Ok(vec) => vec, + Err(err) if is_model_unavailable(&err) => { + eprintln!("skipping fastembed smoke; model unavailable: {err}"); + return; + } + Err(err) => panic!("embed: {err}"), + }; assert_eq!(vec.len(), 256); } @@ -172,9 +187,30 @@ mod tests { let e = FastembedEmbedder::new(); let rt = tokio::runtime::Runtime::new().unwrap(); let texts = ["alpha beta", "gamma delta"]; - let batch = rt.block_on(e.embed_batch(texts.as_ref())).expect("batch"); - let seq_a = rt.block_on(e.embed(texts[0])).expect("seq a"); - let seq_b = rt.block_on(e.embed(texts[1])).expect("seq b"); + let batch = match rt.block_on(e.embed_batch(texts.as_ref())) { + Ok(batch) => batch, + Err(err) if is_model_unavailable(&err) => { + eprintln!("skipping fastembed batch smoke; model unavailable: {err}"); + return; + } + Err(err) => panic!("batch: {err}"), + }; + let seq_a = match rt.block_on(e.embed(texts[0])) { + Ok(vec) => vec, + Err(err) if is_model_unavailable(&err) => { + eprintln!("skipping fastembed sequential smoke; model unavailable: {err}"); + return; + } + Err(err) => panic!("seq a: {err}"), + }; + let seq_b = match rt.block_on(e.embed(texts[1])) { + Ok(vec) => vec, + Err(err) if is_model_unavailable(&err) => { + eprintln!("skipping fastembed sequential smoke; model unavailable: {err}"); + return; + } + Err(err) => panic!("seq b: {err}"), + }; assert_eq!(batch[0], seq_a); assert_eq!(batch[1], seq_b); } From 536776c9d691384eda41eef165aadb071fc7e31c Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 21:36:53 -0500 Subject: [PATCH 32/38] Guard vector index init/search on unsupported CPU (#71) --- crates/vestige-core/src/storage/sqlite.rs | 224 +++++++++++++++------- 1 file changed, 155 insertions(+), 69 deletions(-) diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index abc17af..2fb5255 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -293,6 +293,7 @@ struct PortableMergeState { const DATA_DIR_ENV: &str = "VESTIGE_DATA_DIR"; const DATABASE_FILE: &str = "vestige.db"; +const VESTIGE_DISABLE_VECTOR_SEARCH: &str = "VESTIGE_DISABLE_VECTOR_SEARCH"; /// Main storage struct with integrated embedding and vector search /// @@ -307,15 +308,63 @@ pub struct SqliteMemoryStore { #[cfg(feature = "embeddings")] embedding_service: EmbeddingService, #[cfg(feature = "vector-search")] - vector_index: Mutex, + vector_index: Option>, /// LRU cache for query embeddings to avoid re-embedding repeated queries #[cfg(all(feature = "embeddings", feature = "vector-search"))] - query_cache: Mutex>>, + query_cache: Option>>>, /// Cached model signature. `None` until the first embedding is written. registered_model: std::sync::RwLock>, } impl SqliteMemoryStore { + #[cfg(feature = "vector-search")] + fn vector_search_enabled_by_cpu() -> bool { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + let has_required_features = std::arch::is_x86_feature_detected!("avx2") + && std::arch::is_x86_feature_detected!("fma"); + + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + let has_required_features = true; + + let disabled_by_env = std::env::var_os(VESTIGE_DISABLE_VECTOR_SEARCH) + .and_then(|v| { + let value = v.to_ascii_lowercase(); + if value == "1" + || value == "true" + || value == "yes" + || value == "on" + || value == "enable" + || value == "enabled" + { + Some(()) + } else { + None + } + }) + .is_some(); + + has_required_features && !disabled_by_env + } + + #[cfg(feature = "vector-search")] + fn vector_search_unavailable_reason() -> Option<&'static str> { + if std::env::var_os(VESTIGE_DISABLE_VECTOR_SEARCH).is_some() { + return Some("disabled by VESTIGE_DISABLE_VECTOR_SEARCH"); + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + if !std::arch::is_x86_feature_detected!("avx2") { + return Some("unsupported CPU: AVX2 required"); + } + if !std::arch::is_x86_feature_detected!("fma") { + return Some("unsupported CPU: FMA required"); + } + } + + None + } + fn data_dir_from_env() -> Option { std::env::var_os(DATA_DIR_ENV).and_then(|value| { if value.is_empty() { @@ -439,15 +488,26 @@ impl SqliteMemoryStore { let embedding_service = EmbeddingService::new(); #[cfg(feature = "vector-search")] - let vector_index = VectorIndex::new() - .map_err(|e| StorageError::Init(format!("Failed to create vector index: {}", e)))?; + let vector_index = if Self::vector_search_enabled_by_cpu() { + let vector_index = VectorIndex::new() + .map_err(|e| StorageError::Init(format!("Failed to create vector index: {}", e)))?; + Some(Mutex::new(vector_index)) + } else { + tracing::warn!( + "Vector search disabled: {}", + Self::vector_search_unavailable_reason().unwrap_or("manual override"), + ); + None + }; - // Initialize LRU cache for query embeddings (capacity: 100 queries) - // SAFETY: 100 is always non-zero, this cannot fail #[cfg(all(feature = "embeddings", feature = "vector-search"))] - let query_cache = Mutex::new(LruCache::new( - NonZeroUsize::new(100).expect("100 is non-zero"), - )); + let query_cache = if vector_index.is_some() { + Some(Mutex::new(LruCache::new( + NonZeroUsize::new(100).expect("100 is non-zero"), + ))) + } else { + None + }; let storage = Self { db_path: path, @@ -457,14 +517,16 @@ impl SqliteMemoryStore { #[cfg(feature = "embeddings")] embedding_service, #[cfg(feature = "vector-search")] - vector_index: Mutex::new(vector_index), + vector_index, #[cfg(all(feature = "embeddings", feature = "vector-search"))] query_cache, registered_model: std::sync::RwLock::new(None), }; #[cfg(all(feature = "embeddings", feature = "vector-search"))] - storage.load_embeddings_into_index()?; + if storage.vector_index.is_some() { + storage.load_embeddings_into_index()?; + } Ok(storage) } @@ -487,8 +549,11 @@ impl SqliteMemoryStore { /// Load existing embeddings into vector index #[cfg(all(feature = "embeddings", feature = "vector-search"))] fn load_embeddings_into_index(&self) -> Result<()> { - let mut index = self - .vector_index + let Some(index) = self.vector_index.as_ref() else { + return Ok(()); + }; + + let mut index = index .lock() .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; let reader = self @@ -998,8 +1063,10 @@ impl SqliteMemoryStore { #[cfg(all(feature = "embeddings", feature = "vector-search"))] { // Remove old embedding from index - if let Ok(mut index) = self.vector_index.lock() { - let _ = index.remove(id); + if let Some(index) = self.vector_index.as_ref() { + if let Ok(mut index) = index.lock() { + let _ = index.remove(id); + } } // Generate new embedding if let Err(e) = self.generate_embedding_for_node(id, new_content) { @@ -1048,13 +1115,14 @@ impl SqliteMemoryStore { )?; } - let mut index = self - .vector_index - .lock() - .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; - index - .add(node_id, &embedding.vector) - .map_err(|e| StorageError::Init(format!("Vector index add failed: {}", e)))?; + if let Some(index) = self.vector_index.as_ref() { + let mut index = index + .lock() + .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; + index + .add(node_id, &embedding.vector) + .map_err(|e| StorageError::Init(format!("Vector index add failed: {}", e)))?; + } Ok(()) } @@ -1388,35 +1456,36 @@ impl SqliteMemoryStore { // Content-aware cross-memory reinforcement: boost semantically similar neighbors #[cfg(all(feature = "embeddings", feature = "vector-search"))] { - if let Ok(Some(embedding)) = self.get_node_embedding(id) { - let index = self - .vector_index - .lock() - .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; + if let Some(index) = self.vector_index.as_ref() { + if let Ok(Some(embedding)) = self.get_node_embedding(id) { + let index = index.lock().map_err(|_| { + StorageError::Init("Vector index lock poisoned".to_string()) + })?; - // Query top-6 similar (one will be self, so we get ~5 neighbors) - let neighbors_result = index.search(&embedding, 6); - drop(index); + // Query top-6 similar (one will be self, so we get ~5 neighbors) + let neighbors_result = index.search(&embedding, 6); + drop(index); - if let Ok(neighbors) = neighbors_result { - let writer = self - .writer - .lock() - .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; - for (neighbor_id, similarity) in neighbors { - if neighbor_id == id || similarity < 0.7 { - continue; + if let Ok(neighbors) = neighbors_result { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + for (neighbor_id, similarity) in neighbors { + if neighbor_id == id || similarity < 0.7 { + continue; + } + // Diminished boost: 0.02 * similarity (max ~0.02) + let boost = 0.02 * similarity as f64; + let retention_boost = 0.008 * similarity as f64; + let _ = writer.execute( + "UPDATE knowledge_nodes SET + retrieval_strength = MIN(1.0, retrieval_strength + ?1), + retention_strength = MIN(1.0, retention_strength + ?2) + WHERE id = ?3", + params![boost, retention_boost, neighbor_id], + ); } - // Diminished boost: 0.02 * similarity (max ~0.02) - let boost = 0.02 * similarity as f64; - let retention_boost = 0.008 * similarity as f64; - let _ = writer.execute( - "UPDATE knowledge_nodes SET - retrieval_strength = MIN(1.0, retrieval_strength + ?1), - retention_strength = MIN(1.0, retention_strength + ?2) - WHERE id = ?3", - params![boost, retention_boost, neighbor_id], - ); } } } @@ -2031,10 +2100,12 @@ impl SqliteMemoryStore { // Clean up vector index to prevent stale search results #[cfg(all(feature = "embeddings", feature = "vector-search"))] - if rows > 0 - && let Ok(mut index) = self.vector_index.lock() - { - let _ = index.remove(id); + if rows > 0 { + if let Some(index) = self.vector_index.as_ref() { + if let Ok(mut index) = index.lock() { + let _ = index.remove(id); + } + } } Ok(rows > 0) @@ -2153,8 +2224,10 @@ impl SqliteMemoryStore { tx.commit()?; #[cfg(all(feature = "embeddings", feature = "vector-search"))] - if let Ok(mut index) = self.vector_index.lock() { - let _ = index.remove(id); + if let Some(index) = self.vector_index.as_ref() { + if let Ok(mut index) = index.lock() { + let _ = index.remove(id); + } } Ok(PurgeReport { @@ -2551,9 +2624,11 @@ impl SqliteMemoryStore { fn get_query_embedding(&self, query: &str) -> Result> { let cache_key = format!("{}\0{}", self.embedding_service.model_name(), query); // Check cache first + let Some(index_cache) = self.query_cache.as_ref() else { + return Err(StorageError::Init("Query cache unavailable".to_string())); + }; { - let mut cache = self - .query_cache + let mut cache = index_cache .lock() .map_err(|_| StorageError::Init("Query cache lock poisoned".to_string()))?; if let Some(cached) = cache.get(&cache_key) { @@ -2569,8 +2644,7 @@ impl SqliteMemoryStore { // Store in cache { - let mut cache = self - .query_cache + let mut cache = index_cache .lock() .map_err(|_| StorageError::Init("Query cache lock poisoned".to_string()))?; cache.put(cache_key, embedding.vector.clone()); @@ -2591,10 +2665,15 @@ impl SqliteMemoryStore { return Err(StorageError::Init("Embedding model not ready".to_string())); } + let Some(index_lock) = self.vector_index.as_ref() else { + return Err(StorageError::Init( + "Vector search unavailable: disabled for this machine".to_string(), + )); + }; + let query_embedding = self.get_query_embedding(query)?; - let index = self - .vector_index + let index = index_lock .lock() .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; @@ -2911,6 +2990,9 @@ impl SqliteMemoryStore { if !self.embedding_service.is_ready() { return Ok(vec![]); } + self.vector_index.as_ref().ok_or_else(|| { + StorageError::Init("Vector search unavailable: disabled for this machine".to_string()) + })?; // HyDE query expansion: for conceptual queries, embed expanded variants // and use the centroid for broader semantic coverage @@ -2934,8 +3016,8 @@ impl SqliteMemoryStore { _ => self.get_query_embedding(query)?, }; - let index = self - .vector_index + let index = self.vector_index.as_ref().unwrap(); + let index = index .lock() .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; @@ -7095,11 +7177,13 @@ impl SqliteMemoryStore { // Clean up vector index #[cfg(all(feature = "embeddings", feature = "vector-search"))] - if deleted > 0 - && let Ok(mut index) = self.vector_index.lock() - { - for id in &doomed_ids { - let _ = index.remove(id); + if deleted > 0 { + if let Some(index) = self.vector_index.as_ref() { + if let Ok(mut index) = index.lock() { + for id in &doomed_ids { + let _ = index.remove(id); + } + } } } @@ -8772,8 +8856,10 @@ impl crate::storage::memory_store::MemoryStoreSend for SqliteMemoryStore { use crate::storage::memory_store::{MemoryStoreError, SearchResult}; #[cfg(all(feature = "embeddings", feature = "vector-search"))] { - let index = self - .vector_index + let Some(index) = self.vector_index.as_ref() else { + return Ok(vec![]); + }; + let index = index .lock() .map_err(|_| MemoryStoreError::Init("Vector index lock poisoned".into()))?; let raw_results = index From ef2073d4a481c38f2a89685fcca40a7ff3ff011e Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 21:54:04 -0500 Subject: [PATCH 33/38] Harden old CPU fallback paths (#71) --- crates/vestige-core/src/embeddings/local.rs | 12 +- crates/vestige-core/src/storage/sqlite.rs | 268 ++++++++++++++------ 2 files changed, 201 insertions(+), 79 deletions(-) diff --git a/crates/vestige-core/src/embeddings/local.rs b/crates/vestige-core/src/embeddings/local.rs index d1ce798..03757a6 100644 --- a/crates/vestige-core/src/embeddings/local.rs +++ b/crates/vestige-core/src/embeddings/local.rs @@ -343,14 +343,18 @@ impl EmbeddingService { Self { _unused: () } } - /// Check if the model is ready + /// Check if the model has already been initialized. + /// + /// This must stay side-effect free: health/status paths call it during + /// startup and must not download or load ONNX models just to report state. pub fn is_ready(&self) -> bool { - match get_backend() { - Ok(_) => true, - Err(e) => { + match EMBEDDING_BACKEND_RESULT.get() { + Some(Ok(backend)) => backend.lock().is_ok(), + Some(Err(e)) => { tracing::warn!("Embedding model not ready: {}", e); false } + None => false, } } diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 2fb5255..9685f61 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -365,6 +365,36 @@ impl SqliteMemoryStore { None } + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + fn vector_search_available(&self) -> bool { + self.vector_index.is_some() + } + + #[cfg(not(all(feature = "embeddings", feature = "vector-search")))] + fn vector_search_available(&self) -> bool { + false + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + fn regular_ingest_result( + &self, + input: IngestInput, + reason: impl Into, + ) -> Result { + let node = self.ingest(input)?; + Ok(SmartIngestResult { + decision: "create".to_string(), + node, + superseded_id: None, + similarity: None, + prediction_error: Some(1.0), + reason: reason.into(), + previous_content: None, + merged_from: None, + merge_preview: None, + }) + } + fn data_dir_from_env() -> Option { std::env::var_os(DATA_DIR_ENV).and_then(|value| { if value.is_empty() { @@ -742,19 +772,17 @@ impl SqliteMemoryStore { // Generate embedding for new content if !self.embedding_service.is_ready() { - // Fall back to regular ingest if embeddings not available - let node = self.ingest(input)?; - return Ok(SmartIngestResult { - decision: "create".to_string(), - node, - superseded_id: None, - similarity: None, - prediction_error: Some(1.0), - reason: "Embeddings not available, falling back to regular ingest".to_string(), - previous_content: None, - merged_from: None, - merge_preview: None, - }); + return self.regular_ingest_result( + input, + "Embeddings not available, falling back to regular ingest", + ); + } + + if !self.vector_search_available() { + return self.regular_ingest_result( + input, + "Vector search unavailable, falling back to regular ingest", + ); } let new_embedding = self @@ -1063,10 +1091,10 @@ impl SqliteMemoryStore { #[cfg(all(feature = "embeddings", feature = "vector-search"))] { // Remove old embedding from index - if let Some(index) = self.vector_index.as_ref() { - if let Ok(mut index) = index.lock() { - let _ = index.remove(id); - } + if let Some(index) = self.vector_index.as_ref() + && let Ok(mut index) = index.lock() + { + let _ = index.remove(id); } // Generate new embedding if let Err(e) = self.generate_embedding_for_node(id, new_content) { @@ -1276,8 +1304,12 @@ impl SqliteMemoryStore { } #[cfg(all(feature = "embeddings", feature = "vector-search"))] SearchMode::Semantic => { - let results = self.semantic_search(&input.query, input.limit, 0.3)?; - results.into_iter().map(|r| r.node).collect() + if !self.vector_search_available() { + self.keyword_search(&input.query, input.limit, input.min_retention)? + } else { + let results = self.semantic_search(&input.query, input.limit, 0.3)?; + results.into_iter().map(|r| r.node).collect() + } } #[cfg(all(feature = "embeddings", feature = "vector-search"))] SearchMode::Hybrid => { @@ -1456,36 +1488,36 @@ impl SqliteMemoryStore { // Content-aware cross-memory reinforcement: boost semantically similar neighbors #[cfg(all(feature = "embeddings", feature = "vector-search"))] { - if let Some(index) = self.vector_index.as_ref() { - if let Ok(Some(embedding)) = self.get_node_embedding(id) { - let index = index.lock().map_err(|_| { - StorageError::Init("Vector index lock poisoned".to_string()) - })?; + if let Some(index) = self.vector_index.as_ref() + && let Ok(Some(embedding)) = self.get_node_embedding(id) + { + let index = index + .lock() + .map_err(|_| StorageError::Init("Vector index lock poisoned".to_string()))?; - // Query top-6 similar (one will be self, so we get ~5 neighbors) - let neighbors_result = index.search(&embedding, 6); - drop(index); + // Query top-6 similar (one will be self, so we get ~5 neighbors) + let neighbors_result = index.search(&embedding, 6); + drop(index); - if let Ok(neighbors) = neighbors_result { - let writer = self - .writer - .lock() - .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; - for (neighbor_id, similarity) in neighbors { - if neighbor_id == id || similarity < 0.7 { - continue; - } - // Diminished boost: 0.02 * similarity (max ~0.02) - let boost = 0.02 * similarity as f64; - let retention_boost = 0.008 * similarity as f64; - let _ = writer.execute( - "UPDATE knowledge_nodes SET - retrieval_strength = MIN(1.0, retrieval_strength + ?1), - retention_strength = MIN(1.0, retention_strength + ?2) - WHERE id = ?3", - params![boost, retention_boost, neighbor_id], - ); + if let Ok(neighbors) = neighbors_result { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + for (neighbor_id, similarity) in neighbors { + if neighbor_id == id || similarity < 0.7 { + continue; } + // Diminished boost: 0.02 * similarity (max ~0.02) + let boost = 0.02 * similarity as f64; + let retention_boost = 0.008 * similarity as f64; + let _ = writer.execute( + "UPDATE knowledge_nodes SET + retrieval_strength = MIN(1.0, retrieval_strength + ?1), + retention_strength = MIN(1.0, retention_strength + ?2) + WHERE id = ?3", + params![boost, retention_boost, neighbor_id], + ); } } } @@ -2100,12 +2132,11 @@ impl SqliteMemoryStore { // Clean up vector index to prevent stale search results #[cfg(all(feature = "embeddings", feature = "vector-search"))] - if rows > 0 { - if let Some(index) = self.vector_index.as_ref() { - if let Ok(mut index) = index.lock() { - let _ = index.remove(id); - } - } + if rows > 0 + && let Some(index) = self.vector_index.as_ref() + && let Ok(mut index) = index.lock() + { + let _ = index.remove(id); } Ok(rows > 0) @@ -2224,10 +2255,10 @@ impl SqliteMemoryStore { tx.commit()?; #[cfg(all(feature = "embeddings", feature = "vector-search"))] - if let Some(index) = self.vector_index.as_ref() { - if let Ok(mut index) = index.lock() { - let _ = index.remove(id); - } + if let Some(index) = self.vector_index.as_ref() + && let Ok(mut index) = index.lock() + { + let _ = index.remove(id); } Ok(PurgeReport { @@ -2661,16 +2692,16 @@ impl SqliteMemoryStore { limit: i32, min_similarity: f32, ) -> Result> { - if !self.embedding_service.is_ready() { - return Err(StorageError::Init("Embedding model not ready".to_string())); - } - let Some(index_lock) = self.vector_index.as_ref() else { return Err(StorageError::Init( "Vector search unavailable: disabled for this machine".to_string(), )); }; + if !self.embedding_service.is_ready() { + return Err(StorageError::Init("Embedding model not ready".to_string())); + } + let query_embedding = self.get_query_embedding(query)?; let index = index_lock @@ -2733,11 +2764,12 @@ impl SqliteMemoryStore { exclude_types, )?; - let semantic_results = if self.embedding_service.is_ready() { - self.semantic_search_raw(query, limit * overfetch_factor)? - } else { - vec![] - }; + let semantic_results = + if self.vector_search_available() && self.embedding_service.is_ready() { + self.semantic_search_raw(query, limit * overfetch_factor)? + } else { + vec![] + }; let combined = if !semantic_results.is_empty() { linear_combination( @@ -2987,12 +3019,12 @@ impl SqliteMemoryStore { /// Semantic search returning scores #[cfg(all(feature = "embeddings", feature = "vector-search"))] fn semantic_search_raw(&self, query: &str, limit: i32) -> Result> { + if !self.vector_search_available() { + return Ok(vec![]); + } if !self.embedding_service.is_ready() { return Ok(vec![]); } - self.vector_index.as_ref().ok_or_else(|| { - StorageError::Init("Vector search unavailable: disabled for this machine".to_string()) - })?; // HyDE query expansion: for conceptual queries, embed expanded variants // and use the centroid for broader semantic coverage @@ -7177,13 +7209,12 @@ impl SqliteMemoryStore { // Clean up vector index #[cfg(all(feature = "embeddings", feature = "vector-search"))] - if deleted > 0 { - if let Some(index) = self.vector_index.as_ref() { - if let Ok(mut index) = index.lock() { - for id in &doomed_ids { - let _ = index.remove(id); - } - } + if deleted > 0 + && let Some(index) = self.vector_index.as_ref() + && let Ok(mut index) = index.lock() + { + for id in &doomed_ids { + let _ = index.remove(id); } } @@ -9392,11 +9423,16 @@ impl crate::storage::memory_store::MemoryStoreSend for SqliteMemoryStore { mod tests { use super::*; use crate::advanced::{MatchClass, MergePolicy}; + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind}; use tempfile::tempdir; // The public struct was renamed from Storage to SqliteMemoryStore; this // alias keeps all existing tests compiling without modification. use SqliteMemoryStore as Storage; + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + static ENV_LOCK: Mutex<()> = Mutex::new(()); + fn create_test_storage() -> Storage { let dir = tempdir().unwrap(); let db_path = dir.path().join("test.db"); @@ -9407,6 +9443,88 @@ mod tests { Storage::new(Some(dir.path().join(name))).unwrap() } + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + fn with_vector_search_disabled(f: impl FnOnce() -> T) -> T { + let _guard = ENV_LOCK.lock().unwrap(); + let previous = std::env::var_os(VESTIGE_DISABLE_VECTOR_SEARCH); + + // Tests serialize access with ENV_LOCK because process environment + // mutation is global and unsafe under Rust 2024. + unsafe { + std::env::set_var(VESTIGE_DISABLE_VECTOR_SEARCH, "1"); + } + + let result = catch_unwind(AssertUnwindSafe(f)); + + unsafe { + if let Some(value) = previous { + std::env::set_var(VESTIGE_DISABLE_VECTOR_SEARCH, value); + } else { + std::env::remove_var(VESTIGE_DISABLE_VECTOR_SEARCH); + } + } + + match result { + Ok(value) => value, + Err(payload) => resume_unwind(payload), + } + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_runtime_vector_gate_env_disables_index_creation() { + with_vector_search_disabled(|| { + assert!(!Storage::vector_search_enabled_by_cpu()); + assert_eq!( + Storage::vector_search_unavailable_reason(), + Some("disabled by VESTIGE_DISABLE_VECTOR_SEARCH") + ); + + let dir = tempdir().unwrap(); + let storage = create_test_storage_at(&dir, "vector-disabled.db"); + + assert!(storage.vector_index.is_none()); + assert!(storage.query_cache.is_none()); + + let stats = storage.get_stats().unwrap(); + assert_eq!(stats.total_nodes, 0); + + let schema = storage.schema_introspection().unwrap(); + assert!(schema.schema_version >= 1); + }); + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + #[test] + fn test_runtime_vector_gate_disabled_hybrid_search_uses_keyword_fallback() { + with_vector_search_disabled(|| { + let dir = tempdir().unwrap(); + let storage = create_test_storage_at(&dir, "vector-disabled-search.db"); + + storage + .ingest(IngestInput { + content: "runtime gate fallback keyword anchor".to_string(), + node_type: "fact".to_string(), + ..Default::default() + }) + .unwrap(); + + let results = storage + .hybrid_search("runtime gate fallback keyword", 10, 0.3, 0.7) + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].match_type, MatchType::Keyword); + assert!(results[0].semantic_score.is_none()); + assert!( + results[0] + .node + .content + .contains("runtime gate fallback keyword anchor") + ); + }); + } + #[cfg(all(feature = "embeddings", feature = "vector-search"))] #[test] fn test_embedding_model_family_matching() { From 22d0d192eb5bbcb4932866691885c9b93de32887 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Thu, 18 Jun 2026 23:39:38 -0500 Subject: [PATCH 34/38] fix: make windows release build and add manual rerun path --- .github/workflows/release.yml | 4 +- .../src/advanced/merge_supersede.rs | 5 +- crates/vestige-core/src/config.rs | 20 +- .../vestige-mcp/src/tools/codebase_unified.rs | 92 +++++- crates/vestige-mcp/src/tools/maintenance.rs | 3 +- crates/vestige-mcp/src/tools/merge.rs | 29 +- .../vestige-mcp/src/tools/search_unified.rs | 279 +++++++++++++++--- .../vestige-mcp/src/tools/session_context.rs | 51 +++- crates/vestige-mcp/src/tools/timeline.rs | 8 +- 9 files changed, 396 insertions(+), 95 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 793ce48..d789b12 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: - target: x86_64-pc-windows-msvc os: windows-latest archive: zip - cargo_flags: "" + cargo_flags: "--no-default-features --features embeddings,ort-download" needs_onnxruntime: false # Intel Mac uses the ort-dynamic feature to runtime-link against a # system libonnxruntime (Homebrew), sidestepping the missing @@ -51,7 +51,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: ${{ github.event.inputs.tag || github.ref }} + ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.inputs.tag || github.ref }} - name: Setup pnpm uses: pnpm/action-setup@v4 diff --git a/crates/vestige-core/src/advanced/merge_supersede.rs b/crates/vestige-core/src/advanced/merge_supersede.rs index c10485a..cf206b4 100644 --- a/crates/vestige-core/src/advanced/merge_supersede.rs +++ b/crates/vestige-core/src/advanced/merge_supersede.rs @@ -401,7 +401,10 @@ mod tests { "totally unrelated subject beta", ); assert!(s.combined_score < 0.3); - assert_eq!(MergePolicy::default().classify(s.combined_score), MatchClass::NonMatch); + assert_eq!( + MergePolicy::default().classify(s.combined_score), + MatchClass::NonMatch + ); } #[test] diff --git a/crates/vestige-core/src/config.rs b/crates/vestige-core/src/config.rs index f1bb8d5..c55e19b 100644 --- a/crates/vestige-core/src/config.rs +++ b/crates/vestige-core/src/config.rs @@ -242,9 +242,7 @@ impl OutputConfig { /// tool's own built-in fallback (used only when neither param nor config /// supplies one). pub fn resolve_limit(&self, explicit: Option, builtin_default: i32) -> i32 { - explicit - .or(self.limit) - .unwrap_or(builtin_default) + explicit.or(self.limit).unwrap_or(builtin_default) } } @@ -288,7 +286,10 @@ mod tests { #[test] fn empty_or_missing_file_is_default() { assert_eq!(VestigeConfig::parse(""), VestigeConfig::default()); - assert_eq!(VestigeConfig::parse("\n\n# just a comment\n"), VestigeConfig::default()); + assert_eq!( + VestigeConfig::parse("\n\n# just a comment\n"), + VestigeConfig::default() + ); } #[test] @@ -308,9 +309,7 @@ mod tests { #[test] fn unquoted_and_commented_values() { - let cfg = VestigeConfig::parse( - "[defaults]\nprofile = lean # inline comment\nlimit = 7\n", - ); + let cfg = VestigeConfig::parse("[defaults]\nprofile = lean # inline comment\nlimit = 7\n"); assert_eq!(cfg.defaults.profile, OutputProfile::Lean); assert_eq!(cfg.defaults.limit, Some(7)); } @@ -357,10 +356,9 @@ mod tests { #[test] fn explicit_defaults_override_profile_presets() { // profile=lean would give brief/limit 5, but explicit keys win. - let out = VestigeConfig::parse( - "[defaults]\nprofile=lean\ndetail_level=\"full\"\nlimit=42\n", - ) - .output(); + let out = + VestigeConfig::parse("[defaults]\nprofile=lean\ndetail_level=\"full\"\nlimit=42\n") + .output(); assert_eq!(out.detail_level, "full"); assert_eq!(out.limit, Some(42)); } diff --git a/crates/vestige-mcp/src/tools/codebase_unified.rs b/crates/vestige-mcp/src/tools/codebase_unified.rs index 70d8007..e24021d 100644 --- a/crates/vestige-mcp/src/tools/codebase_unified.rs +++ b/crates/vestige-mcp/src/tools/codebase_unified.rs @@ -428,7 +428,13 @@ mod tests { async fn test_invalid_action_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "action": "invalid" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid action")); } @@ -443,7 +449,13 @@ mod tests { "files": ["src/lib.rs"], "codebase": "vestige" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "remember_pattern"); @@ -459,7 +471,13 @@ mod tests { "action": "remember_pattern", "description": "Some description" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'name' is required")); } @@ -471,7 +489,13 @@ mod tests { "action": "remember_pattern", "name": "Test Pattern" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'description' is required")); } @@ -484,7 +508,13 @@ mod tests { "name": " ", "description": "Some description" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -500,7 +530,13 @@ mod tests { "files": ["src/storage.rs"], "codebase": "vestige" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "remember_decision"); @@ -515,7 +551,13 @@ mod tests { "action": "remember_decision", "rationale": "Some rationale" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'decision' is required")); } @@ -527,7 +569,13 @@ mod tests { "action": "remember_decision", "decision": "Use SQLite" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("'rationale' is required")); } @@ -540,7 +588,13 @@ mod tests { "decision": " ", "rationale": "Something" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -552,7 +606,13 @@ mod tests { "action": "get_context", "codebase": "nonexistent" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "get_context"); @@ -571,7 +631,9 @@ mod tests { "description": "A test pattern", "codebase": "myproject" }); - execute(&storage, &cog, &OutputConfig::default(), Some(save_args)).await.unwrap(); + execute(&storage, &cog, &OutputConfig::default(), Some(save_args)) + .await + .unwrap(); // Now retrieve let get_args = serde_json::json!({ @@ -588,7 +650,13 @@ mod tests { async fn test_get_context_no_codebase() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "action": "get_context" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); assert_eq!(value["action"], "get_context"); diff --git a/crates/vestige-mcp/src/tools/maintenance.rs b/crates/vestige-mcp/src/tools/maintenance.rs index 9dfb50e..ef1a927 100644 --- a/crates/vestige-mcp/src/tools/maintenance.rs +++ b/crates/vestige-mcp/src/tools/maintenance.rs @@ -143,8 +143,7 @@ pub async fn execute_system_status( ) -> Result { // Parse arguments (all optional, including the args envelope itself). let parsed: SystemStatusArgs = match args { - Some(v) => serde_json::from_value(v) - .map_err(|e| format!("Invalid arguments: {}", e))?, + Some(v) => serde_json::from_value(v).map_err(|e| format!("Invalid arguments: {}", e))?, None => SystemStatusArgs::default(), }; let include_schema = parsed.schema_introspection.unwrap_or(false); diff --git a/crates/vestige-mcp/src/tools/merge.rs b/crates/vestige-mcp/src/tools/merge.rs index f836f5f..70a2683 100644 --- a/crates/vestige-mcp/src/tools/merge.rs +++ b/crates/vestige-mcp/src/tools/merge.rs @@ -152,7 +152,11 @@ pub fn merge_policy_schema() -> Value { // ============================================================================ /// Route a merge/supersede tool call by tool name. -pub async fn execute(storage: &Arc, tool: &str, args: Option) -> Result { +pub async fn execute( + storage: &Arc, + tool: &str, + args: Option, +) -> Result { match tool { "merge_candidates" => merge_candidates(storage, args), "plan_merge" => plan_merge(storage, args), @@ -304,7 +308,8 @@ fn plan_supersede(storage: &Arc, args: Option) -> Result Value { - let requires_confirm = plan.classification != vestige_core::MatchClass::Match || !policy.auto_apply; + let requires_confirm = + plan.classification != vestige_core::MatchClass::Match || !policy.auto_apply; json!({ "planId": plan.id, "kind": plan.kind.as_str(), @@ -393,7 +398,9 @@ fn merge_undo(storage: &Arc, args: Option) -> Result { // No id => return the reflog so the caller can pick one. - let ops = storage.list_merge_operations(20).map_err(|e| e.to_string())?; + let ops = storage + .list_merge_operations(20) + .map_err(|e| e.to_string())?; let log: Vec = ops .iter() .map(|op| { @@ -478,7 +485,9 @@ fn merge_policy(storage: &Arc, args: Option) -> Result config file > // built-in default. The explicit arg is validated; the config fallback is // already validated at load time. - let detail_level_owned = - output_config.resolve_detail_level(args.detail_level.as_deref()); + let detail_level_owned = output_config.resolve_detail_level(args.detail_level.as_deref()); let detail_level = match detail_level_owned.as_str() { "brief" => "brief", "full" => "full", @@ -1030,7 +1029,13 @@ mod tests { async fn test_search_empty_query_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "query": "" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -1039,7 +1044,13 @@ mod tests { async fn test_search_whitespace_only_query_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "query": " \t\n " }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("empty")); } @@ -1056,7 +1067,13 @@ mod tests { async fn test_search_missing_query_field_fails() { let (storage, _dir) = test_storage().await; let args = serde_json::json!({ "limit": 10 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid arguments")); } @@ -1094,9 +1111,14 @@ mod tests { "query": "OPENAI_API_KEY", "limit": 5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) - .await - .unwrap(); + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await + .unwrap(); assert_eq!(result["method"], "concrete"); assert_eq!(result["concrete"], true); @@ -1118,9 +1140,14 @@ mod tests { "query": uuid, "limit": 5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) - .await - .unwrap(); + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await + .unwrap(); assert_eq!(result["method"], "concrete"); assert_eq!(result["results"][0]["id"], target); @@ -1144,9 +1171,14 @@ mod tests { "query": "mlx_lm.server", "limit": 5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) - .await - .unwrap(); + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await + .unwrap(); assert_eq!(result["method"], "concrete"); assert_eq!(result["results"][0]["id"], target); @@ -1166,7 +1198,13 @@ mod tests { "query": "test", "limit": 0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); } @@ -1180,7 +1218,13 @@ mod tests { "query": "test", "limit": 1000 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); } @@ -1193,7 +1237,13 @@ mod tests { "query": "test", "limit": -5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); } @@ -1210,7 +1260,13 @@ mod tests { "query": "test", "min_retention": -0.5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); } @@ -1223,7 +1279,13 @@ mod tests { "query": "test", "min_retention": 1.5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; // Should succeed but may return no results (retention > 1.0 clamped to 1.0) assert!(result.is_ok()); } @@ -1241,7 +1303,13 @@ mod tests { "query": "test", "min_similarity": -0.5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); } @@ -1254,7 +1322,13 @@ mod tests { "query": "test", "min_similarity": 1.5 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; // Should succeed but may return no results assert!(result.is_ok()); } @@ -1269,7 +1343,13 @@ mod tests { ingest_test_content(&storage, "The Rust programming language is memory safe.").await; let args = serde_json::json!({ "query": "rust" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1289,7 +1369,13 @@ mod tests { "query": "python", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1311,7 +1397,13 @@ mod tests { "limit": 2, "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1325,7 +1417,13 @@ mod tests { // Don't ingest anything - database is empty let args = serde_json::json!({ "query": "anything" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1342,7 +1440,13 @@ mod tests { "query": "testing", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1375,7 +1479,13 @@ mod tests { "query": "item", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1461,7 +1571,13 @@ mod tests { "detail_level": "brief", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1490,7 +1606,13 @@ mod tests { "detail_level": "full", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1517,7 +1639,13 @@ mod tests { "query": "default", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1546,7 +1674,13 @@ mod tests { "query": "test", "detail_level": "invalid_level" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid detail_level")); } @@ -1575,7 +1709,13 @@ mod tests { "token_budget": 200, "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1602,7 +1742,13 @@ mod tests { "token_budget": 150, "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1621,7 +1767,13 @@ mod tests { "query": "no budget", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -1677,11 +1829,7 @@ mod tests { /// Helper that ingests a memory with specific tags. The base /// `ingest_test_content` helper passes `tags: vec![]`, which is fine /// for legacy tests but not for tag_prefix coverage. - async fn ingest_with_tags( - storage: &Arc, - content: &str, - tags: Vec<&str>, - ) -> String { + async fn ingest_with_tags(storage: &Arc, content: &str, tags: Vec<&str>) -> String { let input = IngestInput { content: content.to_string(), node_type: "fact".to_string(), @@ -1725,7 +1873,13 @@ mod tests { "tag_prefix": "meeting:", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok(), "{:?}", result); let value = result.unwrap(); let results = value["results"].as_array().unwrap(); @@ -1768,7 +1922,13 @@ mod tests { "tag_prefix": "project:", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); let results = value["results"].as_array().unwrap(); @@ -1799,7 +1959,13 @@ mod tests { "query": "audit", "min_similarity": 0.0 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); let results = value["results"].as_array().unwrap(); @@ -1832,7 +1998,13 @@ mod tests { "concrete": true, "tag_prefix": "meeting:" }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok(), "{:?}", result); let value = result.unwrap(); assert_eq!(value["method"], "concrete"); @@ -1897,8 +2069,14 @@ mod tests { .unwrap(); assert_eq!(value["profile"], "lean"); if let Some(first) = value["results"].as_array().and_then(|a| a.first()) { - assert!(first.get("combinedScore").is_none(), "lean must drop scores"); - assert!(first.get("createdAt").is_none(), "lean must drop timestamps"); + assert!( + first.get("combinedScore").is_none(), + "lean must drop scores" + ); + assert!( + first.get("createdAt").is_none(), + "lean must drop timestamps" + ); } } @@ -1910,9 +2088,14 @@ mod tests { ingest_test_content(&storage, "Default profile preserved content.").await; let args = serde_json::json!({ "query": "default preserved", "min_similarity": 0.0 }); - let value = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) - .await - .unwrap(); + let value = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await + .unwrap(); assert_eq!(value["detailLevel"], "summary"); assert_eq!(value["profile"], "default"); if let Some(first) = value["results"].as_array().and_then(|a| a.first()) { diff --git a/crates/vestige-mcp/src/tools/session_context.rs b/crates/vestige-mcp/src/tools/session_context.rs index 68e5c6b..97e52ff 100644 --- a/crates/vestige-mcp/src/tools/session_context.rs +++ b/crates/vestige-mcp/src/tools/session_context.rs @@ -568,7 +568,13 @@ mod tests { let args = serde_json::json!({ "queries": ["user preferences", "project context"] }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -596,7 +602,13 @@ mod tests { "queries": ["memory"], "token_budget": 200 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -632,7 +644,13 @@ mod tests { "queries": ["expandable test memory"], "token_budget": 150 }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -663,7 +681,13 @@ mod tests { "include_intentions": false, "include_predictions": false }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -697,7 +721,13 @@ mod tests { "topics": ["performance"] } }); - let result = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)).await; + let result = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await; assert!(result.is_ok()); let value = result.unwrap(); @@ -715,9 +745,14 @@ mod tests { // Default profile -> profile echoed, dates present. let args = serde_json::json!({ "queries": ["profile content"] }); - let value = execute(&storage, &test_cognitive(), &OutputConfig::default(), Some(args)) - .await - .unwrap(); + let value = execute( + &storage, + &test_cognitive(), + &OutputConfig::default(), + Some(args), + ) + .await + .unwrap(); assert_eq!(value["profile"], "default"); // Lean profile -> profile echoed as lean. The memory line must not carry diff --git a/crates/vestige-mcp/src/tools/timeline.rs b/crates/vestige-mcp/src/tools/timeline.rs index 5c73357..7b3dcbf 100644 --- a/crates/vestige-mcp/src/tools/timeline.rs +++ b/crates/vestige-mcp/src/tools/timeline.rs @@ -417,7 +417,9 @@ mod tests { // Limit 5 against 12 total — before the fix, `retain` on `concept` // would operate on the 5 most recent rows (all `fact`) and find 0. let args = serde_json::json!({ "node_type": "concept", "limit": 5 }); - let value = execute(&storage, &OutputConfig::default(), Some(args)).await.unwrap(); + let value = execute(&storage, &OutputConfig::default(), Some(args)) + .await + .unwrap(); assert_eq!( value["totalMemories"], 2, "Both sparse concepts should survive a limit smaller than the dominant set" @@ -455,7 +457,9 @@ mod tests { } let args = serde_json::json!({ "tags": ["rare"], "limit": 5 }); - let value = execute(&storage, &OutputConfig::default(), Some(args)).await.unwrap(); + let value = execute(&storage, &OutputConfig::default(), Some(args)) + .await + .unwrap(); assert_eq!( value["totalMemories"], 2, "Both sparse-tag matches should survive a limit smaller than the dominant set" From 50e7f2d0fb6c226f7e081a67266f10e1ffece21e Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Fri, 19 Jun 2026 01:21:59 -0500 Subject: [PATCH 35/38] feat(connectors): external-source connector layer + GitHub Issues (#57) Make Vestige a durable, local, semantically-searchable retrieval layer over an external system of record (GitHub Issues first), citing back to the canonical record. Unlike a live ticket-system MCP proxy, Vestige keeps a durable embedded index: searchable offline, joinable with the rest of memory, temporally versioned, and re-syncable idempotently with no duplication. Phases 1-2 of #57 plus a GitHub reference connector and source-aware search: - Source envelope on KnowledgeNode/IngestInput (source_system, source_id, source_url, source_updated_at, content_hash, synced_at, source_project, source_type, source_author). Migration V17: nullable columns (additive), partial UNIQUE index on (source_system, source_id), connector_cursors table. - Idempotent sync primitives in vestige-core: upsert_by_source (content-hash change detection), connector cursor checkpoints, reconcile_source_tombstones (invalidate-don't-delete via bitemporal valid_until). - Connector contract + run_sync driver + GitHub Issues connector behind the optional `connectors` feature (on by default in vestige-mcp, off in the core library default so non-connector consumers link no HTTP client). - source_sync MCP tool ({"repo": "owner/name"}); token from GITHUB_TOKEN env only. Search results gain a sourceRecord citation for connector memories. Adversarial review fixes: GitHub `since` Z-form (the `+00:00` offset corrupted the cursor server-side), un-tombstone clears superseded_by too, cursor never advances past a failing record, Link next-url host-pinned (token-leak guard), records_seen counts new records only. Verified: cargo check/test/clippy -D warnings green across the workspace (default and connectors features); 483 core tests pass. Version bump to 2.1.27 and tag deferred to release. Refs #57 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 58 +- Cargo.lock | 99 +++ crates/vestige-core/Cargo.toml | 14 + crates/vestige-core/src/connectors/github.rs | 570 ++++++++++++++++ crates/vestige-core/src/connectors/mod.rs | 372 +++++++++++ .../vestige-core/src/consolidation/phases.rs | 1 + crates/vestige-core/src/lib.rs | 6 + crates/vestige-core/src/memory/mod.rs | 2 +- crates/vestige-core/src/memory/node.rs | 101 +++ crates/vestige-core/src/storage/migrations.rs | 183 +++++- crates/vestige-core/src/storage/mod.rs | 7 +- crates/vestige-core/src/storage/sqlite.rs | 610 +++++++++++++++++- crates/vestige-mcp/Cargo.toml | 6 +- crates/vestige-mcp/src/bin/cli.rs | 2 + crates/vestige-mcp/src/bin/restore.rs | 1 + crates/vestige-mcp/src/cognitive.rs | 1 + crates/vestige-mcp/src/server.rs | 25 +- crates/vestige-mcp/src/tools/changelog.rs | 1 + .../vestige-mcp/src/tools/codebase_unified.rs | 2 + .../vestige-mcp/src/tools/cross_reference.rs | 1 + crates/vestige-mcp/src/tools/dream.rs | 5 + crates/vestige-mcp/src/tools/explore.rs | 4 + crates/vestige-mcp/src/tools/feedback.rs | 2 + crates/vestige-mcp/src/tools/graph.rs | 3 + crates/vestige-mcp/src/tools/health.rs | 2 + crates/vestige-mcp/src/tools/maintenance.rs | 4 + .../vestige-mcp/src/tools/memory_unified.rs | 1 + crates/vestige-mcp/src/tools/mod.rs | 2 + crates/vestige-mcp/src/tools/restore.rs | 2 + crates/vestige-mcp/src/tools/review.rs | 1 + .../vestige-mcp/src/tools/search_unified.rs | 189 ++++-- .../vestige-mcp/src/tools/session_context.rs | 2 + crates/vestige-mcp/src/tools/smart_ingest.rs | 2 + crates/vestige-mcp/src/tools/source_sync.rs | 187 ++++++ crates/vestige-mcp/src/tools/suppress.rs | 1 + crates/vestige-mcp/src/tools/timeline.rs | 2 + docs/CONNECTORS.md | 150 +++++ tests/e2e/src/harness/db_manager.rs | 1 + tests/e2e/src/mocks/fixtures.rs | 1 + 39 files changed, 2538 insertions(+), 85 deletions(-) create mode 100644 crates/vestige-core/src/connectors/github.rs create mode 100644 crates/vestige-core/src/connectors/mod.rs create mode 100644 crates/vestige-mcp/src/tools/source_sync.rs create mode 100644 docs/CONNECTORS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5edf533..2918af4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,63 @@ All notable changes to Vestige will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased] — "External-Source Connectors" + +> Bump `version` in the workspace `Cargo.toml`, both crates, `server.json`, and +> `package.json` to `2.1.27` at release/tag time, and date this heading. + +Roadmap [#57](https://github.com/samvallad33/vestige/issues/57), **Phase 1–3**: +Vestige can now act as a durable, local, semantically-searchable retrieval layer +over an external system of record — starting with GitHub Issues — without +replacing it. The external system stays canonical; Vestige **indexes, connects, +retrieves, and cites back** to the source record. + +Unlike a live ticket-system MCP proxy (which holds no state and is rate-limited +per query), Vestige keeps a durable embedded index: searchable **offline**, +**semantically**, joinable with the rest of your memory, temporally versioned, +and re-syncable **idempotently** with no duplication. To our knowledge no other +local-first memory layer combines native connectors, external-URL provenance, +content-hash idempotent sync, and tombstoning of vanished records. + +### Added + +- **`source_sync` MCP tool** — point Vestige at a GitHub repo + (`{"repo": "owner/name"}`) and it indexes every issue + its comments as + source-aware memories. Re-running updates changed issues in place (no + duplicates); `reconcile: true` tombstones issues no longer visible upstream. + Auth via the `GITHUB_TOKEN` (or `VESTIGE_GITHUB_TOKEN`) environment variable; + public repos work without a token at a lower rate limit. +- **Source envelope** on every memory — structured, machine-readable provenance + (`source_system`, `source_id`, `source_url`, `source_updated_at`, + `content_hash`, `synced_at`, `source_project`, `source_type`, `source_author`) + distinct from the legacy free-form `source` label. Search results gain a + `sourceRecord` object (with the canonical `url`) **only** for + connector-ingested memories, so an agent can cite and follow the source. +- **Idempotent sync primitives** (`vestige-core`): `upsert_by_source` (keyed on + `(source_system, source_id)`, content-hash change detection), per-connector + cursor checkpoints (`connector_cursors`), and `reconcile_source_tombstones` + (invalidate-don't-delete via the bitemporal `valid_until`, so a vanished + record is retained for audit but drops out of current retrieval). +- **Connector contract** (`vestige_core::connectors`) — a small source-agnostic + `Connector` trait + `run_sync` driver (cursor overlap window, incremental + paging, optional deletion reconcile) and a GitHub Issues reference connector + behind the optional `connectors` cargo feature (on by default in the MCP + server, off in the core library's default features so non-connector consumers + link no HTTP client). + +### Database + +- **Migration V17** — nine nullable source-envelope columns on `knowledge_nodes` + (additive; every existing memory is untouched), a partial UNIQUE index on + `(source_system, source_id)` enforcing one memory per external record while + costing nothing for envelope-less legacy rows, and the `connector_cursors` + checkpoint table. Idempotent on replay, following the established + `add_column_if_missing` pattern. + +### Notes + +- Local-first and optional: with no `source_sync` call, behavior is unchanged. + The default core-library build does not link an HTTP client. ## [2.1.26] - 2026-06-15 — "Configurable Output" diff --git a/Cargo.lock b/Cargo.lock index 20e3853..2a88b74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,6 +521,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.44" @@ -1681,8 +1687,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1692,9 +1700,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1932,6 +1942,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots 1.0.6", ] [[package]] @@ -2519,6 +2530,12 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lzma-rust2" version = "0.15.7" @@ -3335,6 +3352,61 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -3571,6 +3643,8 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", @@ -3578,6 +3652,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -3587,6 +3662,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots 1.0.6", ] [[package]] @@ -3636,6 +3712,12 @@ dependencies = [ "sqlite-wasm-rs", ] +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustix" version = "1.1.4" @@ -3670,6 +3752,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] @@ -4162,6 +4245,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokenizers" version = "0.22.2" @@ -4684,6 +4782,7 @@ dependencies = [ "git2", "lru", "notify", + "reqwest", "rusqlite", "serde", "serde_json", diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index 0b8cb25..96a71b3 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -57,6 +57,12 @@ qwen3-embeddings = ["embeddings", "fastembed/qwen3", "dep:candle-core"] # Backwards-compatible feature alias from the original v2.1.0 naming. qwen3-reranker = ["qwen3-embeddings"] +# External-source connectors (#57). The connector *contract*, normalization, +# and content-hashing are always compiled (pure, no network). This feature adds +# the network-backed reference connectors (GitHub Issues, …) via `reqwest`, so +# the default local-first build never links an HTTP client. +connectors = ["dep:reqwest"] + # Metal GPU acceleration on Apple Silicon (significantly faster inference) metal = ["fastembed/metal"] @@ -132,6 +138,14 @@ lru = "0.16" trait-variant = "0.1" blake3 = "1" +# ============================================================================ +# OPTIONAL: External-source connectors (#57) +# ============================================================================ +# HTTP client for network-backed reference connectors (GitHub Issues, Redmine). +# rustls so connectors build with no system OpenSSL dependency. Behind the +# `connectors` feature — the default local-first build does not link reqwest. +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } + [dev-dependencies] tempfile = "3" criterion = { version = "0.5", features = ["html_reports"] } diff --git a/crates/vestige-core/src/connectors/github.rs b/crates/vestige-core/src/connectors/github.rs new file mode 100644 index 0000000..a373dcf --- /dev/null +++ b/crates/vestige-core/src/connectors/github.rs @@ -0,0 +1,570 @@ +//! GitHub Issues connector (#57). +//! +//! Indexes a repository's issues + comments into source-aware Vestige memories +//! so an agent can search and reason over the full issue history **offline**, +//! **semantically**, and **cited back to the canonical issue URL**. Unlike the +//! official GitHub MCP server — a stateless live API proxy — this builds a +//! durable, embedded, temporally-versioned local index. +//! +//! ## Incremental sync (per the connector sync contract) +//! +//! - `state=all` so closing an issue is not mistaken for a deletion. +//! - `sort=updated&direction=asc` so we page forward in cursor order and a +//! mid-run interruption resumes safely. +//! - `since=` filters on `updated_at`; the overlap + the +//! `content_hash` no-op makes re-scans safe and cheap. +//! - `Link: rel="next"` drives pagination (never hand-built page urls). +//! - Entries carrying a `pull_request` key are dropped (PRs are not issues). +//! - Per issue we fold the body + comments into one memory; the hash covers +//! the stable fields only (title, body, state, labels, comments) — never the +//! cursor timestamp or volatile counts. +//! +//! GitHub has no deletion feed, so deletions are reconciled out-of-band via +//! [`list_live_ids`](Connector::list_live_ids). + +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +use super::{ + Connector, ConnectorError, ConnectorResult, FetchPage, NormalizedRecord, content_hash, +}; +use crate::memory::SourceEnvelope; + +const API_ROOT: &str = "https://api.github.com"; +const USER_AGENT: &str = concat!("vestige-connector/", env!("CARGO_PKG_VERSION")); +const PER_PAGE: u32 = 100; + +/// Configuration for a GitHub Issues connector instance. +#[derive(Debug, Clone)] +pub struct GithubConfig { + /// Repository owner (user or org). + pub owner: String, + /// Repository name. + pub repo: String, + /// Personal access token. Optional for public repos (60 req/hr + /// unauthenticated) but strongly recommended (5000 req/hr authenticated). + pub token: Option, + /// Override the API root (for GitHub Enterprise or tests). + pub api_root: Option, + /// Max comments to fold into one issue memory (defense against huge threads). + pub max_comments: usize, +} + +impl GithubConfig { + pub fn new(owner: impl Into, repo: impl Into) -> Self { + Self { + owner: owner.into(), + repo: repo.into(), + token: None, + api_root: None, + max_comments: 50, + } + } + + pub fn with_token(mut self, token: Option) -> Self { + self.token = token; + self + } + + fn scope(&self) -> String { + format!("{}/{}", self.owner, self.repo) + } + + fn root(&self) -> &str { + self.api_root.as_deref().unwrap_or(API_ROOT) + } +} + +/// A GitHub Issues connector bound to one repository. +pub struct GithubConnector { + config: GithubConfig, + scope: String, + client: reqwest::Client, +} + +impl GithubConnector { + pub fn new(config: GithubConfig) -> ConnectorResult { + if config.owner.is_empty() || config.repo.is_empty() { + return Err(ConnectorError::Config( + "owner and repo are required".to_string(), + )); + } + let client = reqwest::Client::builder() + .user_agent(USER_AGENT) + .build() + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + let scope = config.scope(); + Ok(Self { + config, + scope, + client, + }) + } + + fn auth(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + let req = req + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28"); + match &self.config.token { + Some(t) => req.bearer_auth(t), + None => req, + } + } + + /// Map an HTTP response status into a connector error, honoring rate-limit + /// signals so the driver can back off politely. + fn classify_status(resp: &reqwest::Response) -> Option { + let status = resp.status(); + if status.is_success() { + return None; + } + // Primary rate limit: 403/429 with remaining=0. + if status.as_u16() == 403 || status.as_u16() == 429 { + let remaining = resp + .headers() + .get("x-ratelimit-remaining") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()); + if remaining == Some(0) || status.as_u16() == 429 { + let retry = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .map(std::time::Duration::from_secs); + return Some(ConnectorError::RateLimited(retry)); + } + } + Some(ConnectorError::Source { + status: status.as_u16(), + message: status + .canonical_reason() + .unwrap_or("request failed") + .to_string(), + }) + } + + /// Parse the `Link` header for the `rel="next"` url, if any. + /// + /// The `next` url comes from the server response, so we pin it to the + /// configured API host before following it: otherwise a malicious or + /// compromised endpoint could redirect the connector — which attaches the + /// bearer token to every request — to an attacker-controlled URL and + /// exfiltrate the credential (SSRF / token leak). `expected_host` is the + /// host of the connector's API root. + fn next_link(resp: &reqwest::Response, expected_host: Option<&str>) -> Option { + let link = resp.headers().get(reqwest::header::LINK)?.to_str().ok()?; + for part in link.split(',') { + let part = part.trim(); + if part.contains("rel=\"next\"") + && let (Some(start), Some(end)) = (part.find('<'), part.find('>')) + && start < end + { + let url = &part[start + 1..end]; + // Host-pin: only follow a next-url on the same host as the API + // root we were configured with. + if let Some(expected) = expected_host { + match reqwest::Url::parse(url) { + Ok(parsed) if parsed.host_str() == Some(expected) => { + return Some(url.to_string()); + } + _ => { + tracing::warn!( + next_url = url, + "dropping cross-host Link next url (host pin)" + ); + return None; + } + } + } + return Some(url.to_string()); + } + } + None + } + + /// Host of the configured API root, used to pin Link `next` urls. + fn api_host(&self) -> Option { + reqwest::Url::parse(self.config.root()) + .ok() + .and_then(|u| u.host_str().map(|h| h.to_string())) + } + + /// Fetch the comments for one issue (a single page; capped by `max_comments`). + async fn fetch_comments(&self, issue_number: u64) -> ConnectorResult> { + let url = format!( + "{}/repos/{}/{}/issues/{}/comments?per_page={}", + self.config.root(), + self.config.owner, + self.config.repo, + issue_number, + self.config.max_comments.min(100), + ); + let resp = self + .auth(self.client.get(&url)) + .send() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if let Some(err) = Self::classify_status(&resp) { + return Err(err); + } + resp.json::>() + .await + .map_err(|e| ConnectorError::Transport(e.to_string())) + } + + /// Fold a raw issue + its comments into one normalized memory record. + fn normalize(&self, issue: &RawIssue, comments: &[RawComment]) -> NormalizedRecord { + let author = issue.user.as_ref().map(|u| u.login.clone()); + + // Human-readable content: header + body + chronological comments. + let mut content = format!( + "[{}#{}] {}\nState: {}\n", + self.scope, issue.number, issue.title, issue.state + ); + if let Some(body) = &issue.body + && !body.trim().is_empty() + { + content.push('\n'); + content.push_str(body.trim()); + content.push('\n'); + } + let mut sorted_comments: Vec<&RawComment> = comments.iter().collect(); + sorted_comments.sort_by_key(|c| c.id); + for c in &sorted_comments { + let who = c.user.as_ref().map(|u| u.login.as_str()).unwrap_or("?"); + content.push_str(&format!("\n— {who}: {}", c.body.trim())); + } + + // Labels, sorted for a stable hash. + let mut labels: Vec = issue.labels.iter().map(|l| l.name.clone()).collect(); + labels.sort(); + + // Stable content hash — meaning only, never the cursor timestamp or + // volatile counts. Comments contribute their id+body in id order. + let comments_blob = sorted_comments + .iter() + .map(|c| format!("{}:{}", c.id, c.body.trim())) + .collect::>() + .join("\u{1f}"); + let labels_blob = labels.join(","); + let number_str = issue.number.to_string(); + let body_str = issue.body.clone().unwrap_or_default(); + let hash = content_hash(&[ + ("number", &number_str), + ("title", &issue.title), + ("state", &issue.state), + ("body", &body_str), + ("labels", &labels_blob), + ("comments", &comments_blob), + ]); + + let mut tags = vec![ + "github".to_string(), + "issue".to_string(), + format!("state:{}", issue.state), + ]; + tags.extend(labels.into_iter().map(|l| format!("label:{l}"))); + + let envelope = SourceEnvelope { + source_system: Some("github".to_string()), + source_id: Some(issue.number.to_string()), + source_url: Some(issue.html_url.clone()), + source_updated_at: DateTime::parse_from_rfc3339(&issue.updated_at) + .ok() + .map(|d| d.with_timezone(&Utc)), + content_hash: Some(hash), + synced_at: Some(Utc::now()), + source_project: Some(self.scope.clone()), + source_type: Some("issue".to_string()), + source_author: author, + }; + + NormalizedRecord { + content, + tags, + envelope, + } + } +} + +impl Connector for GithubConnector { + fn source_system(&self) -> &str { + "github" + } + + fn scope(&self) -> &str { + &self.scope + } + + async fn fetch_updated( + &self, + since: Option>, + cursor: Option, + ) -> ConnectorResult { + // `cursor` is a full next-page url from a previous Link header; on the + // first page we build the url from owner/repo + since. + let url = match cursor { + Some(u) => u, + None => { + let mut u = format!( + "{}/repos/{}/{}/issues?state=all&sort=updated&direction=asc&per_page={}", + self.config.root(), + self.config.owner, + self.config.repo, + PER_PAGE, + ); + if let Some(s) = since { + // GitHub documents the `since` format as YYYY-MM-DDTHH:MM:SSZ. + // `to_rfc3339()` emits the `+00:00` offset form, and the `+` + // is a reserved query char that the server decodes as a + // space — corrupting the timestamp and silently re-fetching + // all history every run. Emit the `Z` form (no reserved + // char, exact documented format) instead. + let since_z = s.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + u.push_str(&format!("&since={since_z}")); + } + u + } + }; + + let resp = self + .auth(self.client.get(&url)) + .send() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if let Some(err) = Self::classify_status(&resp) { + return Err(err); + } + let next_cursor = Self::next_link(&resp, self.api_host().as_deref()); + let issues: Vec = resp + .json() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + + let mut records = Vec::new(); + for issue in &issues { + // Drop pull requests — "every PR is an issue, but not vice versa". + if issue.pull_request.is_some() { + continue; + } + // Fetch comments only when the issue has any. + let comments = if issue.comments > 0 { + self.fetch_comments(issue.number).await.unwrap_or_default() + } else { + Vec::new() + }; + records.push(self.normalize(issue, &comments)); + } + + Ok(FetchPage { + records, + next_cursor, + }) + } + + async fn list_live_ids(&self) -> ConnectorResult>> { + // Enumerate all issue numbers (ids only) for the reconcile pass, paging + // via Link. Cheap relative to full sync (no comment fetch, no bodies). + let mut ids = Vec::new(); + let mut url = Some(format!( + "{}/repos/{}/{}/issues?state=all&per_page={}", + self.config.root(), + self.config.owner, + self.config.repo, + PER_PAGE, + )); + while let Some(u) = url { + let resp = self + .auth(self.client.get(&u)) + .send() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if let Some(err) = Self::classify_status(&resp) { + return Err(err); + } + let next = Self::next_link(&resp, self.api_host().as_deref()); + let issues: Vec = resp + .json() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + for issue in issues { + if issue.pull_request.is_none() { + ids.push(issue.number.to_string()); + } + } + url = next; + } + Ok(Some(ids)) + } +} + +// --------------------------------------------------------------------------- +// Raw GitHub API shapes (only the fields we use) +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct RawIssue { + number: u64, + title: String, + #[serde(default)] + body: Option, + state: String, + html_url: String, + updated_at: String, + #[serde(default)] + comments: u64, + #[serde(default)] + labels: Vec, + #[serde(default)] + user: Option, + /// Present iff this "issue" is actually a pull request. + #[serde(default)] + pull_request: Option, +} + +#[derive(Debug, Deserialize)] +struct RawLabel { + name: String, +} + +#[derive(Debug, Deserialize)] +struct RawUser { + login: String, +} + +#[derive(Debug, Deserialize)] +struct RawComment { + id: u64, + body: String, + #[serde(default)] + user: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn issue(number: u64, title: &str, body: &str, state: &str) -> RawIssue { + RawIssue { + number, + title: title.to_string(), + body: Some(body.to_string()), + state: state.to_string(), + html_url: format!("https://github.com/o/r/issues/{number}"), + updated_at: "2026-06-19T00:00:00Z".to_string(), + comments: 0, + labels: vec![RawLabel { + name: "bug".to_string(), + }], + user: Some(RawUser { + login: "octocat".to_string(), + }), + pull_request: None, + } + } + + fn connector() -> GithubConnector { + GithubConnector::new(GithubConfig::new("o", "r")).unwrap() + } + + #[test] + fn normalize_builds_keyed_envelope_with_citation() { + let c = connector(); + let rec = c.normalize(&issue(57, "Connectors", "Add Redmine", "open"), &[]); + let env = &rec.envelope; + assert!(env.has_key()); + assert_eq!(env.source_system.as_deref(), Some("github")); + assert_eq!(env.source_id.as_deref(), Some("57")); + assert_eq!( + env.source_url.as_deref(), + Some("https://github.com/o/r/issues/57") + ); + assert_eq!(env.source_project.as_deref(), Some("o/r")); + assert!(rec.content.contains("Connectors")); + assert!(rec.tags.contains(&"state:open".to_string())); + assert!(rec.tags.contains(&"label:bug".to_string())); + } + + #[test] + fn hash_stable_across_label_order_and_changes_on_edit() { + let c = connector(); + let mut a = issue(1, "T", "body", "open"); + a.labels = vec![ + RawLabel { name: "b".into() }, + RawLabel { name: "a".into() }, + ]; + let mut b = issue(1, "T", "body", "open"); + b.labels = vec![ + RawLabel { name: "a".into() }, + RawLabel { name: "b".into() }, + ]; + let ha = c.normalize(&a, &[]).envelope.content_hash; + let hb = c.normalize(&b, &[]).envelope.content_hash; + assert_eq!(ha, hb, "label order must not change the hash"); + + // Editing the body must change the hash. + let edited = c.normalize(&issue(1, "T", "EDITED", "open"), &[]).envelope.content_hash; + assert_ne!(ha, edited); + + // Closing the issue changes state → changes the hash (not a no-op). + let closed = c.normalize(&issue(1, "T", "body", "closed"), &[]).envelope.content_hash; + assert_ne!(ha, closed); + } + + #[test] + fn comments_fold_in_id_order_and_affect_hash() { + let c = connector(); + let comments = vec![ + RawComment { + id: 2, + body: "second".into(), + user: Some(RawUser { login: "x".into() }), + }, + RawComment { + id: 1, + body: "first".into(), + user: Some(RawUser { login: "y".into() }), + }, + ]; + let rec = c.normalize(&issue(1, "T", "body", "open"), &comments); + // Folded in id order regardless of input order. + let first_pos = rec.content.find("first").unwrap(); + let second_pos = rec.content.find("second").unwrap(); + assert!(first_pos < second_pos, "comments must fold in id order"); + + let no_comments = c.normalize(&issue(1, "T", "body", "open"), &[]).envelope.content_hash; + assert_ne!( + rec.envelope.content_hash, no_comments, + "comments must contribute to the hash" + ); + } + + #[test] + fn rejects_empty_owner_repo() { + assert!(GithubConnector::new(GithubConfig::new("", "r")).is_err()); + assert!(GithubConnector::new(GithubConfig::new("o", "")).is_err()); + } + + #[test] + fn since_uses_z_form_not_plus_offset() { + // Regression: to_rfc3339() emits `+00:00`; the `+` decodes to a space + // server-side and corrupts the cursor. We must emit the `Z` form. + let ts = DateTime::parse_from_rfc3339("2026-06-19T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let z = ts.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + assert_eq!(z, "2026-06-19T00:00:00Z"); + assert!(!z.contains('+'), "since must not contain a reserved '+'"); + } + + #[test] + fn next_link_host_pin_drops_cross_host_url() { + // The host-pin parsing logic (used to prevent token exfiltration via a + // malicious Link header) must reject a different host. + let same = reqwest::Url::parse("https://api.github.com/x?page=2").unwrap(); + let other = reqwest::Url::parse("https://evil.example/x?page=2").unwrap(); + assert_eq!(same.host_str(), Some("api.github.com")); + assert_ne!(other.host_str(), Some("api.github.com")); + } +} diff --git a/crates/vestige-core/src/connectors/mod.rs b/crates/vestige-core/src/connectors/mod.rs new file mode 100644 index 0000000..033e065 --- /dev/null +++ b/crates/vestige-core/src/connectors/mod.rs @@ -0,0 +1,372 @@ +//! External-source connectors (#57). +//! +//! A connector turns records in a long-lived external system (a ticket tracker, +//! an issue board, a support queue) into source-aware Vestige memories, so an +//! investigative agent can search and reason over years of history **offline**, +//! **semantically**, and **cited back to the canonical record** — something no +//! live ticket-system MCP proxy can do. +//! +//! ## Layering +//! +//! - The [`Connector`] contract, [`NormalizedRecord`] shape, and the stable +//! [`content_hash`] are pure (no network) and always compiled, so the sync +//! semantics are unit-testable without hitting an API. +//! - Network-backed reference connectors (e.g. [`github`]) live behind the +//! `connectors` cargo feature so the default local-first build links no HTTP +//! client. +//! +//! ## Sync contract (the part that makes re-running safe) +//! +//! Every connector produces [`NormalizedRecord`]s. Each carries a +//! [`SourceEnvelope`](crate::memory::SourceEnvelope) whose +//! `(source_system, source_id)` is the idempotency key and whose `content_hash` +//! is the change detector. The driver routes each record through +//! [`upsert_by_source`](crate::storage::SqliteMemoryStore::upsert_by_source): +//! +//! - unseen record → insert +//! - changed `content_hash` → update in place (+ re-embed) +//! - same `content_hash` → no-op (only liveness advances) +//! +//! Because neither GitHub nor Redmine expose a deletion feed, deletions are +//! handled out-of-band by a periodic reconcile pass +//! ([`reconcile_source_tombstones`](crate::storage::SqliteMemoryStore::reconcile_source_tombstones)). + +use chrono::{DateTime, Utc}; + +use crate::memory::{IngestInput, SourceEnvelope}; +use crate::storage::ConnectorCursor; + +#[cfg(feature = "connectors")] +pub mod github; + +/// A single external record, already normalized into the fields Vestige needs. +/// +/// The connector is responsible for flattening a possibly-rich source record +/// (an issue plus its comments / journals / status changes) into a single +/// retrievable `content` blob plus the structured envelope. Keeping one memory +/// per logical record (rather than per comment) keeps retrieval coherent and +/// the idempotency key simple. +#[derive(Debug, Clone)] +pub struct NormalizedRecord { + /// Human-readable content to embed and search over. + pub content: String, + /// Tags for categorization (e.g. `["github", "issue", "state:open"]`). + pub tags: Vec, + /// The provenance envelope. `source_system`, `source_id`, and `content_hash` + /// MUST be set for idempotent upsert. + pub envelope: SourceEnvelope, +} + +impl NormalizedRecord { + /// Convert into an [`IngestInput`] ready for `upsert_by_source`. + pub fn into_ingest_input(self) -> IngestInput { + IngestInput { + content: self.content, + node_type: "event".to_string(), + source: self.envelope.source_url.clone(), + tags: self.tags, + source_envelope: Some(self.envelope), + ..Default::default() + } + } +} + +/// One page of records plus the cursor needed to fetch the next page. +#[derive(Debug, Clone, Default)] +pub struct FetchPage { + pub records: Vec, + /// Opaque token to resume after this page, or `None` when exhausted. + pub next_cursor: Option, +} + +/// Errors a connector can surface. +#[derive(Debug, thiserror::Error)] +pub enum ConnectorError { + #[error("configuration error: {0}")] + Config(String), + #[error("transport error: {0}")] + Transport(String), + #[error("rate limited; retry after {0:?}")] + RateLimited(Option), + #[error("source error ({status}): {message}")] + Source { status: u16, message: String }, +} + +pub type ConnectorResult = Result; + +/// The contract every external-source connector implements. +/// +/// Intentionally minimal: fetch a window of records updated since a cursor, +/// page through them, and (separately) enumerate currently-live ids for the +/// deletion-reconcile pass. The driver owns persistence, embedding, and cursor +/// checkpointing — a connector is just a typed, incremental reader. +#[allow(async_fn_in_trait)] +pub trait Connector { + /// Stable system identifier written into every envelope (`github`, …). + fn source_system(&self) -> &str; + + /// The scope this connector instance is bound to (`owner/repo`, project id). + fn scope(&self) -> &str; + + /// Fetch one page of records whose source-updated time is `>= since` + /// (inclusive on purpose — see the overlap note below), resuming from + /// `cursor` when provided. Records should be returned in ascending + /// update-time order so a mid-run interruption resumes safely. + /// + /// Callers pass `since = checkpoint − overlap` (a few minutes) so a record + /// written with a slightly-behind upstream clock, or one sharing the exact + /// boundary second, is never skipped. The `content_hash` short-circuit in + /// `upsert_by_source` makes the resulting re-scan free. + async fn fetch_updated( + &self, + since: Option>, + cursor: Option, + ) -> ConnectorResult; + + /// Enumerate the ids currently visible upstream for this scope, for the + /// deletion-reconcile pass. Cheap (ids only). `None` means the connector + /// cannot enumerate, so the driver must skip reconciliation rather than + /// tombstone everything. + async fn list_live_ids(&self) -> ConnectorResult>> { + Ok(None) + } +} + +/// Recommended overlap subtracted from the saved cursor before the next fetch, +/// to absorb clock skew and same-second boundary updates (the `>=` window). +pub const CURSOR_OVERLAP_SECS: i64 = 120; + +/// Summary of one sync run, returned to the caller / surfaced by the MCP tool. +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct SyncReport { + pub source_system: String, + pub scope: String, + pub created: usize, + pub updated: usize, + pub unchanged: usize, + pub tombstoned: usize, + /// New high-water mark persisted as the cursor for the next run. + pub new_cursor: Option>, + /// Whether a deletion-reconcile pass ran this time. + pub reconciled: bool, + /// Non-fatal warnings (e.g. a page that failed and was skipped). + pub warnings: Vec, +} + +/// Drive a full incremental sync of one connector into the store (#57). +/// +/// This is the orchestration the MCP `source_sync` tool calls. It: +/// 1. loads the saved checkpoint and starts from `cursor − overlap` (the `>=` +/// window that prevents missing same-second / clock-skewed updates); +/// 2. pages the connector forward in update order, routing each record through +/// [`upsert_by_source`](crate::storage::SqliteMemoryStore::upsert_by_source) +/// (insert / update-in-place / no-op by content hash); +/// 3. advances the cursor to the max `source_updated_at` actually observed, +/// persisting it only after the run so a crash re-scans rather than skips; +/// 4. optionally reconciles deletions when `reconcile` is set and the connector +/// can enumerate live ids. +/// +/// `max_pages` bounds a single run (so a first sync of a 15-year tracker can be +/// resumed across calls rather than blocking on one enormous fetch). +pub async fn run_sync( + store: &crate::storage::SqliteMemoryStore, + connector: &C, + reconcile: bool, + max_pages: usize, +) -> ConnectorResult { + use crate::storage::SourceUpsertOutcome; + + let source_system = connector.source_system().to_string(); + let scope = connector.scope().to_string(); + + let mut report = SyncReport { + source_system: source_system.clone(), + scope: scope.clone(), + ..Default::default() + }; + + // 1. Load checkpoint, apply the overlap window. + let checkpoint = store + .get_connector_cursor(&source_system, &scope) + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + let since = checkpoint + .cursor_updated_at + .map(|c| c - chrono::Duration::seconds(CURSOR_OVERLAP_SECS)); + + // 2. Page forward, upserting each record. + let mut cursor: Option = None; + let mut max_seen = checkpoint.cursor_updated_at; + // Oldest source_updated_at among records that FAILED to upsert this run. We + // must not advance the persisted cursor past this, or the failed record — + // fetched in ascending update order — would fall outside the next run's + // `since` window and never be retried (a silent permanent gap). + let mut oldest_failure: Option> = None; + // Count of genuinely new records (Created). Unchanged re-scans of the + // overlap window must not inflate the running total. + let mut created_this_run = 0i64; + + for _ in 0..max_pages.max(1) { + let page = connector.fetch_updated(since, cursor.clone()).await?; + for record in page.records { + let observed = record.envelope.source_updated_at; + match store.upsert_by_source(record.into_ingest_input()) { + Ok(res) => { + match res.outcome { + SourceUpsertOutcome::Created => { + report.created += 1; + created_this_run += 1; + } + SourceUpsertOutcome::Updated => report.updated += 1, + SourceUpsertOutcome::Unchanged => report.unchanged += 1, + } + if let Some(ts) = observed + && max_seen.map(|m| ts > m).unwrap_or(true) + { + max_seen = Some(ts); + } + } + Err(e) => { + report.warnings.push(format!("upsert failed: {e}")); + if let Some(ts) = observed + && oldest_failure.map(|f| ts < f).unwrap_or(true) + { + oldest_failure = Some(ts); + } + } + } + } + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + + // Clamp the cursor so we never advance past a record that failed this run. + // Subtract one second so the next run's inclusive `since` re-includes it. + if let Some(failed_at) = oldest_failure { + let clamp_to = failed_at - chrono::Duration::seconds(1); + max_seen = Some(match max_seen { + Some(m) if m < clamp_to => m, + _ => clamp_to, + }); + } + + // 3. Optional deletion reconciliation. + let mut reconciled = false; + if reconcile { + match connector.list_live_ids().await { + Ok(Some(live_ids)) => { + match store.reconcile_source_tombstones(&source_system, &scope, &live_ids) { + Ok(r) => { + report.tombstoned = r.tombstoned.len(); + reconciled = true; + } + Err(e) => report.warnings.push(format!("reconcile failed: {e}")), + } + } + Ok(None) => report + .warnings + .push("connector cannot enumerate live ids; skipped reconcile".to_string()), + Err(e) => report.warnings.push(format!("list_live_ids failed: {e}")), + } + } + report.reconciled = reconciled; + report.new_cursor = max_seen; + + // 4. Persist the checkpoint (only after the run). + let now = Utc::now(); + let new_checkpoint = ConnectorCursor { + source_system: source_system.clone(), + scope: scope.clone(), + cursor_updated_at: max_seen, + last_synced_at: Some(now), + last_full_reconcile_at: if reconciled { + Some(now) + } else { + checkpoint.last_full_reconcile_at + }, + // Accumulate only NEW records, so re-scanning the overlap window (which + // reports Unchanged) does not inflate the running total. + records_seen: checkpoint.records_seen + created_this_run, + }; + store + .save_connector_cursor(&new_checkpoint) + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + + Ok(report) +} + +/// Compute a stable content hash over the record's meaning. +/// +/// Stability requirements (so re-syncing an unchanged record is a true no-op): +/// - **key order independent** — callers pass `(field, value)` pairs which we +/// sort before hashing, so map/field ordering never changes the digest; +/// - **volatile fields excluded** — the caller must omit the cursor timestamp, +/// view/comment counts, and ephemeral permission flags (hash the meaning, +/// not the metadata); +/// - **collision-resistant** — BLAKE3 (already a Vestige dependency). +/// +/// Comment/journal arrays should be flattened into the pairs in a stable order +/// (sorted by their own id) by the caller before hashing. +pub fn content_hash(fields: &[(&str, &str)]) -> String { + let mut pairs: Vec<(&str, &str)> = fields.to_vec(); + pairs.sort_by(|a, b| a.0.cmp(b.0).then(a.1.cmp(b.1))); + + let mut hasher = blake3::Hasher::new(); + for (k, v) in pairs { + // Length-prefix each field so ("ab","c") can't collide with ("a","bc"). + hasher.update(&(k.len() as u64).to_le_bytes()); + hasher.update(k.as_bytes()); + hasher.update(&(v.len() as u64).to_le_bytes()); + hasher.update(v.as_bytes()); + } + hasher.finalize().to_hex().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_hash_is_order_independent() { + let a = content_hash(&[("title", "Crash"), ("body", "stacktrace"), ("state", "open")]); + let b = content_hash(&[("state", "open"), ("title", "Crash"), ("body", "stacktrace")]); + assert_eq!(a, b, "reordering fields must not change the hash"); + } + + #[test] + fn content_hash_changes_with_content() { + let a = content_hash(&[("body", "v1")]); + let b = content_hash(&[("body", "v2")]); + assert_ne!(a, b, "different content must hash differently"); + } + + #[test] + fn content_hash_no_boundary_collision() { + // ("ab","c") vs ("a","bc") must differ thanks to length prefixing. + let a = content_hash(&[("ab", "c")]); + let b = content_hash(&[("a", "bc")]); + assert_ne!(a, b); + } + + #[test] + fn normalized_record_carries_envelope_into_input() { + let rec = NormalizedRecord { + content: "issue body".to_string(), + tags: vec!["github".to_string()], + envelope: SourceEnvelope { + source_system: Some("github".to_string()), + source_id: Some("42".to_string()), + source_url: Some("https://example/42".to_string()), + content_hash: Some("h".to_string()), + ..Default::default() + }, + }; + let input = rec.into_ingest_input(); + assert_eq!(input.content, "issue body"); + assert_eq!(input.source.as_deref(), Some("https://example/42")); + let env = input.source_envelope.unwrap(); + assert!(env.has_key()); + assert_eq!(env.source_id.as_deref(), Some("42")); + } +} diff --git a/crates/vestige-core/src/consolidation/phases.rs b/crates/vestige-core/src/consolidation/phases.rs index abf6019..7f74489 100644 --- a/crates/vestige-core/src/consolidation/phases.rs +++ b/crates/vestige-core/src/consolidation/phases.rs @@ -889,6 +889,7 @@ mod tests { embedding_model: None, suppression_count: 0, suppressed_at: None, + source_envelope: None, } } diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index 15dbdbf..e832828 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -82,6 +82,7 @@ /// Optional `vestige.toml` configuration (Phase 2: Configurable Output). pub mod config; +pub mod connectors; pub mod consolidation; pub mod embedder; pub mod fsrs; @@ -134,6 +135,7 @@ pub use memory::{ SearchMode, SearchResult, SimilarityResult, + SourceEnvelope, TableIntrospection, TemporalRange, }; @@ -166,6 +168,7 @@ pub use storage::{ CompositionNeighborRecord, CompositionOutcomeRecord, ConnectionRecord, + ConnectorCursor, ConsolidationHistoryRecord, Domain, DreamHistoryRecord, @@ -184,10 +187,13 @@ pub use storage::{ PortableArchive, PortableImportMode, PortableImportReport, + ReconcileReport, Result, SchedulingState, SearchQuery, SmartIngestResult, + SourceUpsertOutcome, + SourceUpsertResult, SqliteMemoryStore, StateTransitionRecord, Storage, diff --git a/crates/vestige-core/src/memory/mod.rs b/crates/vestige-core/src/memory/mod.rs index 8cd618e..af9daeb 100644 --- a/crates/vestige-core/src/memory/mod.rs +++ b/crates/vestige-core/src/memory/mod.rs @@ -10,7 +10,7 @@ mod node; mod strength; mod temporal; -pub use node::{IngestInput, KnowledgeNode, NodeType, RecallInput, SearchMode}; +pub use node::{IngestInput, KnowledgeNode, NodeType, RecallInput, SearchMode, SourceEnvelope}; pub use strength::{DualStrength, StrengthDecay}; pub use temporal::{TemporalRange, TemporalValidity}; diff --git a/crates/vestige-core/src/memory/node.rs b/crates/vestige-core/src/memory/node.rs index 9785387..c400afc 100644 --- a/crates/vestige-core/src/memory/node.rs +++ b/crates/vestige-core/src/memory/node.rs @@ -79,6 +79,91 @@ impl std::fmt::Display for NodeType { } } +// ============================================================================ +// SOURCE ENVELOPE (#57) +// ============================================================================ + +/// Structured provenance for a memory that mirrors a record in an external +/// system of record (a Redmine issue, a GitHub Issue, a Jira ticket, a support +/// thread, …). +/// +/// The product boundary (#57): the external system stays canonical. Vestige +/// **indexes, connects, retrieves, and cites back**; it does not replace the +/// ticket tracker. This envelope carries exactly the fields a connector needs +/// to do that without leaking stale data: +/// +/// - `(source_system, source_id)` is the **idempotency key**. Re-running a sync +/// upserts the same logical record instead of duplicating it. +/// - `content_hash` is the **change detector**. If a re-fetched record hashes +/// to the stored value, the upsert is a no-op (only `synced_at` advances), +/// so an incremental re-scan never churns the index or the embedding model. +/// - `source_url` is the **citation**. Search results link back to the +/// canonical record so the agent can follow it for authoritative detail. +/// - `source_updated_at` is the **cursor field** the connector checkpoints on. +/// +/// Every field is optional at the type level so partial connectors and manual +/// imports can populate only what they have, but a real connector should always +/// set `source_system`, `source_id`, and `content_hash`. +#[non_exhaustive] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SourceEnvelope { + /// External system this record came from, e.g. `redmine`, `github`, `jira`. + /// Namespaces `source_id` so two systems can share numeric ids. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_system: Option, + /// Stable native id in the source system (Redmine issue id, GitHub issue + /// number/node id, …). Combined with `source_system` it is the upsert key. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_id: Option, + /// Canonical URL back to the record so retrieval can cite the source. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_url: Option, + /// When the source record was last updated upstream (the connector cursor + /// field — Redmine `updated_on`, GitHub `updated_at`). RFC 3339. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_updated_at: Option>, + /// Stable hash of the normalized record content. Idempotency / change key. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content_hash: Option, + /// When the connector last observed this record live. Drives tombstone + /// reconciliation (a record not seen in a full reconcile pass is gone). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub synced_at: Option>, + /// Project / repo / space the record belongs to (Redmine project, GitHub + /// `owner/repo`). Used for scoped sync and search filters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_project: Option, + /// Record type within the source (`issue`, `comment`, `journal`, …). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_type: Option, + /// Author / reporter of the record in the source system. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_author: Option, +} + +impl SourceEnvelope { + /// True once the two fields a connector needs for idempotent upsert are + /// present. Manual imports that only set `source_url` are not "keyed". + pub fn has_key(&self) -> bool { + self.source_system.is_some() && self.source_id.is_some() + } + + /// True if every field is unset — used to collapse an all-`None` envelope + /// back to `None` on the node so legacy rows stay clean. + pub fn is_empty(&self) -> bool { + self.source_system.is_none() + && self.source_id.is_none() + && self.source_url.is_none() + && self.source_updated_at.is_none() + && self.content_hash.is_none() + && self.synced_at.is_none() + && self.source_project.is_none() + && self.source_type.is_none() + && self.source_author.is_none() + } +} + // ============================================================================ // KNOWLEDGE NODE // ============================================================================ @@ -188,6 +273,15 @@ pub struct KnowledgeNode { /// Timestamp of the most recent suppression (for 24h labile window). #[serde(skip_serializing_if = "Option::is_none")] pub suppressed_at: Option>, + + // ========== Source Envelope (#57, external-source connectors) ========== + /// Structured provenance for memories ingested from an external system + /// (Redmine, GitHub Issues, Jira, …). `None` for memories created directly + /// by an agent or the user — the legacy free-form `source` string above + /// remains the human-readable label; this envelope is the machine-readable, + /// idempotency- and citation-bearing record. See [`SourceEnvelope`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_envelope: Option, } impl Default for KnowledgeNode { @@ -224,6 +318,7 @@ impl Default for KnowledgeNode { embedding_model: None, suppression_count: 0, suppressed_at: None, + source_envelope: None, } } } @@ -291,6 +386,11 @@ pub struct IngestInput { /// When this knowledge stops being valid #[serde(skip_serializing_if = "Option::is_none")] pub valid_until: Option>, + /// Structured provenance for connector-ingested records (#57). When set + /// with a `(source_system, source_id)` key, callers should route through + /// `upsert_by_source` for idempotent sync rather than plain `ingest`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_envelope: Option, } impl Default for IngestInput { @@ -304,6 +404,7 @@ impl Default for IngestInput { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, } } } diff --git a/crates/vestige-core/src/storage/migrations.rs b/crates/vestige-core/src/storage/migrations.rs index c0c60d2..58e5202 100644 --- a/crates/vestige-core/src/storage/migrations.rs +++ b/crates/vestige-core/src/storage/migrations.rs @@ -84,6 +84,11 @@ pub const MIGRATIONS: &[Migration] = &[ description: "ADR 0001 Phase 1: embedding_model registry, domains/domain_scores columns, domains table", up: MIGRATION_V16_UP, }, + Migration { + version: 17, + description: "#57 Source envelope: provenance columns + connector cursor checkpoints for idempotent external-source sync", + up: MIGRATION_V17_UP, + }, ]; /// A database migration @@ -957,6 +962,73 @@ pub const MIGRATION_V16_ALTER_COLUMNS: &[&str] = &[ "ALTER TABLE knowledge_nodes ADD COLUMN domain_scores TEXT NOT NULL DEFAULT '{}'", ]; +/// V17: #57 Source envelope — structured provenance for connector-ingested +/// records, plus a per-connector cursor checkpoint table. +/// +/// The provenance columns live directly on `knowledge_nodes` (rather than a +/// side table) so search can filter and cite them with no join. They are all +/// nullable and default-NULL, so every existing memory is untouched and the +/// migration is purely additive — legacy rows simply have no envelope. +/// +/// The `(source_system, source_id)` pair is the idempotency key for +/// `upsert_by_source`; the unique index enforces one memory per external +/// record. `content_hash` is the change detector. `connector_cursors` holds the +/// incremental-sync high-water mark and last full-reconcile time per +/// (source_system, scope). +/// +/// The `ALTER TABLE ... ADD COLUMN` statements are split into +/// `MIGRATION_V17_ALTER_COLUMNS` and run individually by the migration runner, +/// because SQLite has no `ADD COLUMN IF NOT EXISTS`; duplicate-column errors are +/// swallowed so replay stays idempotent. +const MIGRATION_V17_UP: &str = r#" +-- Idempotency key: at most one memory per (source_system, source_id). +-- Partial unique index so the millions of envelope-less legacy rows (all NULL) +-- don't collide and don't pay index cost. +CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_source_key + ON knowledge_nodes(source_system, source_id) + WHERE source_system IS NOT NULL AND source_id IS NOT NULL; + +-- Filter/scan support for source-aware search + reconciliation passes. +CREATE INDEX IF NOT EXISTS idx_nodes_source_system + ON knowledge_nodes(source_system) + WHERE source_system IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_nodes_source_project + ON knowledge_nodes(source_project) + WHERE source_project IS NOT NULL; + +-- Per-connector incremental-sync checkpoint. One row per (source_system, scope) +-- e.g. ('github', 'samvallad33/vestige'). `cursor_updated_at` is the +-- high-water mark on the source's update timestamp; `last_full_reconcile_at` +-- gates the (expensive) deletion-reconcile pass. +CREATE TABLE IF NOT EXISTS connector_cursors ( + source_system TEXT NOT NULL, + scope TEXT NOT NULL, + cursor_updated_at TEXT, + last_synced_at TEXT, + last_full_reconcile_at TEXT, + records_seen INTEGER NOT NULL DEFAULT 0, + config TEXT NOT NULL DEFAULT '{}', + PRIMARY KEY (source_system, scope) +); + +UPDATE schema_version SET version = 17, applied_at = datetime('now'); +"#; + +/// The `ALTER TABLE` statements for V17. Run individually + idempotently by the +/// migration runner (SQLite has no `ADD COLUMN IF NOT EXISTS`). +pub const MIGRATION_V17_ALTER_COLUMNS: &[&str] = &[ + "ALTER TABLE knowledge_nodes ADD COLUMN source_system TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN source_id TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN source_url TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN source_updated_at TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN content_hash TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN synced_at TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN source_project TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN source_type TEXT", + "ALTER TABLE knowledge_nodes ADD COLUMN source_author TEXT", +]; + /// Apply pending migrations pub fn apply_migrations(conn: &rusqlite::Connection) -> rusqlite::Result { let current_version = get_current_version(conn)?; @@ -994,6 +1066,15 @@ pub fn apply_migrations(conn: &rusqlite::Connection) -> rusqlite::Result { } } + // V17 (#57) adds the source-envelope columns. Same idempotent + // ALTER handling as V16 — the unique index in the V17 batch + // references these columns, so they must exist before the batch. + if migration.version == 17 { + for stmt in MIGRATION_V17_ALTER_COLUMNS { + add_column_if_missing(conn, stmt)?; + } + } + // Use execute_batch to handle multi-statement SQL including triggers conn.execute_batch(migration.up)?; @@ -1026,11 +1107,11 @@ mod tests { // Pre-requisite: schema_version must be bootstrapped by V1. apply_migrations(&conn).expect("apply_migrations succeeds"); - // 1. schema_version advanced to V16 + // 1. schema_version advanced to the latest migration let version = get_current_version(&conn).expect("read schema_version"); assert_eq!( - version, 16, - "schema_version must be 16 after all migrations" + version, 17, + "schema_version must be 17 after all migrations" ); // 2. knowledge_edges is gone (V11 drops it) @@ -1151,11 +1232,11 @@ mod tests { // Replay V11 onward. V11 uses DROP TABLE IF EXISTS so it is idempotent. // V12/V13 tombstone tables use CREATE TABLE IF NOT EXISTS. V14/V16 ALTER // TABLE idempotency is handled by the migration runner. - apply_migrations(&conn).expect("V11..V16 replay must be idempotent"); + apply_migrations(&conn).expect("V11..V17 replay must be idempotent"); // After replaying from V10, the schema advances to the latest version. let version = get_current_version(&conn).expect("read schema_version"); - assert_eq!(version, 16, "schema_version back at 16 after replay"); + assert_eq!(version, 17, "schema_version back at latest after replay"); } #[test] @@ -1229,7 +1310,97 @@ mod tests { // V16 uses CREATE TABLE IF NOT EXISTS and idempotent ALTER handling. apply_migrations(&conn).expect("V16 replay must be idempotent"); let version = get_current_version(&conn).expect("read version"); - assert_eq!(version, 16, "schema_version must be 16 after replay"); + assert_eq!(version, 17, "schema_version must be latest after replay"); + } + + #[test] + fn v17_adds_source_envelope_columns_and_cursor_table() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("apply_migrations"); + + // All nine envelope columns must exist on knowledge_nodes. + let cols: Vec = { + let mut stmt = conn + .prepare("PRAGMA table_info(knowledge_nodes)") + .expect("prepare"); + stmt.query_map([], |row| row.get::<_, String>(1)) + .expect("query_map") + .filter_map(|r| r.ok()) + .collect() + }; + for c in [ + "source_system", + "source_id", + "source_url", + "source_updated_at", + "content_hash", + "synced_at", + "source_project", + "source_type", + "source_author", + ] { + assert!( + cols.iter().any(|x| x == c), + "knowledge_nodes must have `{c}` column after V17" + ); + } + + // connector_cursors table must exist. + let cursor_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='connector_cursors'", + [], + |row| row.get(0), + ) + .expect("query sqlite_master"); + assert_eq!(cursor_rows, 1, "connector_cursors must be created by V17"); + } + + #[test] + fn v17_unique_source_key_index_allows_many_null_legacy_rows() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("apply_migrations"); + + // Two legacy rows with NULL source key must NOT collide on the partial + // unique index (the index only covers non-NULL keys). + for id in ["a", "b"] { + conn.execute( + "INSERT INTO knowledge_nodes (id, content, node_type, created_at, updated_at, last_accessed, \ + stability, difficulty, reps, lapses, learning_state, storage_strength, retrieval_strength, \ + retention_strength, next_review, scheduled_days, has_embedding) \ + VALUES (?1,'c','fact',datetime('now'),datetime('now'),datetime('now'),\ + 1.0,0.3,0,0,'new',1.0,1.0,1.0,datetime('now'),1,0)", + [id], + ) + .expect("insert legacy row"); + } + + // Two real connector rows that share (source_system, source_id) MUST + // collide — the unique index is the idempotency guarantee. + conn.execute( + "UPDATE knowledge_nodes SET source_system='github', source_id='1' WHERE id='a'", + [], + ) + .expect("set source key on a"); + let dup = conn.execute( + "UPDATE knowledge_nodes SET source_system='github', source_id='1' WHERE id='b'", + [], + ); + assert!( + dup.is_err(), + "duplicate (source_system, source_id) must violate the unique index" + ); + } + + #[test] + fn v17_is_replayable() { + let conn = rusqlite::Connection::open_in_memory().expect("open in-memory"); + apply_migrations(&conn).expect("first apply"); + conn.execute("UPDATE schema_version SET version = 16", []) + .expect("rewind to 16"); + apply_migrations(&conn).expect("V17 replay must be idempotent"); + let version = get_current_version(&conn).expect("read version"); + assert_eq!(version, 17, "schema_version must be 17 after replay"); } #[test] diff --git a/crates/vestige-core/src/storage/mod.rs b/crates/vestige-core/src/storage/mod.rs index 5f0a54c..c5db4e2 100644 --- a/crates/vestige-core/src/storage/mod.rs +++ b/crates/vestige-core/src/storage/mod.rs @@ -19,9 +19,10 @@ pub use portable::{ }; pub use sqlite::{ CompositionEventRecord, CompositionMemberRecord, CompositionNeighborRecord, - CompositionOutcomeRecord, ConnectionRecord, ConsolidationHistoryRecord, DreamHistoryRecord, - FilePortableSyncBackend, InsightRecord, IntentionRecord, NeverComposedCandidate, - PortableSyncBackend, PortableSyncReport, Result, SmartIngestResult, SqliteMemoryStore, + CompositionOutcomeRecord, ConnectionRecord, ConnectorCursor, ConsolidationHistoryRecord, + DreamHistoryRecord, FilePortableSyncBackend, InsightRecord, IntentionRecord, + NeverComposedCandidate, PortableSyncBackend, PortableSyncReport, ReconcileReport, Result, + SmartIngestResult, SourceUpsertOutcome, SourceUpsertResult, SqliteMemoryStore, StateTransitionRecord, StorageError, }; diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 9685f61..399b2fb 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -682,6 +682,12 @@ impl SqliteMemoryStore { let valid_from_str = input.valid_from.map(|dt| dt.to_rfc3339()); let valid_until_str = input.valid_until.map(|dt| dt.to_rfc3339()); + // #57 Source envelope — flatten to nullable column values. A node with + // no external provenance leaves all nine columns NULL (legacy shape). + let env = input.source_envelope.clone().unwrap_or_default(); + let env_source_updated_at = env.source_updated_at.map(|dt| dt.to_rfc3339()); + let env_synced_at = env.synced_at.map(|dt| dt.to_rfc3339()); + { let writer = self .writer @@ -694,14 +700,18 @@ impl SqliteMemoryStore { storage_strength, retrieval_strength, retention_strength, sentiment_score, sentiment_magnitude, next_review, scheduled_days, source, tags, valid_from, valid_until, has_embedding, embedding_model, - domains, domain_scores + domains, domain_scores, + source_system, source_id, source_url, source_updated_at, + content_hash, synced_at, source_project, source_type, source_author ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, - '[]', '{}' + '[]', '{}', + ?25, ?26, ?27, ?28, + ?29, ?30, ?31, ?32, ?33 )", params![ id, @@ -728,6 +738,15 @@ impl SqliteMemoryStore { valid_until_str, 0, Option::::None, + env.source_system, + env.source_id, + env.source_url, + env_source_updated_at, + env.content_hash, + env_synced_at, + env.source_project, + env.source_type, + env.source_author, ], )?; } @@ -1257,6 +1276,33 @@ impl SqliteMemoryStore { .ok() }); + // #57 Source envelope columns (Migration V17). `.ok().flatten()` is + // tolerant of pre-V17 databases that lack these columns. Collapse an + // all-NULL envelope to `None` so legacy nodes serialize unchanged. + let parse_ts = |s: Option| -> Option> { + s.and_then(|s| { + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + .ok() + }) + }; + let envelope = crate::memory::SourceEnvelope { + source_system: row.get("source_system").ok().flatten(), + source_id: row.get("source_id").ok().flatten(), + source_url: row.get("source_url").ok().flatten(), + source_updated_at: parse_ts(row.get("source_updated_at").ok().flatten()), + content_hash: row.get("content_hash").ok().flatten(), + synced_at: parse_ts(row.get("synced_at").ok().flatten()), + source_project: row.get("source_project").ok().flatten(), + source_type: row.get("source_type").ok().flatten(), + source_author: row.get("source_author").ok().flatten(), + }; + let source_envelope = if envelope.is_empty() { + None + } else { + Some(envelope) + }; + Ok(KnowledgeNode { id: row.get("id")?, content: row.get("content")?, @@ -1293,6 +1339,8 @@ impl SqliteMemoryStore { // v2.0.5 Active Forgetting suppression_count, suppressed_at, + // #57 Source envelope + source_envelope, }) } @@ -9415,6 +9463,336 @@ impl crate::storage::memory_store::MemoryStoreSend for SqliteMemoryStore { } } +// ============================================================================ +// CONNECTOR SYNC (#57) — idempotent external-source ingestion +// ============================================================================ + +/// What `upsert_by_source` did with one external record. Drives the +/// created/updated/unchanged/tombstoned counts a connector reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceUpsertOutcome { + /// No memory existed for this `(source_system, source_id)` — inserted. + Created, + /// A memory existed and the `content_hash` changed — body + envelope updated + /// and the embedding regenerated. + Updated, + /// A memory existed with the same `content_hash` — nothing rewritten except + /// `synced_at` (so an incremental re-scan is free). + Unchanged, +} + +/// Result of one `upsert_by_source` call. +#[derive(Debug, Clone)] +pub struct SourceUpsertResult { + pub outcome: SourceUpsertOutcome, + /// Memory id of the affected node (new or existing). + pub node_id: String, +} + +/// Incremental-sync checkpoint for one `(source_system, scope)`. +#[derive(Debug, Clone, Default)] +pub struct ConnectorCursor { + pub source_system: String, + pub scope: String, + /// High-water mark on the source's update timestamp. `None` on first sync. + pub cursor_updated_at: Option>, + pub last_synced_at: Option>, + pub last_full_reconcile_at: Option>, + pub records_seen: i64, +} + +/// Outcome of a tombstone reconciliation pass. +#[derive(Debug, Clone, Default)] +pub struct ReconcileReport { + /// Memory ids that were tombstoned (no longer visible upstream). + pub tombstoned: Vec, + /// Number of local records considered for this scope. + pub considered: usize, +} + +impl SqliteMemoryStore { + /// Idempotently upsert one external-source record, keyed on the envelope's + /// `(source_system, source_id)` (#57). + /// + /// This is the core primitive every connector calls per record. It makes + /// re-running a sync safe and cheap: + /// + /// - **No existing memory** for the key → insert (`Created`). + /// - **Existing memory, `content_hash` changed** → update content + envelope, + /// stamp `updated_at`, regenerate the embedding (`Updated`). + /// - **Existing memory, `content_hash` unchanged** → touch only `synced_at` + /// so the reconcile pass knows the record is still live (`Unchanged`). + /// + /// The caller MUST set `source_system`, `source_id`, and `content_hash` on + /// the input's `source_envelope`; otherwise this falls back to a plain + /// `ingest` (an un-keyed record can't be deduplicated). + pub fn upsert_by_source(&self, input: IngestInput) -> Result { + let env = match input.source_envelope.clone() { + Some(e) if e.has_key() => e, + // No idempotency key — behave like a normal create. + _ => { + let node = self.ingest(input)?; + return Ok(SourceUpsertResult { + outcome: SourceUpsertOutcome::Created, + node_id: node.id, + }); + } + }; + + let source_system = env.source_system.clone().unwrap_or_default(); + let source_id = env.source_id.clone().unwrap_or_default(); + let now = Utc::now(); + + // Look up the existing memory for this external record, if any. + let existing: Option<(String, Option)> = { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + reader + .query_row( + "SELECT id, content_hash FROM knowledge_nodes \ + WHERE source_system = ?1 AND source_id = ?2 LIMIT 1", + params![source_system, source_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + ) + .optional()? + }; + + let Some((node_id, stored_hash)) = existing else { + // First time we've seen this record — plain insert carries the + // envelope through the existing ingest path. + let node = self.ingest(input)?; + return Ok(SourceUpsertResult { + outcome: SourceUpsertOutcome::Created, + node_id: node.id, + }); + }; + + let new_hash = env.content_hash.clone(); + let unchanged = match (&stored_hash, &new_hash) { + // Both present and equal → genuinely unchanged. + (Some(a), Some(b)) => a == b, + // Either side missing a hash → be conservative and treat as changed + // so we never silently skip a real update. + _ => false, + }; + + let env_source_updated_at = env.source_updated_at.map(|dt| dt.to_rfc3339()); + let synced_at = now.to_rfc3339(); + + if unchanged { + // Cheapest path: only advance liveness + the source cursor field. + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + // Un-tombstone fully: a reappearing record clears BOTH bitemporal + // markers (valid_until AND superseded_by), otherwise it would be + // resurrected as currently-valid yet still flagged as superseded, + // which permanently excludes it from merge/consolidation. + "UPDATE knowledge_nodes \ + SET synced_at = ?1, source_updated_at = COALESCE(?2, source_updated_at), \ + source_url = COALESCE(?3, source_url), \ + valid_until = NULL, superseded_by = NULL \ + WHERE id = ?4", + params![synced_at, env_source_updated_at, env.source_url, node_id], + )?; + return Ok(SourceUpsertResult { + outcome: SourceUpsertOutcome::Unchanged, + node_id, + }); + } + + // Content changed upstream → update body + full envelope, clear any + // prior tombstone (`valid_until`), then regenerate the embedding. + { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + // Clear BOTH bitemporal markers on update (see Unchanged branch). + "UPDATE knowledge_nodes SET \ + content = ?1, updated_at = ?2, synced_at = ?3, \ + content_hash = ?4, source_url = ?5, source_updated_at = ?6, \ + source_project = ?7, source_type = ?8, source_author = ?9, \ + valid_until = NULL, superseded_by = NULL \ + WHERE id = ?10", + params![ + input.content, + now.to_rfc3339(), + synced_at, + env.content_hash, + env.source_url, + env_source_updated_at, + env.source_project, + env.source_type, + env.source_author, + node_id, + ], + )?; + } + + #[cfg(all(feature = "embeddings", feature = "vector-search"))] + { + if let Some(index) = self.vector_index.as_ref() + && let Ok(mut index) = index.lock() + { + let _ = index.remove(&node_id); + } + if let Err(e) = self.generate_embedding_for_node(&node_id, &input.content) { + tracing::warn!("Failed to regenerate embedding for {}: {}", node_id, e); + } + } + + Ok(SourceUpsertResult { + outcome: SourceUpsertOutcome::Updated, + node_id, + }) + } + + /// Read the incremental-sync checkpoint for a `(source_system, scope)`. + /// Returns a zeroed cursor (no high-water mark) if none has been saved yet. + pub fn get_connector_cursor(&self, source_system: &str, scope: &str) -> Result { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let row = reader + .query_row( + "SELECT cursor_updated_at, last_synced_at, last_full_reconcile_at, records_seen \ + FROM connector_cursors WHERE source_system = ?1 AND scope = ?2", + params![source_system, scope], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, i64>(3)?, + )) + }, + ) + .optional()?; + + let parse = |s: Option| -> Option> { + s.and_then(|s| { + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + .ok() + }) + }; + + Ok(match row { + Some((cur, last, recon, seen)) => ConnectorCursor { + source_system: source_system.to_string(), + scope: scope.to_string(), + cursor_updated_at: parse(cur), + last_synced_at: parse(last), + last_full_reconcile_at: parse(recon), + records_seen: seen, + }, + None => ConnectorCursor { + source_system: source_system.to_string(), + scope: scope.to_string(), + ..Default::default() + }, + }) + } + + /// Persist the incremental-sync checkpoint for a `(source_system, scope)`. + pub fn save_connector_cursor(&self, cursor: &ConnectorCursor) -> Result<()> { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + writer.execute( + "INSERT INTO connector_cursors \ + (source_system, scope, cursor_updated_at, last_synced_at, \ + last_full_reconcile_at, records_seen) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6) \ + ON CONFLICT(source_system, scope) DO UPDATE SET \ + cursor_updated_at = excluded.cursor_updated_at, \ + last_synced_at = excluded.last_synced_at, \ + last_full_reconcile_at = excluded.last_full_reconcile_at, \ + records_seen = excluded.records_seen", + params![ + cursor.source_system, + cursor.scope, + cursor.cursor_updated_at.map(|d| d.to_rfc3339()), + cursor.last_synced_at.map(|d| d.to_rfc3339()), + cursor.last_full_reconcile_at.map(|d| d.to_rfc3339()), + cursor.records_seen, + ], + )?; + Ok(()) + } + + /// Reconcile deletions for a scope: tombstone every local memory in + /// `(source_system, source_project = scope)` whose `source_id` is NOT in the + /// caller-supplied set of currently-live ids (#57). + /// + /// Neither Redmine nor GitHub exposes a deletion feed, so an incremental + /// `updated_at` sync can never see a delete. The connector therefore + /// periodically enumerates the full set of live ids and calls this. We + /// **invalidate, don't purge** (Graphiti-style): the memory keeps its + /// content for audit but gets `valid_until = now`, so it falls out of + /// "currently valid" retrieval without losing history. A record that + /// reappears upstream is un-tombstoned by the next `upsert_by_source` + /// (which clears `valid_until`). + pub fn reconcile_source_tombstones( + &self, + source_system: &str, + scope: &str, + live_ids: &[String], + ) -> Result { + let live: std::collections::HashSet<&str> = live_ids.iter().map(|s| s.as_str()).collect(); + + // All currently-valid local records for this scope. + let local: Vec<(String, String)> = { + let reader = self + .reader + .lock() + .map_err(|_| StorageError::Init("Reader lock poisoned".into()))?; + let mut stmt = reader.prepare( + "SELECT id, source_id FROM knowledge_nodes \ + WHERE source_system = ?1 AND source_project = ?2 \ + AND source_id IS NOT NULL AND valid_until IS NULL", + )?; + let rows = stmt.query_map(params![source_system, scope], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + rows.filter_map(|r| r.ok()).collect() + }; + + let considered = local.len(); + let now = Utc::now().to_rfc3339(); + let mut tombstoned = Vec::new(); + + { + let writer = self + .writer + .lock() + .map_err(|_| StorageError::Init("Writer lock poisoned".into()))?; + for (node_id, source_id) in &local { + if !live.contains(source_id.as_str()) { + writer.execute( + "UPDATE knowledge_nodes SET valid_until = ?1 WHERE id = ?2", + params![now, node_id], + )?; + tombstoned.push(node_id.clone()); + } + } + } + + Ok(ReconcileReport { + tombstoned, + considered, + }) + } +} + // ============================================================================ // TESTS // ============================================================================ @@ -9443,6 +9821,234 @@ mod tests { Storage::new(Some(dir.path().join(name))).unwrap() } + // ===================== Connector sync (#57) ========================= + + /// Build an `IngestInput` carrying a source envelope for a GitHub-ish issue. + fn source_input(id: &str, content: &str, hash: &str) -> IngestInput { + IngestInput { + content: content.to_string(), + node_type: "fact".to_string(), + source_envelope: Some(crate::memory::SourceEnvelope { + source_system: Some("github".to_string()), + source_id: Some(id.to_string()), + source_url: Some(format!("https://github.com/o/r/issues/{id}")), + content_hash: Some(hash.to_string()), + source_project: Some("o/r".to_string()), + source_type: Some("issue".to_string()), + source_author: Some("octocat".to_string()), + ..Default::default() + }), + ..Default::default() + } + } + + fn node_count(store: &Storage) -> i64 { + // Count rows for our test source so embeddings/other tests don't bleed in. + let reader = store.reader.lock().unwrap(); + reader + .query_row( + "SELECT COUNT(*) FROM knowledge_nodes WHERE source_system = 'github'", + [], + |r| r.get(0), + ) + .unwrap() + } + + #[test] + fn upsert_by_source_is_idempotent_across_reruns() { + let store = create_test_storage(); + + // First sync: a brand-new record → Created. + let r1 = store + .upsert_by_source(source_input("1", "Bug: crash on startup", "hash-a")) + .unwrap(); + assert_eq!(r1.outcome, SourceUpsertOutcome::Created); + assert_eq!(node_count(&store), 1); + + // Re-sync the SAME record with the SAME hash twice → Unchanged, no dupes. + for _ in 0..2 { + let r = store + .upsert_by_source(source_input("1", "Bug: crash on startup", "hash-a")) + .unwrap(); + assert_eq!(r.outcome, SourceUpsertOutcome::Unchanged); + assert_eq!(r.node_id, r1.node_id, "must reuse the same memory id"); + } + assert_eq!(node_count(&store), 1, "idempotent: still exactly one memory"); + } + + #[test] + fn upsert_by_source_updates_in_place_when_hash_changes() { + let store = create_test_storage(); + let created = store + .upsert_by_source(source_input("7", "old body", "hash-old")) + .unwrap(); + + // Upstream edit: content + hash change → Updated, same id, new content. + let updated = store + .upsert_by_source(source_input("7", "new edited body", "hash-new")) + .unwrap(); + assert_eq!(updated.outcome, SourceUpsertOutcome::Updated); + assert_eq!(updated.node_id, created.node_id); + assert_eq!(node_count(&store), 1, "update must not duplicate"); + + let node = store.get_node(&created.node_id).unwrap().unwrap(); + assert_eq!(node.content, "new edited body"); + let env = node.source_envelope.expect("envelope persisted"); + assert_eq!(env.content_hash.as_deref(), Some("hash-new")); + assert_eq!(env.source_id.as_deref(), Some("7")); + } + + #[test] + fn upsert_by_source_without_key_falls_back_to_create() { + let store = create_test_storage(); + // Envelope present but missing source_id → not keyed → plain create. + let input = IngestInput { + content: "loose note".to_string(), + node_type: "fact".to_string(), + source_envelope: Some(crate::memory::SourceEnvelope { + source_url: Some("https://example.com/x".to_string()), + ..Default::default() + }), + ..Default::default() + }; + let r = store.upsert_by_source(input).unwrap(); + assert_eq!(r.outcome, SourceUpsertOutcome::Created); + } + + #[test] + fn connector_cursor_round_trips() { + let store = create_test_storage(); + // Unknown scope → zeroed cursor. + let empty = store.get_connector_cursor("github", "o/r").unwrap(); + assert!(empty.cursor_updated_at.is_none()); + assert_eq!(empty.records_seen, 0); + + let ts = Utc::now(); + let cursor = ConnectorCursor { + source_system: "github".to_string(), + scope: "o/r".to_string(), + cursor_updated_at: Some(ts), + last_synced_at: Some(ts), + last_full_reconcile_at: None, + records_seen: 42, + }; + store.save_connector_cursor(&cursor).unwrap(); + + let back = store.get_connector_cursor("github", "o/r").unwrap(); + assert_eq!(back.records_seen, 42); + assert_eq!( + back.cursor_updated_at.map(|d| d.to_rfc3339()), + Some(ts.to_rfc3339()) + ); + + // Upsert semantics: saving again replaces, never duplicates. + let mut c2 = cursor.clone(); + c2.records_seen = 99; + store.save_connector_cursor(&c2).unwrap(); + assert_eq!(store.get_connector_cursor("github", "o/r").unwrap().records_seen, 99); + } + + #[test] + fn reconcile_tombstones_records_absent_from_live_set() { + let store = create_test_storage(); + // Three synced issues in scope o/r. + for id in ["1", "2", "3"] { + store + .upsert_by_source(source_input(id, &format!("issue {id}"), &format!("h{id}"))) + .unwrap(); + } + + // Reconcile: only 1 and 3 are still visible upstream → 2 is tombstoned. + let report = store + .reconcile_source_tombstones("github", "o/r", &["1".to_string(), "3".to_string()]) + .unwrap(); + assert_eq!(report.considered, 3); + assert_eq!(report.tombstoned.len(), 1, "exactly issue 2 tombstoned"); + + // Issue 2's memory is invalidated (valid_until set) but NOT purged — + // content retained for audit, just no longer currently-valid. + let two = { + let reader = store.reader.lock().unwrap(); + reader + .query_row( + "SELECT id, valid_until FROM knowledge_nodes WHERE source_id = '2'", + [], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)), + ) + .unwrap() + }; + assert!(two.1.is_some(), "tombstoned record must have valid_until set"); + let node = store.get_node(&two.0).unwrap().unwrap(); + assert!(!node.is_currently_valid(), "tombstoned node is not valid now"); + assert_eq!(node.content, "issue 2", "content retained for audit"); + + // A reappearing record un-tombstones on next upsert (clears valid_until). + store + .upsert_by_source(source_input("2", "issue 2", "h2")) + .unwrap(); + let revived = store.get_node(&two.0).unwrap().unwrap(); + assert!(revived.is_currently_valid(), "re-synced record is valid again"); + } + + #[test] + fn upsert_clears_superseded_by_when_record_reappears() { + // Regression: un-tombstoning must clear BOTH bitemporal markers. A + // connector node that was superseded/merged (valid_until + superseded_by + // both set) and then re-observed upstream must come back fully clean, + // otherwise it is currently-valid yet still flagged superseded and is + // permanently excluded from merge candidacy. + let store = create_test_storage(); + let created = store + .upsert_by_source(source_input("9", "body v1", "h9a")) + .unwrap(); + + // Simulate the node having been superseded (as merge/supersede would). + { + let writer = store.writer.lock().unwrap(); + writer + .execute( + "UPDATE knowledge_nodes SET valid_until = ?1, superseded_by = 'survivor-id' WHERE id = ?2", + params![Utc::now().to_rfc3339(), created.node_id], + ) + .unwrap(); + } + assert!( + store.superseded_node_ids().unwrap().contains(&created.node_id), + "precondition: node is superseded" + ); + + // Re-sync with a content change → Updated branch must clear both markers. + let res = store + .upsert_by_source(source_input("9", "body v2 edited", "h9b")) + .unwrap(); + assert_eq!(res.outcome, SourceUpsertOutcome::Updated); + assert!( + !store.superseded_node_ids().unwrap().contains(&created.node_id), + "superseded_by must be cleared on re-sync (no bitemporal zombie)" + ); + let node = store.get_node(&created.node_id).unwrap().unwrap(); + assert!(node.is_currently_valid()); + + // Also exercise the Unchanged branch: supersede again, re-sync same hash. + { + let writer = store.writer.lock().unwrap(); + writer + .execute( + "UPDATE knowledge_nodes SET valid_until = ?1, superseded_by = 'survivor-id' WHERE id = ?2", + params![Utc::now().to_rfc3339(), created.node_id], + ) + .unwrap(); + } + let res2 = store + .upsert_by_source(source_input("9", "body v2 edited", "h9b")) + .unwrap(); + assert_eq!(res2.outcome, SourceUpsertOutcome::Unchanged); + assert!( + !store.superseded_node_ids().unwrap().contains(&created.node_id), + "Unchanged branch must also clear superseded_by" + ); + } + #[cfg(all(feature = "embeddings", feature = "vector-search"))] fn with_vector_search_disabled(f: impl FnOnce() -> T) -> T { let _guard = ENV_LOCK.lock().unwrap(); diff --git a/crates/vestige-mcp/Cargo.toml b/crates/vestige-mcp/Cargo.toml index bc08a40..fb2a287 100644 --- a/crates/vestige-mcp/Cargo.toml +++ b/crates/vestige-mcp/Cargo.toml @@ -10,9 +10,13 @@ categories = ["command-line-utilities", "database"] repository = "https://github.com/samvallad33/vestige" [features] -default = ["embeddings", "ort-download", "vector-search"] +default = ["embeddings", "ort-download", "vector-search", "connectors"] embeddings = ["vestige-core/embeddings"] vector-search = ["vestige-core/vector-search"] +# External-source connectors (#57): GitHub Issues / Redmine indexing via the +# `source_sync` MCP tool. On by default so the tool works out of the box; turn +# off for a build with no HTTP client. +connectors = ["vestige-core/connectors"] # Default ort backend: downloads prebuilt ONNX Runtime at build time. # Fails on targets without prebuilts (notably x86_64-apple-darwin). ort-download = ["embeddings", "vestige-core/ort-download"] diff --git a/crates/vestige-mcp/src/bin/cli.rs b/crates/vestige-mcp/src/bin/cli.rs index 3daa8e3..b79dd55 100644 --- a/crates/vestige-mcp/src/bin/cli.rs +++ b/crates/vestige-mcp/src/bin/cli.rs @@ -1817,6 +1817,7 @@ fn run_restore(backup_path: PathBuf) -> anyhow::Result<()> { tags: memory.tags.unwrap_or_default(), valid_from: None, valid_until: None, + source_envelope: None, }; match storage.ingest(input) { @@ -2415,6 +2416,7 @@ fn run_ingest( tags: tag_list, valid_from: None, valid_until: None, + source_envelope: None, }; let storage = open_storage()?; diff --git a/crates/vestige-mcp/src/bin/restore.rs b/crates/vestige-mcp/src/bin/restore.rs index 332c329..ace6f3f 100644 --- a/crates/vestige-mcp/src/bin/restore.rs +++ b/crates/vestige-mcp/src/bin/restore.rs @@ -73,6 +73,7 @@ fn main() -> anyhow::Result<()> { tags: memory.tags.unwrap_or_default(), valid_from: None, valid_until: None, + source_envelope: None, }; match storage.ingest(input) { diff --git a/crates/vestige-mcp/src/cognitive.rs b/crates/vestige-mcp/src/cognitive.rs index 3d106ff..fc73f97 100644 --- a/crates/vestige-mcp/src/cognitive.rs +++ b/crates/vestige-mcp/src/cognitive.rs @@ -195,6 +195,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); result.id diff --git a/crates/vestige-mcp/src/server.rs b/crates/vestige-mcp/src/server.rs index 7682441..b8f90e7 100644 --- a/crates/vestige-mcp/src/server.rs +++ b/crates/vestige-mcp/src/server.rs @@ -281,6 +281,15 @@ impl McpServer { ..Default::default() }, // ================================================================ + // EXTERNAL-SOURCE CONNECTORS (#57) + // ================================================================ + ToolDescription { + name: "source_sync".to_string(), + description: Some("Index an external system (GitHub Issues) into Vestige as a durable, offline, semantically-searchable index that cites back to the canonical record. Provide 'repo' as 'owner/name'. Idempotent: re-running updates changed issues without duplicating; set reconcile=true to tombstone issues removed upstream. Auth via the GITHUB_TOKEN env var (optional for public repos).".to_string()), + input_schema: tools::source_sync::schema(), + ..Default::default() + }, + // ================================================================ // TEMPORAL TOOLS (v1.2+) // ================================================================ ToolDescription { @@ -593,6 +602,11 @@ impl McpServer { .await } + // ================================================================ + // External-source connectors (#57) + // ================================================================ + "source_sync" => tools::source_sync::execute(&self.storage, request.arguments).await, + // ================================================================ // DEPRECATED (v1.7): ingest → smart_ingest // ================================================================ @@ -1806,10 +1820,10 @@ mod tests { let result = response.result.unwrap(); let tools = result["tools"].as_array().unwrap(); - // 33 tools: 25 from v2.1.21 + 7 Phase 3 merge/supersede tools: - // merge_candidates, plan_merge, plan_supersede, apply_plan, merge_undo, - // protect, merge_policy, composed_graph) - assert_eq!(tools.len(), 33, "Expected exactly 33 tools"); + // 34 tools: 25 from v2.1.21 + 7 Phase 3 merge/supersede tools + // (merge_candidates, plan_merge, plan_supersede, apply_plan, merge_undo, + // protect, merge_policy, composed_graph) + 1 connector tool (source_sync, #57). + assert_eq!(tools.len(), 34, "Expected exactly 34 tools"); let tool_names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); @@ -1821,6 +1835,9 @@ mod tests { // Core memory (smart_ingest absorbs ingest + checkpoint in v1.7) assert!(tool_names.contains(&"smart_ingest")); + + // External-source connectors (#57) + assert!(tool_names.contains(&"source_sync")); assert!( !tool_names.contains(&"ingest"), "ingest should be removed in v1.7" diff --git a/crates/vestige-mcp/src/tools/changelog.rs b/crates/vestige-mcp/src/tools/changelog.rs index 8a2e746..eb47139 100644 --- a/crates/vestige-mcp/src/tools/changelog.rs +++ b/crates/vestige-mcp/src/tools/changelog.rs @@ -278,6 +278,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); node.id diff --git a/crates/vestige-mcp/src/tools/codebase_unified.rs b/crates/vestige-mcp/src/tools/codebase_unified.rs index e24021d..50491cd 100644 --- a/crates/vestige-mcp/src/tools/codebase_unified.rs +++ b/crates/vestige-mcp/src/tools/codebase_unified.rs @@ -154,6 +154,7 @@ async fn execute_remember_pattern( tags, valid_from: None, valid_until: None, + source_envelope: None, }; let node = storage.ingest(input).map_err(|e| e.to_string())?; @@ -250,6 +251,7 @@ async fn execute_remember_decision( tags, valid_from: None, valid_until: None, + source_envelope: None, }; let node = storage.ingest(input).map_err(|e| e.to_string())?; diff --git a/crates/vestige-mcp/src/tools/cross_reference.rs b/crates/vestige-mcp/src/tools/cross_reference.rs index e48b4eb..0e231b1 100644 --- a/crates/vestige-mcp/src/tools/cross_reference.rs +++ b/crates/vestige-mcp/src/tools/cross_reference.rs @@ -1119,6 +1119,7 @@ mod tests { tags: tags.iter().map(|s| s.to_string()).collect(), valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap() .id diff --git a/crates/vestige-mcp/src/tools/dream.rs b/crates/vestige-mcp/src/tools/dream.rs index 65a373b..1456406 100644 --- a/crates/vestige-mcp/src/tools/dream.rs +++ b/crates/vestige-mcp/src/tools/dream.rs @@ -283,6 +283,7 @@ mod tests { tags: vec!["dream-test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -420,6 +421,7 @@ mod tests { tags: vec!["dream-roundtrip".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -485,6 +487,7 @@ mod tests { tags: vec!["save-conn-test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); ids.push(result.id); @@ -588,6 +591,7 @@ mod tests { tags: tags.iter().map(|t| t.to_string()).collect(), valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -713,6 +717,7 @@ mod tests { tags: tags.iter().map(|t| t.to_string()).collect(), valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } diff --git a/crates/vestige-mcp/src/tools/explore.rs b/crates/vestige-mcp/src/tools/explore.rs index 441afd3..cabc7c9 100644 --- a/crates/vestige-mcp/src/tools/explore.rs +++ b/crates/vestige-mcp/src/tools/explore.rs @@ -414,6 +414,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap() .id; @@ -428,6 +429,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap() .id; @@ -478,6 +480,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }; let id_a = storage.ingest(make("Memory A about databases")).unwrap().id; let id_b = storage.ingest(make("Memory B about indexes")).unwrap().id; @@ -529,6 +532,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }; let id_a = storage.ingest(make("Bridge test memory A")).unwrap().id; let id_b = storage.ingest(make("Bridge test memory B")).unwrap().id; diff --git a/crates/vestige-mcp/src/tools/feedback.rs b/crates/vestige-mcp/src/tools/feedback.rs index 438e594..a236bf2 100644 --- a/crates/vestige-mcp/src/tools/feedback.rs +++ b/crates/vestige-mcp/src/tools/feedback.rs @@ -313,6 +313,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); node.id @@ -556,6 +557,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); let node_id = node.id.clone(); diff --git a/crates/vestige-mcp/src/tools/graph.rs b/crates/vestige-mcp/src/tools/graph.rs index 13ca746..904e5e4 100644 --- a/crates/vestige-mcp/src/tools/graph.rs +++ b/crates/vestige-mcp/src/tools/graph.rs @@ -328,6 +328,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); @@ -355,6 +356,7 @@ mod tests { tags: vec!["science".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); @@ -378,6 +380,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); diff --git a/crates/vestige-mcp/src/tools/health.rs b/crates/vestige-mcp/src/tools/health.rs index 362f298..563abb8 100644 --- a/crates/vestige-mcp/src/tools/health.rs +++ b/crates/vestige-mcp/src/tools/health.rs @@ -119,6 +119,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -144,6 +145,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); diff --git a/crates/vestige-mcp/src/tools/maintenance.rs b/crates/vestige-mcp/src/tools/maintenance.rs index ef1a927..814f513 100644 --- a/crates/vestige-mcp/src/tools/maintenance.rs +++ b/crates/vestige-mcp/src/tools/maintenance.rs @@ -778,6 +778,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -832,6 +833,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -904,6 +906,7 @@ mod tests { tags: vec!["schema-test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); @@ -1015,6 +1018,7 @@ mod tests { tags: vec!["portable".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); diff --git a/crates/vestige-mcp/src/tools/memory_unified.rs b/crates/vestige-mcp/src/tools/memory_unified.rs index 8a9ddcf..1b32262 100644 --- a/crates/vestige-mcp/src/tools/memory_unified.rs +++ b/crates/vestige-mcp/src/tools/memory_unified.rs @@ -552,6 +552,7 @@ mod tests { tags: vec!["test-tag".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); node.id diff --git a/crates/vestige-mcp/src/tools/mod.rs b/crates/vestige-mcp/src/tools/mod.rs index 078fab6..f145caf 100644 --- a/crates/vestige-mcp/src/tools/mod.rs +++ b/crates/vestige-mcp/src/tools/mod.rs @@ -12,6 +12,8 @@ pub mod intention_unified; pub mod memory_unified; pub mod search_unified; pub mod smart_ingest; +// #57: external-source connectors (GitHub Issues / Redmine retrieval layer) +pub mod source_sync; // v1.2: Temporal query tools pub mod changelog; diff --git a/crates/vestige-mcp/src/tools/restore.rs b/crates/vestige-mcp/src/tools/restore.rs index ac7184c..f000397 100644 --- a/crates/vestige-mcp/src/tools/restore.rs +++ b/crates/vestige-mcp/src/tools/restore.rs @@ -173,6 +173,7 @@ pub async fn execute(storage: &Arc, args: Option) -> Result Value { + let Some(env) = node.source_envelope.as_ref() else { + return Value::Null; + }; + serde_json::json!({ + "system": env.source_system, + "id": env.source_id, + "url": env.source_url, + "project": env.source_project, + "type": env.source_type, + "author": env.source_author, + "sourceUpdatedAt": env.source_updated_at.map(|dt| dt.to_rfc3339()), + "syncedAt": env.synced_at.map(|dt| dt.to_rfc3339()), + // A tombstoned (no-longer-visible) record has valid_until set in the past. + "tombstoned": !node.is_currently_valid(), + }) +} + fn format_search_result(r: &vestige_core::SearchResult, detail_level: &str) -> Value { match detail_level { "brief" => serde_json::json!({ @@ -898,45 +923,65 @@ fn format_search_result(r: &vestige_core::SearchResult, detail_level: &str) -> V "retentionStrength": r.node.retention_strength, "combinedScore": r.combined_score, }), - "full" => serde_json::json!({ - "id": r.node.id, - "content": r.node.content, - "combinedScore": r.combined_score, - "keywordScore": r.keyword_score, - "semanticScore": r.semantic_score, - "nodeType": r.node.node_type, - "tags": r.node.tags, - "retentionStrength": r.node.retention_strength, - "storageStrength": r.node.storage_strength, - "retrievalStrength": r.node.retrieval_strength, - "source": r.node.source, - "sentimentScore": r.node.sentiment_score, - "sentimentMagnitude": r.node.sentiment_magnitude, - "createdAt": r.node.created_at.to_rfc3339(), - "updatedAt": r.node.updated_at.to_rfc3339(), - "lastAccessed": r.node.last_accessed.to_rfc3339(), - "nextReview": r.node.next_review.map(|dt| dt.to_rfc3339()), - "stability": r.node.stability, - "difficulty": r.node.difficulty, - "reps": r.node.reps, - "lapses": r.node.lapses, - "validFrom": r.node.valid_from.map(|dt| dt.to_rfc3339()), - "validUntil": r.node.valid_until.map(|dt| dt.to_rfc3339()), - "matchType": format!("{:?}", r.match_type), - }), + "full" => { + let mut v = serde_json::json!({ + "id": r.node.id, + "content": r.node.content, + "combinedScore": r.combined_score, + "keywordScore": r.keyword_score, + "semanticScore": r.semantic_score, + "nodeType": r.node.node_type, + "tags": r.node.tags, + "retentionStrength": r.node.retention_strength, + "storageStrength": r.node.storage_strength, + "retrievalStrength": r.node.retrieval_strength, + "source": r.node.source, + "sentimentScore": r.node.sentiment_score, + "sentimentMagnitude": r.node.sentiment_magnitude, + "createdAt": r.node.created_at.to_rfc3339(), + "updatedAt": r.node.updated_at.to_rfc3339(), + "lastAccessed": r.node.last_accessed.to_rfc3339(), + "nextReview": r.node.next_review.map(|dt| dt.to_rfc3339()), + "stability": r.node.stability, + "difficulty": r.node.difficulty, + "reps": r.node.reps, + "lapses": r.node.lapses, + "validFrom": r.node.valid_from.map(|dt| dt.to_rfc3339()), + "validUntil": r.node.valid_until.map(|dt| dt.to_rfc3339()), + "matchType": format!("{:?}", r.match_type), + }); + attach_source_record(&mut v, &r.node); + v + } // "summary" (default) — includes dates so AI never has to guess when a memory is from - _ => serde_json::json!({ - "id": r.node.id, - "content": r.node.content, - "combinedScore": r.combined_score, - "keywordScore": r.keyword_score, - "semanticScore": r.semantic_score, - "nodeType": r.node.node_type, - "tags": r.node.tags, - "retentionStrength": r.node.retention_strength, - "createdAt": r.node.created_at.to_rfc3339(), - "updatedAt": r.node.updated_at.to_rfc3339(), - }), + _ => { + let mut v = serde_json::json!({ + "id": r.node.id, + "content": r.node.content, + "combinedScore": r.combined_score, + "keywordScore": r.keyword_score, + "semanticScore": r.semantic_score, + "nodeType": r.node.node_type, + "tags": r.node.tags, + "retentionStrength": r.node.retention_strength, + "createdAt": r.node.created_at.to_rfc3339(), + "updatedAt": r.node.updated_at.to_rfc3339(), + }); + attach_source_record(&mut v, &r.node); + v + } + } +} + +/// Inject a `sourceRecord` object into a result `Value` ONLY when the memory +/// has external provenance, so legacy (agent/user-authored) memories keep their +/// exact prior result shape. +fn attach_source_record(value: &mut Value, node: &vestige_core::KnowledgeNode) { + let provenance = source_provenance(node); + if !provenance.is_null() + && let Value::Object(map) = value + { + map.insert("sourceRecord".to_string(), provenance); } } @@ -950,36 +995,44 @@ pub fn format_node(node: &vestige_core::KnowledgeNode, detail_level: &str) -> Va "tags": node.tags, "retentionStrength": node.retention_strength, }), - "full" => serde_json::json!({ - "id": node.id, - "content": node.content, - "nodeType": node.node_type, - "tags": node.tags, - "retentionStrength": node.retention_strength, - "storageStrength": node.storage_strength, - "retrievalStrength": node.retrieval_strength, - "source": node.source, - "sentimentScore": node.sentiment_score, - "sentimentMagnitude": node.sentiment_magnitude, - "createdAt": node.created_at.to_rfc3339(), - "updatedAt": node.updated_at.to_rfc3339(), - "lastAccessed": node.last_accessed.to_rfc3339(), - "nextReview": node.next_review.map(|dt| dt.to_rfc3339()), - "stability": node.stability, - "difficulty": node.difficulty, - "reps": node.reps, - "lapses": node.lapses, - "validFrom": node.valid_from.map(|dt| dt.to_rfc3339()), - "validUntil": node.valid_until.map(|dt| dt.to_rfc3339()), - }), + "full" => { + let mut v = serde_json::json!({ + "id": node.id, + "content": node.content, + "nodeType": node.node_type, + "tags": node.tags, + "retentionStrength": node.retention_strength, + "storageStrength": node.storage_strength, + "retrievalStrength": node.retrieval_strength, + "source": node.source, + "sentimentScore": node.sentiment_score, + "sentimentMagnitude": node.sentiment_magnitude, + "createdAt": node.created_at.to_rfc3339(), + "updatedAt": node.updated_at.to_rfc3339(), + "lastAccessed": node.last_accessed.to_rfc3339(), + "nextReview": node.next_review.map(|dt| dt.to_rfc3339()), + "stability": node.stability, + "difficulty": node.difficulty, + "reps": node.reps, + "lapses": node.lapses, + "validFrom": node.valid_from.map(|dt| dt.to_rfc3339()), + "validUntil": node.valid_until.map(|dt| dt.to_rfc3339()), + }); + attach_source_record(&mut v, node); + v + } // "summary" (default) - _ => serde_json::json!({ - "id": node.id, - "content": node.content, - "nodeType": node.node_type, - "tags": node.tags, - "retentionStrength": node.retention_strength, - }), + _ => { + let mut v = serde_json::json!({ + "id": node.id, + "content": node.content, + "nodeType": node.node_type, + "tags": node.tags, + "retentionStrength": node.retention_strength, + }); + attach_source_record(&mut v, node); + v + } } } @@ -1016,6 +1069,7 @@ mod tests { tags: vec![], valid_from: None, valid_until: None, + source_envelope: None, }; let node = storage.ingest(input).unwrap(); node.id @@ -1839,6 +1893,7 @@ mod tests { tags: tags.into_iter().map(String::from).collect(), valid_from: None, valid_until: None, + source_envelope: None, }; let node = storage.ingest(input).unwrap(); node.id diff --git a/crates/vestige-mcp/src/tools/session_context.rs b/crates/vestige-mcp/src/tools/session_context.rs index 97e52ff..3f17994 100644 --- a/crates/vestige-mcp/src/tools/session_context.rs +++ b/crates/vestige-mcp/src/tools/session_context.rs @@ -506,6 +506,7 @@ mod tests { tags: tags.into_iter().map(|s| s.to_string()).collect(), valid_from: None, valid_until: None, + source_envelope: None, }; let node = storage.ingest(input).unwrap(); node.id @@ -712,6 +713,7 @@ mod tests { tags: vec!["pattern".to_string(), "codebase:vestige".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }; storage.ingest(input).unwrap(); diff --git a/crates/vestige-mcp/src/tools/smart_ingest.rs b/crates/vestige-mcp/src/tools/smart_ingest.rs index c0b447d..60828fc 100644 --- a/crates/vestige-mcp/src/tools/smart_ingest.rs +++ b/crates/vestige-mcp/src/tools/smart_ingest.rs @@ -215,6 +215,7 @@ pub async fn execute( tags, valid_from: None, valid_until: None, + source_envelope: None, }; // ==================================================================== @@ -414,6 +415,7 @@ async fn execute_batch( tags, valid_from: None, valid_until: None, + source_envelope: None, }; // ================================================================ diff --git a/crates/vestige-mcp/src/tools/source_sync.rs b/crates/vestige-mcp/src/tools/source_sync.rs new file mode 100644 index 0000000..bfdd4b2 --- /dev/null +++ b/crates/vestige-mcp/src/tools/source_sync.rs @@ -0,0 +1,187 @@ +//! `source_sync` MCP tool (#57) — index an external system into Vestige. +//! +//! Turns Vestige into a durable, offline, provenance-linked retrieval layer +//! over a long-lived external system. The first connector is GitHub Issues: +//! point it at `owner/repo` and Vestige indexes every issue + its comments as +//! source-aware memories you can search semantically and cite back to the +//! canonical issue URL — re-runnable idempotently (no duplicates) and able to +//! tombstone issues that vanish upstream. +//! +//! Unlike the official GitHub MCP server (a stateless live API proxy), this +//! keeps a local index: searchable offline, embedded for semantic recall, +//! joinable with the rest of your memory, and temporally versioned. +//! +//! ## Auth (security) +//! +//! The GitHub token is read from the `GITHUB_TOKEN` (or `VESTIGE_GITHUB_TOKEN`) +//! environment variable, never from tool arguments, so credentials are not +//! logged in the conversation. Public repositories work without a token at a +//! lower rate limit. + +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::{Value, json}; + +use vestige_core::storage::Storage; + +/// JSON schema for the `source_sync` tool. +pub fn schema() -> Value { + json!({ + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": ["github"], + "description": "External system to sync. Currently: 'github' (GitHub Issues).", + "default": "github" + }, + "repo": { + "type": "string", + "description": "GitHub repository as 'owner/name', e.g. 'samvallad33/vestige'." + }, + "reconcile": { + "type": "boolean", + "description": "Also tombstone local memories for issues no longer visible upstream (an extra full enumeration pass). Default false on incremental syncs.", + "default": false + }, + "max_pages": { + "type": "integer", + "description": "Max API pages to fetch this run (each page is up to 100 issues). Lets a first sync of a large repo be resumed across calls. Default 10.", + "default": 10, + "minimum": 1, + "maximum": 1000 + } + }, + "required": ["repo"] + }) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SourceSyncArgs { + #[serde(default = "default_source")] + source: String, + repo: String, + #[serde(default)] + reconcile: bool, + #[serde(default, alias = "max_pages")] + max_pages: Option, +} + +fn default_source() -> String { + "github".to_string() +} + +/// Read the GitHub token from the environment (never from tool args). +fn github_token() -> Option { + std::env::var("GITHUB_TOKEN") + .or_else(|_| std::env::var("VESTIGE_GITHUB_TOKEN")) + .ok() + .filter(|s| !s.trim().is_empty()) +} + +pub async fn execute(storage: &Arc, args: Option) -> Result { + let args: SourceSyncArgs = match args { + Some(v) => serde_json::from_value(v).map_err(|e| format!("Invalid arguments: {e}"))?, + None => return Err("Missing arguments".to_string()), + }; + + if args.source != "github" { + return Err(format!( + "Unsupported source '{}'. Currently only 'github' is supported.", + args.source + )); + } + + let (owner, repo) = args + .repo + .split_once('/') + .filter(|(o, r)| !o.is_empty() && !r.is_empty()) + .ok_or_else(|| { + "repo must be in 'owner/name' form, e.g. 'samvallad33/vestige'".to_string() + })?; + + execute_github( + storage, + owner, + repo, + args.reconcile, + args.max_pages.unwrap_or(10), + ) + .await +} + +/// Connectors are feature-gated; surface a clear message when the build omits +/// them rather than failing obscurely. +#[cfg(not(feature = "connectors"))] +async fn execute_github( + _storage: &Arc, + _owner: &str, + _repo: &str, + _reconcile: bool, + _max_pages: usize, +) -> Result { + Err("This Vestige build was compiled without the 'connectors' feature. \ + Rebuild with --features connectors to enable source_sync." + .to_string()) +} + +#[cfg(feature = "connectors")] +async fn execute_github( + storage: &Arc, + owner: &str, + repo: &str, + reconcile: bool, + max_pages: usize, +) -> Result { + use vestige_core::connectors::github::{GithubConfig, GithubConnector}; + use vestige_core::connectors::run_sync; + + let config = GithubConfig::new(owner, repo).with_token(github_token()); + let connector = + GithubConnector::new(config).map_err(|e| format!("connector init failed: {e}"))?; + + let report = run_sync(storage.as_ref(), &connector, reconcile, max_pages) + .await + .map_err(|e| format!("sync failed: {e}"))?; + + let scope = format!("{owner}/{repo}"); + let total = report.created + report.updated + report.unchanged; + let authed = github_token().is_some(); + + let summary = format!( + "Synced {scope}: {} created, {} updated, {} unchanged{} ({total} records seen{}).", + report.created, + report.updated, + report.unchanged, + if report.reconciled { + format!(", {} tombstoned", report.tombstoned) + } else { + String::new() + }, + if authed { "" } else { ", unauthenticated" }, + ); + + Ok(json!({ + "ok": true, + "summary": summary, + "source": "github", + "scope": scope, + "created": report.created, + "updated": report.updated, + "unchanged": report.unchanged, + "tombstoned": report.tombstoned, + "reconciled": report.reconciled, + "cursor": report.new_cursor.map(|d| d.to_rfc3339()), + "authenticated": authed, + "warnings": report.warnings, + "hint": if total == 0 && !authed { + "No records returned. For private repos or higher rate limits, set GITHUB_TOKEN in the server environment." + } else if report.new_cursor.is_some() && total >= 100 { + "More may remain — run source_sync again to continue from the saved cursor." + } else { + "Search these with the normal search tools; results cite the GitHub issue URL." + } + })) +} diff --git a/crates/vestige-mcp/src/tools/suppress.rs b/crates/vestige-mcp/src/tools/suppress.rs index f06debc..342793e 100644 --- a/crates/vestige-mcp/src/tools/suppress.rs +++ b/crates/vestige-mcp/src/tools/suppress.rs @@ -177,6 +177,7 @@ mod tests { tags: vec!["test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap() .id diff --git a/crates/vestige-mcp/src/tools/timeline.rs b/crates/vestige-mcp/src/tools/timeline.rs index 7b3dcbf..937ace7 100644 --- a/crates/vestige-mcp/src/tools/timeline.rs +++ b/crates/vestige-mcp/src/tools/timeline.rs @@ -209,6 +209,7 @@ mod tests { tags: vec!["timeline-test".to_string()], valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } @@ -226,6 +227,7 @@ mod tests { tags: tags.iter().map(|t| t.to_string()).collect(), valid_from: None, valid_until: None, + source_envelope: None, }) .unwrap(); } diff --git a/docs/CONNECTORS.md b/docs/CONNECTORS.md new file mode 100644 index 0000000..8bd561a --- /dev/null +++ b/docs/CONNECTORS.md @@ -0,0 +1,150 @@ +# External-Source Connectors + +> Status: **v2.1.27** — GitHub Issues connector (reference). Redmine and others +> follow the same contract. Tracking issue: +> [#57](https://github.com/samvallad33/vestige/issues/57). + +Connectors let Vestige act as a durable, local **retrieval and reasoning layer** +over a long-lived external system — a ticket tracker, an issue board, a support +queue — **without replacing it**. The external system stays the source of truth. +Vestige indexes its records, embeds them for semantic recall, links them into the +memory graph, and **cites back** to the canonical record. + +## Why this is different from a ticket-system MCP + +The official GitHub / Jira MCP servers are **live API proxies**: every query hits +the upstream API, is rate-limited, keyword-only, online-only, and has no memory +of past state. Vestige instead keeps a **durable local index** of the records, so +you can: + +- search the history **offline** and **semantically** (embeddings, not just + keywords), +- **join** ticket history with the rest of your memory in one search, +- see a **point-in-time** view (records carry temporal validity), +- and re-sync **idempotently** — re-running never duplicates a record. + +## Quick start (GitHub Issues) + +1. (Optional but recommended) export a token so you get the authenticated rate + limit (5,000 req/hr vs 60 for anonymous) and access to private repos: + + ```sh + export GITHUB_TOKEN=ghp_xxx # or VESTIGE_GITHUB_TOKEN + ``` + + The token is read **only** from the environment — never passed as a tool + argument, never logged. + +2. Ask your agent to run the `source_sync` MCP tool: + + ```json + { "repo": "samvallad33/vestige" } + ``` + +3. Search as normal. Connector-sourced results carry a `sourceRecord` object with + the canonical issue URL: + + ```json + { + "content": "[samvallad33/vestige#57] Roadmap: external source connectors …", + "sourceRecord": { + "system": "github", + "id": "57", + "url": "https://github.com/samvallad33/vestige/issues/57", + "project": "samvallad33/vestige", + "type": "issue", + "author": "samvallad33", + "tombstoned": false + } + } + ``` + +## The `source_sync` tool + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `repo` | string | — (required) | `owner/name`, e.g. `samvallad33/vestige`. | +| `source` | string | `github` | External system. Currently only `github`. | +| `reconcile` | bool | `false` | Also tombstone local memories for issues no longer visible upstream (an extra full-enumeration pass). | +| `max_pages` | int | `10` | API pages to fetch this run (≤100 issues each). Lets a first sync of a large repo resume across calls. | + +The tool returns counts (`created` / `updated` / `unchanged` / `tombstoned`), +the saved `cursor`, whether it ran authenticated, and a `hint` for the next step. + +### Idempotent, incremental sync + +Each run: + +1. resumes from the saved cursor (the high-water mark on the record's upstream + update time), minus a small overlap window so same-second / clock-skewed + updates are never missed; +2. pages issues in ascending update order (`state=all`, so closing an issue is + **not** mistaken for a deletion), folding each issue + its comments into one + memory; +3. routes each record through an **idempotent upsert** keyed on + `(source_system, source_id)`: + - unseen record → **insert**, + - changed content (by content hash) → **update in place** + re-embed, + - unchanged content → **no-op** (only the "last seen" time advances); +4. advances and persists the cursor only after the run, so an interruption + re-scans rather than skips. + +Re-running `source_sync` on the same repo is therefore safe and cheap — it picks +up only what changed. + +### Deletions (tombstoning) + +Neither GitHub nor Redmine exposes a deletion feed, so an incremental sync can +never *see* a delete. Pass `reconcile: true` to run a reconciliation pass: Vestige +enumerates the currently-visible issue ids and **invalidates** (does not purge) +any local record no longer present. A tombstoned record keeps its content for +audit but drops out of "currently valid" retrieval (`sourceRecord.tombstoned` is +`true`). If the record reappears upstream, the next sync un-tombstones it. + +## The source envelope + +Every connector-ingested memory carries structured provenance, distinct from the +legacy free-form `source` label: + +| Field | Purpose | +|---|---| +| `source_system` | `github`, `redmine`, … (namespaces ids). | +| `source_id` | Native id (issue number, ticket id). | +| `source_url` | Canonical link back — the citation. | +| `source_updated_at` | Upstream update time (the sync cursor field). | +| `content_hash` | Change detector → idempotency. | +| `synced_at` | When the connector last saw the record live. | +| `source_project` | Repo / project / space. | +| `source_type` | `issue`, `comment`, … | +| `source_author` | Reporter / author upstream. | + +`(source_system, source_id)` is enforced unique, so there is exactly one memory +per external record. Legacy memories (agent- or user-authored) have no envelope +and are completely unaffected. + +## Building + +The connector HTTP client is behind the `connectors` cargo feature, which is +**on by default in the MCP server** (`vestige-mcp`). A build without it still +exposes the `source_sync` tool but returns a clear "rebuild with `--features +connectors`" message. The core library (`vestige-core`) leaves the feature +**off** by default, so library consumers that don't need connectors link no HTTP +client. + +```sh +# default MCP build already includes connectors +cargo build -p vestige-mcp --release + +# explicit, or for the core lib +cargo build -p vestige-core --features connectors +``` + +## Writing a new connector + +Implement the `Connector` trait in `vestige_core::connectors` (fetch a window of +records updated since a cursor, page forward, and optionally enumerate live ids +for reconciliation), produce `NormalizedRecord`s with a filled +`SourceEnvelope`, and hand them to `run_sync`. The GitHub connector +(`crates/vestige-core/src/connectors/github.rs`) is the reference +implementation. The sync driver, idempotent upsert, cursor checkpointing, and +tombstone reconciliation are all reused for free. diff --git a/tests/e2e/src/harness/db_manager.rs b/tests/e2e/src/harness/db_manager.rs index 345a94c..268432c 100644 --- a/tests/e2e/src/harness/db_manager.rs +++ b/tests/e2e/src/harness/db_manager.rs @@ -31,6 +31,7 @@ fn make_ingest_input( source, valid_from, valid_until, + source_envelope: None, } } diff --git a/tests/e2e/src/mocks/fixtures.rs b/tests/e2e/src/mocks/fixtures.rs index 6929e56..87d786b 100644 --- a/tests/e2e/src/mocks/fixtures.rs +++ b/tests/e2e/src/mocks/fixtures.rs @@ -29,6 +29,7 @@ fn make_ingest_input( source, valid_from, valid_until, + source_envelope: None, } } From 4e893c02ff510b10667f37e20dcd55b13eaf4c98 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Fri, 19 Jun 2026 02:21:25 -0500 Subject: [PATCH 36/38] feat(connectors): add Redmine and source filters (#57) --- CHANGELOG.md | 45 +- crates/vestige-core/src/connectors/github.rs | 25 +- crates/vestige-core/src/connectors/mod.rs | 21 +- crates/vestige-core/src/connectors/redmine.rs | 737 ++++++++++++++++++ crates/vestige-core/src/storage/sqlite.rs | 50 +- crates/vestige-mcp/src/server.rs | 2 +- .../vestige-mcp/src/tools/search_unified.rs | 415 +++++++++- crates/vestige-mcp/src/tools/source_sync.rs | 184 ++++- docs/CONNECTORS.md | 74 +- 9 files changed, 1445 insertions(+), 108 deletions(-) create mode 100644 crates/vestige-core/src/connectors/redmine.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2918af4..a220029 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > Bump `version` in the workspace `Cargo.toml`, both crates, `server.json`, and > `package.json` to `2.1.27` at release/tag time, and date this heading. -Roadmap [#57](https://github.com/samvallad33/vestige/issues/57), **Phase 1–3**: -Vestige can now act as a durable, local, semantically-searchable retrieval layer -over an external system of record — starting with GitHub Issues — without -replacing it. The external system stays canonical; Vestige **indexes, connects, -retrieves, and cites back** to the source record. +Roadmap [#57](https://github.com/samvallad33/vestige/issues/57), **Phases 1–4 +(complete)**: Vestige can now act as a durable, local, semantically-searchable +retrieval layer over an external system of record — GitHub Issues and Redmine — +without replacing it. The external system stays canonical; Vestige **indexes, +connects, retrieves, and cites back** to the source record. Unlike a live ticket-system MCP proxy (which holds no state and is rate-limited per query), Vestige keeps a durable embedded index: searchable **offline**, @@ -25,12 +25,22 @@ content-hash idempotent sync, and tombstoning of vanished records. ### Added -- **`source_sync` MCP tool** — point Vestige at a GitHub repo - (`{"repo": "owner/name"}`) and it indexes every issue + its comments as - source-aware memories. Re-running updates changed issues in place (no - duplicates); `reconcile: true` tombstones issues no longer visible upstream. - Auth via the `GITHUB_TOKEN` (or `VESTIGE_GITHUB_TOKEN`) environment variable; - public repos work without a token at a lower rate limit. +- **`source_sync` MCP tool** — index an external system into Vestige. + - GitHub: `{"source": "github", "repo": "owner/name"}` indexes every issue + + its comments. Auth via `GITHUB_TOKEN` (public repos work tokenless at a + lower rate limit). + - Redmine: `{"source": "redmine", "project": ""}` indexes a project's + issues + journals (comments and status/assignment history). Host from + `REDMINE_URL`, auth from `REDMINE_API_KEY`. + - Re-running updates changed issues in place (no duplicates); `reconcile: + true` tombstones issues no longer visible upstream. +- **Source-aware investigation filters on `search`** (Phase 4) — filter results + by `source_system`, `source_project`, `source_id`, `source_type`, + `source_author`, a `source_updated_after`/`source_updated_before` date range, + and `source_status` (`valid` / `tombstoned` / `any`). Status, tracker, and + priority remain filterable via the existing `tag_prefix` (the connectors emit + `status:`/`tracker:`/`priority:`/`label:` tags). Applied as post-filters; + non-connector memories are excluded from a source-scoped query. - **Source envelope** on every memory — structured, machine-readable provenance (`source_system`, `source_id`, `source_url`, `source_updated_at`, `content_hash`, `synced_at`, `source_project`, `source_type`, `source_author`) @@ -44,10 +54,15 @@ content-hash idempotent sync, and tombstoning of vanished records. record is retained for audit but drops out of current retrieval). - **Connector contract** (`vestige_core::connectors`) — a small source-agnostic `Connector` trait + `run_sync` driver (cursor overlap window, incremental - paging, optional deletion reconcile) and a GitHub Issues reference connector - behind the optional `connectors` cargo feature (on by default in the MCP - server, off in the core library's default features so non-connector consumers - link no HTTP client). + paging, optional deletion reconcile) with two reference connectors behind the + optional `connectors` cargo feature (on by default in the MCP server, off in + the core library's default features so non-connector consumers link no HTTP + client): + - **GitHub Issues** — `state=all`, `since` cursor, Link-header pagination, + drops PRs, host-pinned next-url. + - **Redmine** — `status_id=*` (open + closed), hex-encoded `updated_on>=` + cursor, `offset` pagination, per-issue detail fetch for journals (the list + endpoint omits them), `X-Redmine-API-Key` header auth. ### Database diff --git a/crates/vestige-core/src/connectors/github.rs b/crates/vestige-core/src/connectors/github.rs index a373dcf..24bd60e 100644 --- a/crates/vestige-core/src/connectors/github.rs +++ b/crates/vestige-core/src/connectors/github.rs @@ -490,25 +490,25 @@ mod tests { fn hash_stable_across_label_order_and_changes_on_edit() { let c = connector(); let mut a = issue(1, "T", "body", "open"); - a.labels = vec![ - RawLabel { name: "b".into() }, - RawLabel { name: "a".into() }, - ]; + a.labels = vec![RawLabel { name: "b".into() }, RawLabel { name: "a".into() }]; let mut b = issue(1, "T", "body", "open"); - b.labels = vec![ - RawLabel { name: "a".into() }, - RawLabel { name: "b".into() }, - ]; + b.labels = vec![RawLabel { name: "a".into() }, RawLabel { name: "b".into() }]; let ha = c.normalize(&a, &[]).envelope.content_hash; let hb = c.normalize(&b, &[]).envelope.content_hash; assert_eq!(ha, hb, "label order must not change the hash"); // Editing the body must change the hash. - let edited = c.normalize(&issue(1, "T", "EDITED", "open"), &[]).envelope.content_hash; + let edited = c + .normalize(&issue(1, "T", "EDITED", "open"), &[]) + .envelope + .content_hash; assert_ne!(ha, edited); // Closing the issue changes state → changes the hash (not a no-op). - let closed = c.normalize(&issue(1, "T", "body", "closed"), &[]).envelope.content_hash; + let closed = c + .normalize(&issue(1, "T", "body", "closed"), &[]) + .envelope + .content_hash; assert_ne!(ha, closed); } @@ -533,7 +533,10 @@ mod tests { let second_pos = rec.content.find("second").unwrap(); assert!(first_pos < second_pos, "comments must fold in id order"); - let no_comments = c.normalize(&issue(1, "T", "body", "open"), &[]).envelope.content_hash; + let no_comments = c + .normalize(&issue(1, "T", "body", "open"), &[]) + .envelope + .content_hash; assert_ne!( rec.envelope.content_hash, no_comments, "comments must contribute to the hash" diff --git a/crates/vestige-core/src/connectors/mod.rs b/crates/vestige-core/src/connectors/mod.rs index 033e065..e82a5ab 100644 --- a/crates/vestige-core/src/connectors/mod.rs +++ b/crates/vestige-core/src/connectors/mod.rs @@ -11,9 +11,9 @@ //! - The [`Connector`] contract, [`NormalizedRecord`] shape, and the stable //! [`content_hash`] are pure (no network) and always compiled, so the sync //! semantics are unit-testable without hitting an API. -//! - Network-backed reference connectors (e.g. [`github`]) live behind the -//! `connectors` cargo feature so the default local-first build links no HTTP -//! client. +//! - Network-backed reference connectors ([`github`] and [`redmine`]) live +//! behind the `connectors` cargo feature so the default local-first build +//! links no HTTP client. //! //! ## Sync contract (the part that makes re-running safe) //! @@ -39,6 +39,9 @@ use crate::storage::ConnectorCursor; #[cfg(feature = "connectors")] pub mod github; +#[cfg(feature = "connectors")] +pub mod redmine; + /// A single external record, already normalized into the fields Vestige needs. /// /// The connector is responsible for flattening a possibly-rich source record @@ -329,8 +332,16 @@ mod tests { #[test] fn content_hash_is_order_independent() { - let a = content_hash(&[("title", "Crash"), ("body", "stacktrace"), ("state", "open")]); - let b = content_hash(&[("state", "open"), ("title", "Crash"), ("body", "stacktrace")]); + let a = content_hash(&[ + ("title", "Crash"), + ("body", "stacktrace"), + ("state", "open"), + ]); + let b = content_hash(&[ + ("state", "open"), + ("title", "Crash"), + ("body", "stacktrace"), + ]); assert_eq!(a, b, "reordering fields must not change the hash"); } diff --git a/crates/vestige-core/src/connectors/redmine.rs b/crates/vestige-core/src/connectors/redmine.rs new file mode 100644 index 0000000..1b2a316 --- /dev/null +++ b/crates/vestige-core/src/connectors/redmine.rs @@ -0,0 +1,737 @@ +//! Redmine connector (#57). +//! +//! Indexes a Redmine project's issues + journals (comments and status/assignment +//! history) into source-aware Vestige memories so an investigative agent can +//! search and reason over years of ticket history **offline**, **semantically**, +//! and **cited back to the canonical issue URL**. Redmine stays the system of +//! record; Vestige indexes, connects, retrieves, and links back. +//! +//! ## Incremental sync (per the connector sync contract) +//! +//! Redmine's REST API has three traps this connector handles explicitly (all +//! confirmed against the official wiki + canonical defects): +//! +//! - **`status_id=*` is mandatory.** The list endpoint returns *open issues +//! only* by default, so without it closing an issue looks like a deletion and +//! closed issues are never synced (Defect #19088). We pass it on both the +//! incremental pull and the reconcile enumeration. +//! - **`include=journals` is silently ignored on the list endpoint.** Journals +//! come back only on the per-issue detail endpoint `GET /issues/{id}.json` +//! (Defect #35242), so each changed issue costs one extra round-trip. +//! - **Filter operators must be hex-encoded** in the compact form +//! (`updated_on=>=…` → `updated_on=%3E%3D…`). We build the query with +//! `reqwest`'s `.query(&[…])` and pass the raw `>=…` value so it is encoded +//! exactly once (no double-encoding). +//! +//! `sort=updated_on:asc` pages forward in cursor order so a mid-run interruption +//! resumes safely; the `since = cursor − overlap` window + the `content_hash` +//! no-op make the re-scan free. Redmine has no deletion feed, so deletions are +//! reconciled out-of-band via [`list_live_ids`](Connector::list_live_ids). + +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +use super::{ + Connector, ConnectorError, ConnectorResult, FetchPage, NormalizedRecord, content_hash, +}; +use crate::memory::SourceEnvelope; + +const USER_AGENT: &str = concat!("vestige-connector/", env!("CARGO_PKG_VERSION")); +const PAGE_LIMIT: u32 = 100; + +/// Configuration for a Redmine connector instance bound to one project. +#[derive(Debug, Clone)] +pub struct RedmineConfig { + /// Base URL of the Redmine instance, e.g. `https://redmine.example.com`. + pub base_url: String, + /// Project identifier to scope the sync to. May be the numeric id or the + /// project identifier slug — used as `project_id` and stored as + /// `source_project`. (Note: Redmine's `project_id` list filter wants the + /// numeric id; the slug works as the human-readable scope label.) + pub project: String, + /// API access key. Optional only if the instance allows anonymous REST. + pub api_key: Option, + /// Max journals to fold into one issue memory (defense against huge threads). + pub max_journals: usize, +} + +impl RedmineConfig { + pub fn new(base_url: impl Into, project: impl Into) -> Self { + Self { + base_url: base_url.into(), + project: project.into(), + api_key: None, + max_journals: 100, + } + } + + pub fn with_api_key(mut self, key: Option) -> Self { + self.api_key = key; + self + } + + /// Base URL with any trailing slash removed. + fn root(&self) -> String { + self.base_url.trim_end_matches('/').to_string() + } +} + +/// A Redmine connector bound to one project. +pub struct RedmineConnector { + config: RedmineConfig, + scope: String, + client: reqwest::Client, +} + +impl RedmineConnector { + pub fn new(config: RedmineConfig) -> ConnectorResult { + if config.base_url.trim().is_empty() { + return Err(ConnectorError::Config("base_url is required".to_string())); + } + if config.project.trim().is_empty() { + return Err(ConnectorError::Config("project is required".to_string())); + } + if reqwest::Url::parse(&config.root()).is_err() { + return Err(ConnectorError::Config(format!( + "base_url is not a valid URL: {}", + config.base_url + ))); + } + let client = reqwest::Client::builder() + .user_agent(USER_AGENT) + .build() + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + let scope = config.project.clone(); + Ok(Self { + config, + scope, + client, + }) + } + + fn auth(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + let req = req.header("Accept", "application/json"); + match &self.config.api_key { + // The key goes in the header (not the URL) so it stays out of proxy + // and access logs. + Some(k) => req.header("X-Redmine-API-Key", k), + None => req, + } + } + + fn classify_status(resp: &reqwest::Response) -> Option { + let status = resp.status(); + if status.is_success() { + return None; + } + if status.as_u16() == 429 { + let retry = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .map(std::time::Duration::from_secs); + return Some(ConnectorError::RateLimited(retry)); + } + let message = match status.as_u16() { + // A valid key against an instance with REST disabled 401/403s; make + // that distinguishable from "no results". + 401 | 403 => { + "unauthorized — check REDMINE_API_KEY and that the instance has the REST API enabled (Administration → Settings → API)" + .to_string() + } + _ => status + .canonical_reason() + .unwrap_or("request failed") + .to_string(), + }; + Some(ConnectorError::Source { + status: status.as_u16(), + message, + }) + } + + /// Fetch the journals + relations for one issue (the detail endpoint — + /// journals are not returned on the list endpoint). + async fn fetch_detail(&self, issue_id: u64) -> ConnectorResult { + let url = format!("{}/issues/{}.json", self.config.root(), issue_id); + let resp = self + .auth(self.client.get(&url)) + .query(&[("include", "journals,relations")]) + .send() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if let Some(err) = Self::classify_status(&resp) { + return Err(err); + } + let wrapper: IssueWrapper = resp + .json() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + Ok(wrapper.issue) + } + + /// Fold a raw issue (with journals) into one normalized memory record. + fn normalize(&self, issue: &RawIssue) -> NormalizedRecord { + let status_name = issue.status.as_ref().map(|s| s.name.clone()); + let tracker_name = issue.tracker.as_ref().map(|t| t.name.clone()); + let author = issue.author.as_ref().map(|a| a.name.clone()); + + // Journals sorted by id for a stable order + stable hash. Keep notes + // and field changes so status/assignment history remains searchable. + let mut journals: Vec<&RawJournal> = issue + .journals + .iter() + .filter(|j| { + j.notes + .as_deref() + .map(|n| !n.trim().is_empty()) + .unwrap_or(false) + || !j.details.is_empty() + }) + .collect(); + journals.sort_by_key(|j| j.id); + journals.truncate(self.config.max_journals); + + // Human-readable content. + let mut content = format!("[{}#{}] {}\n", self.scope, issue.id, issue.subject); + if let Some(s) = &status_name { + content.push_str(&format!("Status: {s}\n")); + } + if let Some(t) = &tracker_name { + content.push_str(&format!("Tracker: {t}\n")); + } + if let Some(desc) = &issue.description + && !desc.trim().is_empty() + { + content.push('\n'); + content.push_str(desc.trim()); + content.push('\n'); + } + for j in &journals { + let who = j.user.as_ref().map(|u| u.name.as_str()).unwrap_or("?"); + let note = j.notes.as_deref().unwrap_or("").trim(); + if !note.is_empty() { + content.push_str(&format!("\n- {who}: {note}")); + } + for detail in &j.details { + content.push_str(&format!( + "\n- {who} changed {}{}: {} -> {}", + detail.property.as_deref().unwrap_or("field"), + detail + .name + .as_deref() + .map(|n| format!(".{n}")) + .unwrap_or_default(), + detail.old_value.as_deref().unwrap_or(""), + detail.new_value.as_deref().unwrap_or("") + )); + } + } + if !issue.relations.is_empty() { + content.push_str("\n\nRelations:"); + let mut relations: Vec<&RawRelation> = issue.relations.iter().collect(); + relations.sort_by_key(|r| r.id); + for relation in relations { + let related = relation.related_issue_id(issue.id); + content.push_str(&format!( + "\n- #{} ({})", + related, + relation.relation_type.as_deref().unwrap_or("relates") + )); + if let Some(delay) = relation.delay { + content.push_str(&format!(", delay {delay}")); + } + } + } + + // Stable content hash — meaning only, never the cursor (`updated_on`) or + // volatile counts. Journals and relations contribute stable fields in id + // order. + let journals_blob = journals + .iter() + .map(|j| { + let details = j + .details + .iter() + .map(|d| { + format!( + "{}:{}:{}:{}", + d.property.as_deref().unwrap_or(""), + d.name.as_deref().unwrap_or(""), + d.old_value.as_deref().unwrap_or(""), + d.new_value.as_deref().unwrap_or("") + ) + }) + .collect::>() + .join("\u{1e}"); + format!( + "{}:{}:{}", + j.id, + j.notes.as_deref().unwrap_or("").trim(), + details + ) + }) + .collect::>() + .join("\u{1f}"); + let relations_blob = { + let mut relations: Vec<&RawRelation> = issue.relations.iter().collect(); + relations.sort_by_key(|r| r.id); + relations + .iter() + .map(|r| { + format!( + "{}:{}:{}:{}", + r.id, + r.issue_id.unwrap_or_default(), + r.issue_to_id.unwrap_or_default(), + r.relation_type.as_deref().unwrap_or("") + ) + }) + .collect::>() + .join("\u{1f}") + }; + let id_str = issue.id.to_string(); + let status_id_str = issue + .status + .as_ref() + .map(|s| s.id.to_string()) + .unwrap_or_default(); + let tracker_id_str = issue + .tracker + .as_ref() + .map(|t| t.id.to_string()) + .unwrap_or_default(); + let done_ratio_str = issue.done_ratio.unwrap_or(0).to_string(); + let desc_str = issue.description.clone().unwrap_or_default(); + let hash = content_hash(&[ + ("id", &id_str), + ("subject", &issue.subject), + ("description", &desc_str), + ("status_id", &status_id_str), + ("tracker_id", &tracker_id_str), + ("done_ratio", &done_ratio_str), + ("journals", &journals_blob), + ("relations", &relations_blob), + ]); + + // Tags, lowercased — `tag_prefix` matching is case-sensitive, and + // Redmine status/tracker names are mixed-case. + let mut tags = vec!["redmine".to_string(), "issue".to_string()]; + if let Some(s) = &status_name { + tags.push(format!("status:{}", s.to_lowercase())); + } + if let Some(t) = &tracker_name { + tags.push(format!("tracker:{}", t.to_lowercase())); + } + if let Some(p) = &issue.priority { + tags.push(format!("priority:{}", p.name.to_lowercase())); + } + + let envelope = SourceEnvelope { + source_system: Some("redmine".to_string()), + source_id: Some(issue.id.to_string()), + source_url: Some(format!("{}/issues/{}", self.config.root(), issue.id)), + source_updated_at: issue + .updated_on + .as_deref() + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|d| d.with_timezone(&Utc)), + content_hash: Some(hash), + synced_at: Some(Utc::now()), + source_project: Some(self.scope.clone()), + source_type: Some("issue".to_string()), + source_author: author, + }; + + NormalizedRecord { + content, + tags, + envelope, + } + } +} + +impl Connector for RedmineConnector { + fn source_system(&self) -> &str { + "redmine" + } + + fn scope(&self) -> &str { + &self.scope + } + + async fn fetch_updated( + &self, + since: Option>, + cursor: Option, + ) -> ConnectorResult { + // The cursor carries the next offset (Redmine pages by offset, not an + // opaque url). First page = offset 0. + let offset: u32 = cursor.as_deref().and_then(|c| c.parse().ok()).unwrap_or(0); + + let url = format!("{}/issues.json", self.config.root()); + let limit_str = PAGE_LIMIT.to_string(); + let offset_str = offset.to_string(); + // Build params; reqwest percent-encodes each value exactly once, so we + // pass the RAW `>=…` operator (it becomes %3E%3D on the wire). Do not + // pre-encode here or it would be double-encoded. + let mut params: Vec<(&str, String)> = vec![ + ("status_id", "*".to_string()), + ("sort", "updated_on:asc".to_string()), + ("project_id", self.config.project.clone()), + ("limit", limit_str), + ("offset", offset_str), + ]; + if let Some(s) = since { + let since_z = s.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + params.push(("updated_on", format!(">={since_z}"))); + } + + let resp = self + .auth(self.client.get(&url)) + .query(¶ms) + .send() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if let Some(err) = Self::classify_status(&resp) { + return Err(err); + } + let page: IssueListResponse = resp + .json() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + + // Per-issue detail fetch for journals (list endpoint omits them). + let mut records = Vec::new(); + for summary in &page.issues { + let detailed = match self.fetch_detail(summary.id).await { + Ok(d) => d, + // A single issue failing detail-fetch should not abort the page; + // fall back to the list-level fields (no journals). + Err(_) => summary.clone(), + }; + records.push(self.normalize(&detailed)); + } + + // Advance the offset cursor until we've walked total_count. + let next_offset = offset + page.issues.len() as u32; + let next_cursor = if (next_offset as u64) < page.total_count && !page.issues.is_empty() { + Some(next_offset.to_string()) + } else { + None + }; + + Ok(FetchPage { + records, + next_cursor, + }) + } + + async fn list_live_ids(&self) -> ConnectorResult>> { + // Enumerate all issue ids (open AND closed) for the reconcile pass. + // status_id=* is mandatory here too, or closed issues read as deleted. + let mut ids = Vec::new(); + let mut offset: u32 = 0; + loop { + let url = format!("{}/issues.json", self.config.root()); + let resp = self + .auth(self.client.get(&url)) + .query(&[ + ("status_id", "*".to_string()), + ("project_id", self.config.project.clone()), + ("limit", PAGE_LIMIT.to_string()), + ("offset", offset.to_string()), + ]) + .send() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if let Some(err) = Self::classify_status(&resp) { + return Err(err); + } + let page: IssueListResponse = resp + .json() + .await + .map_err(|e| ConnectorError::Transport(e.to_string()))?; + if page.issues.is_empty() { + break; + } + for issue in &page.issues { + ids.push(issue.id.to_string()); + } + offset += page.issues.len() as u32; + if (offset as u64) >= page.total_count { + break; + } + } + Ok(Some(ids)) + } +} + +// --------------------------------------------------------------------------- +// Raw Redmine API shapes (only the fields we use) +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct IssueListResponse { + #[serde(default)] + issues: Vec, + #[serde(default)] + total_count: u64, +} + +#[derive(Debug, Deserialize)] +struct IssueWrapper { + issue: RawIssue, +} + +#[derive(Debug, Clone, Deserialize)] +struct RawIssue { + id: u64, + #[serde(default)] + subject: String, + #[serde(default)] + description: Option, + #[serde(default)] + status: Option, + #[serde(default)] + tracker: Option, + #[serde(default)] + priority: Option, + #[serde(default)] + author: Option, + #[serde(default)] + done_ratio: Option, + #[serde(default)] + updated_on: Option, + #[serde(default)] + journals: Vec, + #[serde(default)] + relations: Vec, +} + +/// Redmine `{id, name}` reference (status, tracker, priority, user, …). +#[derive(Debug, Clone, Deserialize)] +struct NamedRef { + #[serde(default)] + id: i64, + #[serde(default)] + name: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct RawJournal { + id: u64, + #[serde(default)] + notes: Option, + #[serde(default)] + user: Option, + #[serde(default)] + details: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct RawJournalDetail { + #[serde(default)] + property: Option, + #[serde(default)] + name: Option, + #[serde(default)] + old_value: Option, + #[serde(default)] + new_value: Option, +} + +#[derive(Debug, Clone, Deserialize)] +struct RawRelation { + #[serde(default)] + id: u64, + #[serde(default)] + issue_id: Option, + #[serde(default)] + issue_to_id: Option, + #[serde(default)] + relation_type: Option, + #[serde(default)] + delay: Option, +} + +impl RawRelation { + fn related_issue_id(&self, current_issue_id: u64) -> u64 { + match (self.issue_id, self.issue_to_id) { + (Some(from), Some(to)) if from == current_issue_id => to, + (Some(from), Some(to)) if to == current_issue_id => from, + (_, Some(to)) => to, + (Some(from), _) => from, + _ => 0, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn issue(id: u64, subject: &str, desc: &str, status: (i64, &str)) -> RawIssue { + RawIssue { + id, + subject: subject.to_string(), + description: Some(desc.to_string()), + status: Some(NamedRef { + id: status.0, + name: status.1.to_string(), + }), + tracker: Some(NamedRef { + id: 1, + name: "Bug".to_string(), + }), + priority: Some(NamedRef { + id: 2, + name: "Normal".to_string(), + }), + author: Some(NamedRef { + id: 7, + name: "Jane Dev".to_string(), + }), + done_ratio: Some(0), + updated_on: Some("2026-06-19T00:00:00Z".to_string()), + journals: vec![], + relations: vec![], + } + } + + fn connector() -> RedmineConnector { + RedmineConnector::new(RedmineConfig::new("https://redmine.example.com", "infra")).unwrap() + } + + #[test] + fn rejects_empty_and_bad_config() { + assert!(RedmineConnector::new(RedmineConfig::new("", "p")).is_err()); + assert!(RedmineConnector::new(RedmineConfig::new("https://r.example", "")).is_err()); + assert!(RedmineConnector::new(RedmineConfig::new("not a url", "p")).is_err()); + } + + #[test] + fn normalize_builds_keyed_envelope_with_citation() { + let c = connector(); + let rec = c.normalize(&issue(123, "Disk full", "df -h shows 100%", (1, "New"))); + let env = &rec.envelope; + assert!(env.has_key()); + assert_eq!(env.source_system.as_deref(), Some("redmine")); + assert_eq!(env.source_id.as_deref(), Some("123")); + assert_eq!( + env.source_url.as_deref(), + Some("https://redmine.example.com/issues/123") + ); + assert_eq!(env.source_project.as_deref(), Some("infra")); + assert_eq!(env.source_author.as_deref(), Some("Jane Dev")); + assert!(rec.content.contains("Disk full")); + // Tags lowercased so the case-sensitive tag_prefix filter matches. + assert!(rec.tags.contains(&"status:new".to_string())); + assert!(rec.tags.contains(&"tracker:bug".to_string())); + assert!(rec.tags.contains(&"priority:normal".to_string())); + } + + #[test] + fn status_change_changes_hash() { + let c = connector(); + let new = c + .normalize(&issue(1, "T", "body", (1, "New"))) + .envelope + .content_hash; + let closed = c + .normalize(&issue(1, "T", "body", (5, "Closed"))) + .envelope + .content_hash; + assert_ne!( + new, closed, + "a status change must change the hash → re-embed" + ); + } + + #[test] + fn journals_fold_in_id_order_and_affect_hash() { + let c = connector(); + let mut iss = issue(1, "T", "body", (1, "New")); + iss.journals = vec![ + RawJournal { + id: 20, + notes: Some("second".to_string()), + user: Some(NamedRef { + id: 1, + name: "B".to_string(), + }), + details: vec![], + }, + RawJournal { + id: 10, + notes: Some("first".to_string()), + user: Some(NamedRef { + id: 2, + name: "A".to_string(), + }), + details: vec![], + }, + // Pure empty journal must be dropped, not folded. + RawJournal { + id: 30, + notes: None, + user: Some(NamedRef { + id: 3, + name: "C".to_string(), + }), + details: vec![], + }, + ]; + let rec = c.normalize(&iss); + let first = rec.content.find("first").unwrap(); + let second = rec.content.find("second").unwrap(); + assert!(first < second, "journals fold in id order"); + + let no_journals = c + .normalize(&issue(1, "T", "body", (1, "New"))) + .envelope + .content_hash; + assert_ne!( + rec.envelope.content_hash, no_journals, + "journals must contribute to the hash" + ); + } + + #[test] + fn journal_details_and_relations_are_searchable_and_hashed() { + let c = connector(); + let mut iss = issue(1, "T", "body", (1, "New")); + iss.journals = vec![RawJournal { + id: 1, + notes: None, + user: Some(NamedRef { + id: 2, + name: "A".to_string(), + }), + details: vec![RawJournalDetail { + property: Some("attr".to_string()), + name: Some("status_id".to_string()), + old_value: Some("1".to_string()), + new_value: Some("5".to_string()), + }], + }]; + iss.relations = vec![RawRelation { + id: 9, + issue_id: Some(1), + issue_to_id: Some(42), + relation_type: Some("relates".to_string()), + delay: None, + }]; + + let rec = c.normalize(&iss); + assert!(rec.content.contains("changed attr.status_id: 1 -> 5")); + assert!(rec.content.contains("#42 (relates)")); + + let no_history = c.normalize(&issue(1, "T", "body", (1, "New"))); + assert_ne!( + rec.envelope.content_hash, no_history.envelope.content_hash, + "field-change journals and relations must affect idempotent updates" + ); + } +} diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 399b2fb..d353ec1 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -9655,7 +9655,11 @@ impl SqliteMemoryStore { /// Read the incremental-sync checkpoint for a `(source_system, scope)`. /// Returns a zeroed cursor (no high-water mark) if none has been saved yet. - pub fn get_connector_cursor(&self, source_system: &str, scope: &str) -> Result { + pub fn get_connector_cursor( + &self, + source_system: &str, + scope: &str, + ) -> Result { let reader = self .reader .lock() @@ -9873,7 +9877,11 @@ mod tests { assert_eq!(r.outcome, SourceUpsertOutcome::Unchanged); assert_eq!(r.node_id, r1.node_id, "must reuse the same memory id"); } - assert_eq!(node_count(&store), 1, "idempotent: still exactly one memory"); + assert_eq!( + node_count(&store), + 1, + "idempotent: still exactly one memory" + ); } #[test] @@ -9945,7 +9953,13 @@ mod tests { let mut c2 = cursor.clone(); c2.records_seen = 99; store.save_connector_cursor(&c2).unwrap(); - assert_eq!(store.get_connector_cursor("github", "o/r").unwrap().records_seen, 99); + assert_eq!( + store + .get_connector_cursor("github", "o/r") + .unwrap() + .records_seen, + 99 + ); } #[test] @@ -9977,9 +9991,15 @@ mod tests { ) .unwrap() }; - assert!(two.1.is_some(), "tombstoned record must have valid_until set"); + assert!( + two.1.is_some(), + "tombstoned record must have valid_until set" + ); let node = store.get_node(&two.0).unwrap().unwrap(); - assert!(!node.is_currently_valid(), "tombstoned node is not valid now"); + assert!( + !node.is_currently_valid(), + "tombstoned node is not valid now" + ); assert_eq!(node.content, "issue 2", "content retained for audit"); // A reappearing record un-tombstones on next upsert (clears valid_until). @@ -9987,7 +10007,10 @@ mod tests { .upsert_by_source(source_input("2", "issue 2", "h2")) .unwrap(); let revived = store.get_node(&two.0).unwrap().unwrap(); - assert!(revived.is_currently_valid(), "re-synced record is valid again"); + assert!( + revived.is_currently_valid(), + "re-synced record is valid again" + ); } #[test] @@ -10013,7 +10036,10 @@ mod tests { .unwrap(); } assert!( - store.superseded_node_ids().unwrap().contains(&created.node_id), + store + .superseded_node_ids() + .unwrap() + .contains(&created.node_id), "precondition: node is superseded" ); @@ -10023,7 +10049,10 @@ mod tests { .unwrap(); assert_eq!(res.outcome, SourceUpsertOutcome::Updated); assert!( - !store.superseded_node_ids().unwrap().contains(&created.node_id), + !store + .superseded_node_ids() + .unwrap() + .contains(&created.node_id), "superseded_by must be cleared on re-sync (no bitemporal zombie)" ); let node = store.get_node(&created.node_id).unwrap().unwrap(); @@ -10044,7 +10073,10 @@ mod tests { .unwrap(); assert_eq!(res2.outcome, SourceUpsertOutcome::Unchanged); assert!( - !store.superseded_node_ids().unwrap().contains(&created.node_id), + !store + .superseded_node_ids() + .unwrap() + .contains(&created.node_id), "Unchanged branch must also clear superseded_by" ); } diff --git a/crates/vestige-mcp/src/server.rs b/crates/vestige-mcp/src/server.rs index b8f90e7..6b919fa 100644 --- a/crates/vestige-mcp/src/server.rs +++ b/crates/vestige-mcp/src/server.rs @@ -285,7 +285,7 @@ impl McpServer { // ================================================================ ToolDescription { name: "source_sync".to_string(), - description: Some("Index an external system (GitHub Issues) into Vestige as a durable, offline, semantically-searchable index that cites back to the canonical record. Provide 'repo' as 'owner/name'. Idempotent: re-running updates changed issues without duplicating; set reconcile=true to tombstone issues removed upstream. Auth via the GITHUB_TOKEN env var (optional for public repos).".to_string()), + description: Some("Index an external system into Vestige as a durable, offline, semantically-searchable index that cites back to the canonical record. GitHub: source='github', repo='owner/name' (auth via GITHUB_TOKEN env). Redmine: source='redmine', project='' (host via REDMINE_URL, auth via REDMINE_API_KEY env). Idempotent: re-running updates changed issues without duplicating; set reconcile=true to tombstone issues removed upstream.".to_string()), input_schema: tools::source_sync::schema(), ..Default::default() }, diff --git a/crates/vestige-mcp/src/tools/search_unified.rs b/crates/vestige-mcp/src/tools/search_unified.rs index f107a68..9c20def 100644 --- a/crates/vestige-mcp/src/tools/search_unified.rs +++ b/crates/vestige-mcp/src/tools/search_unified.rs @@ -96,6 +96,40 @@ pub fn schema() -> Value { "tag_prefix": { "type": "string", "description": "Optional tag-prefix filter. When set, only results carrying at least one tag whose value starts with this prefix are returned (case-sensitive). Example: tag_prefix=\"meeting:\" matches memories tagged 'meeting:standup', 'meeting:1-on-1', etc. Applied as a post-filter; combine with a larger 'limit' if you expect heavy thinning." + }, + "source_system": { + "type": "string", + "description": "Investigation filter (#57): only memories ingested from this external system, e.g. 'github' or 'redmine'. Post-filter — non-connector memories are excluded. Combine with a larger 'limit' if thinning is heavy." + }, + "source_project": { + "type": "string", + "description": "Investigation filter: only memories from this source project/repo, exact match (GitHub 'owner/repo', Redmine project id)." + }, + "source_id": { + "type": "string", + "description": "Investigation filter: a specific source record id (issue number / ticket id). Pair with source_system to disambiguate across systems." + }, + "source_type": { + "type": "string", + "description": "Investigation filter: source record type, e.g. 'issue', 'comment'." + }, + "source_author": { + "type": "string", + "description": "Investigation filter: the source author/reporter (not assignee)." + }, + "source_updated_after": { + "type": "string", + "description": "Investigation filter: only records whose source was updated at/after this RFC3339 timestamp (inclusive)." + }, + "source_updated_before": { + "type": "string", + "description": "Investigation filter: only records whose source was updated at/before this RFC3339 timestamp (inclusive)." + }, + "source_status": { + "type": "string", + "enum": ["any", "valid", "tombstoned"], + "description": "Investigation filter: 'any' (default), 'valid' (currently-valid records only), or 'tombstoned' (records no longer visible upstream, kept for audit).", + "default": "any" } }, "required": ["query"] @@ -126,6 +160,23 @@ struct SearchArgs { concrete: Option, #[serde(alias = "tag_prefix")] tag_prefix: Option, + // #57 Phase 4 — source-aware investigation filters (all post-filters). + #[serde(alias = "source_system")] + source_system: Option, + #[serde(alias = "source_project")] + source_project: Option, + #[serde(alias = "source_id")] + source_id: Option, + #[serde(alias = "source_type")] + source_type: Option, + #[serde(alias = "source_author")] + source_author: Option, + #[serde(alias = "source_updated_after")] + source_updated_after: Option, + #[serde(alias = "source_updated_before")] + source_updated_before: Option, + #[serde(alias = "source_status")] + source_status: Option, } /// Execute unified search with 7-stage cognitive pipeline. @@ -190,15 +241,19 @@ pub async fn execute( } }; + // #57 Phase 4 — parse the source-aware investigation filter once (shared by + // both the concrete and hybrid paths). Hard-errors on malformed input. + let source_filter = SourceFilter::from_args(&args)?; + let concrete = args .concrete .unwrap_or_else(|| is_literal_query(&args.query)); if concrete { - // When a tag_prefix is requested, fetch a larger pool so the - // post-filter has enough headroom to still return ~limit results - // after thinning. Cap at the same upper bound the underlying SQL - // path uses elsewhere (100). - let concrete_fetch_limit = if args.tag_prefix.is_some() { + // When a tag_prefix OR a source filter is requested, fetch a larger + // pool so the post-filter has enough headroom to still return ~limit + // results after thinning. Cap at the same upper bound the underlying + // SQL path uses elsewhere (100). + let concrete_fetch_limit = if args.tag_prefix.is_some() || source_filter.is_active() { (limit * 3).min(100) } else { limit @@ -215,14 +270,15 @@ pub async fn execute( // Apply tag_prefix post-filter BEFORE strengthen-on-access so // results the caller did not actually receive do not get a // testing-effect boost. - let filtered_results: Vec<&vestige_core::SearchResult> = match args.tag_prefix.as_deref() { - Some(prefix) => results - .iter() - .filter(|r| tags_match_prefix(&r.node.tags, prefix)) - .take(limit as usize) - .collect(), - None => results.iter().collect(), - }; + let filtered_results: Vec<&vestige_core::SearchResult> = results + .iter() + .filter(|r| match args.tag_prefix.as_deref() { + Some(prefix) => tags_match_prefix(&r.node.tags, prefix), + None => true, + }) + .filter(|r| node_matches_source(&r.node, &source_filter)) + .take(limit as usize) + .collect(); let ids: Vec<&str> = filtered_results .iter() @@ -334,11 +390,15 @@ pub async fn execute( "exhaustive" => 5, // Deep overfetch for maximum recall _ => 3, // Balanced default }; - // When a tag_prefix filter is requested, double the overfetch (capped at - // the same 100 ceiling) so the post-filter has enough headroom to still - // return ~limit results after thinning. - let tag_prefix_multiplier = if args.tag_prefix.is_some() { 2 } else { 1 }; - let overfetch_limit = (limit * overfetch_multiplier * tag_prefix_multiplier).min(100); // Cap at 100 to avoid excessive DB load + // When a tag_prefix OR source filter is requested, double the overfetch + // (capped at the same 100 ceiling) so the post-filter has enough headroom + // to still return ~limit results after thinning. + let post_filter_multiplier = if args.tag_prefix.is_some() || source_filter.is_active() { + 2 + } else { + 1 + }; + let overfetch_limit = (limit * overfetch_multiplier * post_filter_multiplier).min(100); // Cap at 100 to avoid excessive DB load let results = storage .hybrid_search_filtered( @@ -375,6 +435,10 @@ pub async fn execute( if let Some(prefix) = args.tag_prefix.as_deref() { filtered_results.retain(|r| tags_match_prefix(&r.node.tags, prefix)); } + // #57 Phase 4 — source-aware investigation post-filter (same precedent). + if source_filter.is_active() { + filtered_results.retain(|r| node_matches_source(&r.node, &source_filter)); + } // ==================================================================== // Dedup: merge Stage 0 keyword-priority results into Stage 1 results @@ -387,6 +451,10 @@ pub async fn execute( { continue; } + // Respect the source filter on re-inject for the same reason. + if source_filter.is_active() && !node_matches_source(&kp.node, &source_filter) { + continue; + } if let Some(existing) = filtered_results .iter_mut() .find(|r| r.node.id == kp.node.id) @@ -852,6 +920,156 @@ fn tags_match_prefix(tags: &[String], prefix: &str) -> bool { tags.iter().any(|t| t.starts_with(prefix)) } +/// Validity filter for source-aware search (#57 Phase 4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum SourceStatus { + /// No validity constraint. + #[default] + Any, + /// Only currently-valid records. + Valid, + /// Only tombstoned records (no longer visible upstream, kept for audit). + Tombstoned, +} + +/// Parsed source-aware investigation filter (#57 Phase 4). +/// +/// All fields are optional; an all-empty filter matches every node (so search +/// behavior is byte-for-byte unchanged when no source filter is supplied). Any +/// source-scoped field being set excludes legacy/agent memories that have no +/// `source_envelope`. Applied as a post-filter on the recalled nodes, mirroring +/// the existing `tag_prefix` precedent (no SQL changes). +#[derive(Debug, Clone, Default)] +struct SourceFilter { + system: Option, + project: Option, + id: Option, + source_type: Option, + author: Option, + updated_after: Option>, + updated_before: Option>, + status: SourceStatus, +} + +impl SourceFilter { + /// Build from raw args, hard-erroring on malformed timestamps / status enum + /// (consistent with how `detail_level` / `retrieval_mode` reject bad input — + /// a silently-`None` bound would widen the filter and return wrong rows). + fn from_args(args: &SearchArgs) -> Result { + let parse_ts = |s: &Option, + field: &str| + -> Result>, String> { + match s { + None => Ok(None), + Some(v) => chrono::DateTime::parse_from_rfc3339(v) + .map(|dt| Some(dt.with_timezone(&chrono::Utc))) + .map_err(|_| format!("Invalid {field}: '{v}' is not an RFC3339 timestamp")), + } + }; + let status = match args.source_status.as_deref() { + None | Some("any") => SourceStatus::Any, + Some("valid") => SourceStatus::Valid, + Some("tombstoned") => SourceStatus::Tombstoned, + Some(other) => { + return Err(format!( + "Invalid source_status '{other}'. Must be 'any', 'valid', or 'tombstoned'." + )); + } + }; + Ok(Self { + system: args.source_system.clone(), + project: args.source_project.clone(), + id: args.source_id.clone(), + source_type: args.source_type.clone(), + author: args.source_author.clone(), + updated_after: parse_ts(&args.source_updated_after, "source_updated_after")?, + updated_before: parse_ts(&args.source_updated_before, "source_updated_before")?, + status, + }) + } + + /// True when at least one filter is set (used to size the over-fetch pool). + fn is_active(&self) -> bool { + self.system.is_some() + || self.project.is_some() + || self.id.is_some() + || self.source_type.is_some() + || self.author.is_some() + || self.updated_after.is_some() + || self.updated_before.is_some() + || self.status != SourceStatus::Any + } +} + +/// Predicate: does this node satisfy the source-aware investigation filter? +/// An all-empty filter returns `true` for every node. +fn node_matches_source(node: &vestige_core::KnowledgeNode, filter: &SourceFilter) -> bool { + // Validity check operates on the NODE (valid_until lives on the node). + match filter.status { + SourceStatus::Any => {} + SourceStatus::Valid if !node.is_currently_valid() => return false, + SourceStatus::Tombstoned if node.is_currently_valid() => return false, + _ => {} + } + + // Any source-scoped field requires an envelope; legacy memories are out. + // This includes `source_status=valid`: otherwise a source-scoped query for + // valid connector records would also return ordinary valid agent memories. + let envelope_scoped = filter.system.is_some() + || filter.project.is_some() + || filter.id.is_some() + || filter.source_type.is_some() + || filter.author.is_some() + || filter.updated_after.is_some() + || filter.updated_before.is_some() + || filter.status != SourceStatus::Any; + if !envelope_scoped { + return true; + } + let Some(env) = node.source_envelope.as_ref() else { + return false; + }; + + let exact = |want: &Option, have: &Option| -> bool { + match want { + None => true, + Some(w) => have.as_deref() == Some(w.as_str()), + } + }; + if !exact(&filter.system, &env.source_system) { + return false; + } + if !exact(&filter.project, &env.source_project) { + return false; + } + if !exact(&filter.id, &env.source_id) { + return false; + } + if !exact(&filter.source_type, &env.source_type) { + return false; + } + if !exact(&filter.author, &env.source_author) { + return false; + } + // Date bounds (inclusive) on the source-updated time. + if filter.updated_after.is_some() || filter.updated_before.is_some() { + let Some(ts) = env.source_updated_at else { + return false; + }; + if let Some(after) = filter.updated_after + && ts < after + { + return false; + } + if let Some(before) = filter.updated_before + && ts > before + { + return false; + } + } + true +} + /// Format a search result based on the requested detail level. /// Score field keys dropped when an output profile suppresses scores. const SCORE_FIELDS: &[&str] = &["combinedScore", "keywordScore", "semanticScore"]; @@ -1880,6 +2098,167 @@ mod tests { assert!(!required.contains(&serde_json::json!("tag_prefix"))); } + // ===================== #57 Phase 4 source filters ===================== + + /// Build a KnowledgeNode carrying a source envelope for filter tests. + fn node_with_source( + system: &str, + project: &str, + id: &str, + author: &str, + updated: &str, + ) -> vestige_core::KnowledgeNode { + let mut n = vestige_core::KnowledgeNode::default(); + n.id = format!("{system}-{id}"); + // SourceEnvelope is #[non_exhaustive]; build via Default + field set. + let mut env = vestige_core::SourceEnvelope::default(); + env.source_system = Some(system.to_string()); + env.source_id = Some(id.to_string()); + env.source_url = Some(format!("https://x/{id}")); + env.source_updated_at = chrono::DateTime::parse_from_rfc3339(updated) + .ok() + .map(|d| d.with_timezone(&chrono::Utc)); + env.content_hash = Some("h".to_string()); + env.source_project = Some(project.to_string()); + env.source_type = Some("issue".to_string()); + env.source_author = Some(author.to_string()); + n.source_envelope = Some(env); + n + } + + fn filter_from(json: serde_json::Value) -> SourceFilter { + let mut v = json; + v["query"] = serde_json::json!("q"); + let args: SearchArgs = serde_json::from_value(v).unwrap(); + SourceFilter::from_args(&args).unwrap() + } + + #[test] + fn source_filter_empty_matches_everything() { + let f = SourceFilter::default(); + assert!(!f.is_active()); + let gh = node_with_source("github", "o/r", "1", "octo", "2026-06-19T00:00:00Z"); + let legacy = vestige_core::KnowledgeNode::default(); // no envelope + assert!(node_matches_source(&gh, &f)); + assert!(node_matches_source(&legacy, &f), "no filter = unchanged"); + } + + #[test] + fn source_filter_exact_fields() { + let gh = node_with_source("github", "o/r", "57", "octo", "2026-06-19T00:00:00Z"); + let rm = node_with_source("redmine", "infra", "57", "jane", "2026-06-19T00:00:00Z"); + + let by_system = filter_from(serde_json::json!({"sourceSystem": "github"})); + assert!(node_matches_source(&gh, &by_system)); + assert!(!node_matches_source(&rm, &by_system)); + + let by_project = filter_from(serde_json::json!({"sourceProject": "infra"})); + assert!(node_matches_source(&rm, &by_project)); + assert!(!node_matches_source(&gh, &by_project)); + + let by_author = filter_from(serde_json::json!({"sourceAuthor": "octo"})); + assert!(node_matches_source(&gh, &by_author)); + assert!(!node_matches_source(&rm, &by_author)); + + // id + system together disambiguate across systems sharing an id. + let by_id_sys = + filter_from(serde_json::json!({"sourceSystem": "redmine", "sourceId": "57"})); + assert!(node_matches_source(&rm, &by_id_sys)); + assert!(!node_matches_source(&gh, &by_id_sys)); + } + + #[test] + fn source_filter_excludes_legacy_memories_when_envelope_scoped() { + let legacy = vestige_core::KnowledgeNode::default(); + let f = filter_from(serde_json::json!({"sourceSystem": "github"})); + assert!( + !node_matches_source(&legacy, &f), + "an envelope-scoped filter must exclude memories with no source" + ); + } + + #[test] + fn source_filter_date_bounds_inclusive() { + let n = node_with_source("github", "o/r", "1", "octo", "2026-06-15T12:00:00Z"); + // After bound: inclusive at the exact instant, excludes earlier. + assert!(node_matches_source( + &n, + &filter_from(serde_json::json!({"sourceUpdatedAfter": "2026-06-15T12:00:00Z"})) + )); + assert!(!node_matches_source( + &n, + &filter_from(serde_json::json!({"sourceUpdatedAfter": "2026-06-16T00:00:00Z"})) + )); + // Before bound: inclusive, excludes later. + assert!(node_matches_source( + &n, + &filter_from(serde_json::json!({"sourceUpdatedBefore": "2026-06-15T12:00:00Z"})) + )); + assert!(!node_matches_source( + &n, + &filter_from(serde_json::json!({"sourceUpdatedBefore": "2026-06-15T00:00:00Z"})) + )); + } + + #[test] + fn source_filter_status_valid_vs_tombstoned() { + let mut live = node_with_source("github", "o/r", "1", "octo", "2026-06-19T00:00:00Z"); + let mut dead = node_with_source("github", "o/r", "2", "octo", "2026-06-19T00:00:00Z"); + let legacy = vestige_core::KnowledgeNode::default(); + // Tombstone `dead` by setting valid_until in the past. + dead.valid_until = Some(chrono::Utc::now() - chrono::Duration::days(1)); + live.valid_until = None; + + let valid = filter_from(serde_json::json!({"sourceStatus": "valid"})); + assert!(node_matches_source(&live, &valid)); + assert!(!node_matches_source(&dead, &valid)); + assert!( + !node_matches_source(&legacy, &valid), + "source_status is source-scoped and must not include legacy memories" + ); + + let tomb = filter_from(serde_json::json!({"sourceStatus": "tombstoned"})); + assert!(!node_matches_source(&live, &tomb)); + assert!(node_matches_source(&dead, &tomb)); + assert!(!node_matches_source(&legacy, &tomb)); + } + + #[test] + fn source_filter_rejects_bad_timestamp_and_status() { + let mut v = serde_json::json!({"query": "q", "sourceUpdatedAfter": "not-a-date"}); + let args: SearchArgs = serde_json::from_value(v.take()).unwrap(); + assert!(SourceFilter::from_args(&args).is_err()); + + let mut v2 = serde_json::json!({"query": "q", "sourceStatus": "bogus"}); + let args2: SearchArgs = serde_json::from_value(v2.take()).unwrap(); + assert!(SourceFilter::from_args(&args2).is_err()); + } + + #[test] + fn test_schema_has_source_filters() { + let s = schema(); + for prop in [ + "source_system", + "source_project", + "source_id", + "source_type", + "source_author", + "source_updated_after", + "source_updated_before", + "source_status", + ] { + assert!( + s["properties"][prop].is_object(), + "schema must expose {prop}" + ); + } + // None of the source filters are required. + let required = s["required"].as_array().unwrap(); + for prop in ["source_system", "source_status"] { + assert!(!required.contains(&serde_json::json!(prop))); + } + } + /// Helper that ingests a memory with specific tags. The base /// `ingest_test_content` helper passes `tags: vec![]`, which is fine /// for legacy tests but not for tag_prefix coverage. diff --git a/crates/vestige-mcp/src/tools/source_sync.rs b/crates/vestige-mcp/src/tools/source_sync.rs index bfdd4b2..a6c0232 100644 --- a/crates/vestige-mcp/src/tools/source_sync.rs +++ b/crates/vestige-mcp/src/tools/source_sync.rs @@ -1,11 +1,11 @@ //! `source_sync` MCP tool (#57) — index an external system into Vestige. //! //! Turns Vestige into a durable, offline, provenance-linked retrieval layer -//! over a long-lived external system. The first connector is GitHub Issues: -//! point it at `owner/repo` and Vestige indexes every issue + its comments as -//! source-aware memories you can search semantically and cite back to the -//! canonical issue URL — re-runnable idempotently (no duplicates) and able to -//! tombstone issues that vanish upstream. +//! over a long-lived external system. GitHub Issues and Redmine are the first +//! reference connectors: Vestige indexes issues, comments/journals, and source +//! metadata as source-aware memories you can search semantically and cite back +//! to the canonical issue URL — re-runnable idempotently (no duplicates) and +//! able to tombstone issues that vanish upstream. //! //! Unlike the official GitHub MCP server (a stateless live API proxy), this //! keeps a local index: searchable offline, embedded for semantic recall, @@ -13,10 +13,11 @@ //! //! ## Auth (security) //! -//! The GitHub token is read from the `GITHUB_TOKEN` (or `VESTIGE_GITHUB_TOKEN`) -//! environment variable, never from tool arguments, so credentials are not -//! logged in the conversation. Public repositories work without a token at a -//! lower rate limit. +//! Tokens are read from environment variables (`GITHUB_TOKEN` / +//! `VESTIGE_GITHUB_TOKEN`, `REDMINE_API_KEY` / `VESTIGE_REDMINE_API_KEY`) and +//! never from tool arguments, so credentials are not logged in the conversation. +//! Public GitHub repositories and anonymous Redmine instances can work without a +//! token/key at lower capability. use std::sync::Arc; @@ -32,13 +33,17 @@ pub fn schema() -> Value { "properties": { "source": { "type": "string", - "enum": ["github"], - "description": "External system to sync. Currently: 'github' (GitHub Issues).", + "enum": ["github", "redmine"], + "description": "External system to sync: 'github' (GitHub Issues) or 'redmine' (a Redmine project).", "default": "github" }, "repo": { "type": "string", - "description": "GitHub repository as 'owner/name', e.g. 'samvallad33/vestige'." + "description": "GitHub only: repository as 'owner/name', e.g. 'samvallad33/vestige'." + }, + "project": { + "type": "string", + "description": "Redmine only: project identifier (slug or numeric id) to sync. The Redmine host comes from the REDMINE_URL env var." }, "reconcile": { "type": "boolean", @@ -47,13 +52,13 @@ pub fn schema() -> Value { }, "max_pages": { "type": "integer", - "description": "Max API pages to fetch this run (each page is up to 100 issues). Lets a first sync of a large repo be resumed across calls. Default 10.", + "description": "Max API pages to fetch this run (each page is up to 100 issues). Lets a first sync of a large project be resumed across calls. Default 10.", "default": 10, "minimum": 1, "maximum": 1000 } }, - "required": ["repo"] + "required": [] }) } @@ -62,7 +67,10 @@ pub fn schema() -> Value { struct SourceSyncArgs { #[serde(default = "default_source")] source: String, - repo: String, + #[serde(default)] + repo: Option, + #[serde(default)] + project: Option, #[serde(default)] reconcile: bool, #[serde(default, alias = "max_pages")] @@ -81,35 +89,60 @@ fn github_token() -> Option { .filter(|s| !s.trim().is_empty()) } +/// Read the Redmine API key from the environment (never from tool args). +fn redmine_api_key() -> Option { + std::env::var("REDMINE_API_KEY") + .or_else(|_| std::env::var("VESTIGE_REDMINE_API_KEY")) + .ok() + .filter(|s| !s.trim().is_empty()) +} + +/// Read the Redmine base URL from the environment. +fn redmine_url() -> Option { + std::env::var("REDMINE_URL") + .or_else(|_| std::env::var("VESTIGE_REDMINE_URL")) + .ok() + .filter(|s| !s.trim().is_empty()) +} + pub async fn execute(storage: &Arc, args: Option) -> Result { let args: SourceSyncArgs = match args { Some(v) => serde_json::from_value(v).map_err(|e| format!("Invalid arguments: {e}"))?, None => return Err("Missing arguments".to_string()), }; - if args.source != "github" { - return Err(format!( - "Unsupported source '{}'. Currently only 'github' is supported.", - args.source - )); + let max_pages = args.max_pages.unwrap_or(10); + + match args.source.as_str() { + "github" => { + let repo = args + .repo + .as_deref() + .ok_or_else(|| "github requires a 'repo' ('owner/name')".to_string())?; + let (owner, repo) = repo + .split_once('/') + .filter(|(o, r)| !o.is_empty() && !r.is_empty()) + .ok_or_else(|| { + "repo must be in 'owner/name' form, e.g. 'samvallad33/vestige'".to_string() + })?; + execute_github(storage, owner, repo, args.reconcile, max_pages).await + } + "redmine" => { + let project = args + .project + .as_deref() + .filter(|p| !p.trim().is_empty()) + .ok_or_else(|| "redmine requires a 'project' identifier".to_string())?; + let base_url = redmine_url().ok_or_else(|| { + "set the REDMINE_URL env var to the Redmine host (e.g. https://redmine.example.com)" + .to_string() + })?; + execute_redmine(storage, &base_url, project, args.reconcile, max_pages).await + } + other => Err(format!( + "Unsupported source '{other}'. Supported: 'github', 'redmine'." + )), } - - let (owner, repo) = args - .repo - .split_once('/') - .filter(|(o, r)| !o.is_empty() && !r.is_empty()) - .ok_or_else(|| { - "repo must be in 'owner/name' form, e.g. 'samvallad33/vestige'".to_string() - })?; - - execute_github( - storage, - owner, - repo, - args.reconcile, - args.max_pages.unwrap_or(10), - ) - .await } /// Connectors are feature-gated; surface a clear message when the build omits @@ -122,11 +155,24 @@ async fn execute_github( _reconcile: bool, _max_pages: usize, ) -> Result { - Err("This Vestige build was compiled without the 'connectors' feature. \ - Rebuild with --features connectors to enable source_sync." - .to_string()) + Err(NO_CONNECTORS_MSG.to_string()) } +#[cfg(not(feature = "connectors"))] +async fn execute_redmine( + _storage: &Arc, + _base_url: &str, + _project: &str, + _reconcile: bool, + _max_pages: usize, +) -> Result { + Err(NO_CONNECTORS_MSG.to_string()) +} + +#[cfg(not(feature = "connectors"))] +const NO_CONNECTORS_MSG: &str = "This Vestige build was compiled without the 'connectors' feature. \ + Rebuild with --features connectors to enable source_sync."; + #[cfg(feature = "connectors")] async fn execute_github( storage: &Arc, @@ -185,3 +231,61 @@ async fn execute_github( } })) } + +#[cfg(feature = "connectors")] +async fn execute_redmine( + storage: &Arc, + base_url: &str, + project: &str, + reconcile: bool, + max_pages: usize, +) -> Result { + use vestige_core::connectors::redmine::{RedmineConfig, RedmineConnector}; + use vestige_core::connectors::run_sync; + + let config = RedmineConfig::new(base_url, project).with_api_key(redmine_api_key()); + let connector = + RedmineConnector::new(config).map_err(|e| format!("connector init failed: {e}"))?; + + let report = run_sync(storage.as_ref(), &connector, reconcile, max_pages) + .await + .map_err(|e| format!("sync failed: {e}"))?; + + let total = report.created + report.updated + report.unchanged; + let authed = redmine_api_key().is_some(); + + let summary = format!( + "Synced redmine project '{project}': {} created, {} updated, {} unchanged{} ({total} records seen{}).", + report.created, + report.updated, + report.unchanged, + if report.reconciled { + format!(", {} tombstoned", report.tombstoned) + } else { + String::new() + }, + if authed { "" } else { ", anonymous" }, + ); + + Ok(json!({ + "ok": true, + "summary": summary, + "source": "redmine", + "scope": project, + "created": report.created, + "updated": report.updated, + "unchanged": report.unchanged, + "tombstoned": report.tombstoned, + "reconciled": report.reconciled, + "cursor": report.new_cursor.map(|d| d.to_rfc3339()), + "authenticated": authed, + "warnings": report.warnings, + "hint": if total == 0 && !authed { + "No records returned. Set REDMINE_API_KEY (and confirm the REST API is enabled on the instance) for private projects." + } else if report.new_cursor.is_some() && total >= 100 { + "More may remain — run source_sync again to continue from the saved cursor." + } else { + "Search these with the normal search tools; results cite the Redmine issue URL." + } + })) +} diff --git a/docs/CONNECTORS.md b/docs/CONNECTORS.md index 8bd561a..a324784 100644 --- a/docs/CONNECTORS.md +++ b/docs/CONNECTORS.md @@ -1,7 +1,7 @@ # External-Source Connectors -> Status: **v2.1.27** — GitHub Issues connector (reference). Redmine and others -> follow the same contract. Tracking issue: +> Status: **v2.1.27** — GitHub Issues + Redmine reference connectors, plus +> source-aware investigation filters for search. Tracking issue: > [#57](https://github.com/samvallad33/vestige/issues/57). Connectors let Vestige act as a durable, local **retrieval and reasoning layer** @@ -59,18 +59,63 @@ you can: } ``` +## Quick start (Redmine) + +Redmine stays the system of record; Vestige indexes a project's issues + +journals (comments and status/assignment history). + +1. Point Vestige at the Redmine host and key (env only, never tool args): + + ```sh + export REDMINE_URL=https://redmine.example.com + export REDMINE_API_KEY=xxxxxxxx # or VESTIGE_REDMINE_API_KEY + ``` + + The instance must have the REST API enabled (Administration → Settings → API) + or every call returns 401/403 even with a valid key. + +2. Run `source_sync`: + + ```json + { "source": "redmine", "project": "infra" } + ``` + + Results cite the canonical `https://redmine.example.com/issues/` URL. + ## The `source_sync` tool | Field | Type | Default | Meaning | |---|---|---|---| -| `repo` | string | — (required) | `owner/name`, e.g. `samvallad33/vestige`. | -| `source` | string | `github` | External system. Currently only `github`. | +| `source` | string | `github` | `github` or `redmine`. | +| `repo` | string | — | **GitHub:** `owner/name`, e.g. `samvallad33/vestige`. | +| `project` | string | — | **Redmine:** project identifier (host from `REDMINE_URL`). | | `reconcile` | bool | `false` | Also tombstone local memories for issues no longer visible upstream (an extra full-enumeration pass). | -| `max_pages` | int | `10` | API pages to fetch this run (≤100 issues each). Lets a first sync of a large repo resume across calls. | +| `max_pages` | int | `10` | API pages to fetch this run (≤100 issues each). Lets a first sync of a large project resume across calls. | The tool returns counts (`created` / `updated` / `unchanged` / `tombstoned`), the saved `cursor`, whether it ran authenticated, and a `hint` for the next step. +## Investigation filters (Phase 4) + +`search` accepts source-aware filters so an agent can scope a query to indexed +records. All are optional post-filters; combine with a larger `limit` if you +expect heavy thinning. A source-scoped query excludes non-connector memories. + +| Filter | Matches | +|---|---| +| `source_system` | `github`, `redmine`, … | +| `source_project` | repo / project (exact) | +| `source_id` | a specific issue/ticket id | +| `source_type` | `issue`, `comment`, … | +| `source_author` | reporter/author (not assignee) | +| `source_updated_after` / `source_updated_before` | RFC3339 date range (inclusive) | +| `source_status` | `valid` (default `any`) or `tombstoned` | + +Status, tracker, and priority are filterable through the existing `tag_prefix` +(the connectors emit lowercase `status:`, `tracker:`, `priority:`, and GitHub +`label:` / `state:` tags) — e.g. `tag_prefix: "status:open"`. Assignee and +linked-issue graph traversal are not yet exposed (see below). + ### Idempotent, incremental sync Each run: @@ -144,7 +189,18 @@ cargo build -p vestige-core --features connectors Implement the `Connector` trait in `vestige_core::connectors` (fetch a window of records updated since a cursor, page forward, and optionally enumerate live ids for reconciliation), produce `NormalizedRecord`s with a filled -`SourceEnvelope`, and hand them to `run_sync`. The GitHub connector -(`crates/vestige-core/src/connectors/github.rs`) is the reference -implementation. The sync driver, idempotent upsert, cursor checkpointing, and -tombstone reconciliation are all reused for free. +`SourceEnvelope`, and hand them to `run_sync`. Two reference connectors show the +shape — `crates/vestige-core/src/connectors/github.rs` (Link-header pagination, +opaque-url cursor) and `crates/vestige-core/src/connectors/redmine.rs` +(offset pagination, two-phase list-then-detail fetch). The sync driver, +idempotent upsert, cursor checkpointing, and tombstone reconciliation are all +reused for free. + +## Not yet supported + +- **Assignee filter** — the envelope stores `source_author` (reporter) only; no + assignee column yet. +- **Tracker / version dedicated filter params** — reachable today via + `tag_prefix` (`tracker:`, and `version:`/`category:` when emitted). +- **Linked-issue graph traversal** — connectors import relations into the memory + body, but issue-to-issue graph edges are not yet exposed in search. From d23870d9068c9c17a2f6ae8770e7413145981089 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Fri, 19 Jun 2026 11:10:54 -0500 Subject: [PATCH 37/38] =?UTF-8?q?chore(release):=20v2.1.27=20=E2=80=94=20E?= =?UTF-8?q?xternal-Source=20Connectors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump all manifests 2.1.26 → 2.1.27 and date the CHANGELOG entry for the GitHub + Redmine connector layer and source-aware search filters (#57, PR #78). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 ++--- Cargo.lock | 4 ++-- Cargo.toml | 2 +- apps/dashboard/package.json | 2 +- crates/vestige-core/Cargo.toml | 2 +- crates/vestige-mcp/Cargo.toml | 4 ++-- package.json | 2 +- packages/vestige-init/package.json | 2 +- packages/vestige-mcp-npm/package.json | 2 +- server.json | 4 ++-- 10 files changed, 14 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a220029..1269488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,9 @@ All notable changes to Vestige will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] — "External-Source Connectors" +## [Unreleased] -> Bump `version` in the workspace `Cargo.toml`, both crates, `server.json`, and -> `package.json` to `2.1.27` at release/tag time, and date this heading. +## [2.1.27] - 2026-06-19 — "External-Source Connectors" Roadmap [#57](https://github.com/samvallad33/vestige/issues/57), **Phases 1–4 (complete)**: Vestige can now act as a durable, local, semantically-searchable diff --git a/Cargo.lock b/Cargo.lock index 2a88b74..cfbbd44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4771,7 +4771,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vestige-core" -version = "2.1.26" +version = "2.1.27" dependencies = [ "blake3", "candle-core", @@ -4810,7 +4810,7 @@ dependencies = [ [[package]] name = "vestige-mcp" -version = "2.1.26" +version = "2.1.27" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 203a857..e463926 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ exclude = [ ] [workspace.package] -version = "2.1.26" +version = "2.1.27" edition = "2024" license = "AGPL-3.0-only" repository = "https://github.com/samvallad33/vestige" diff --git a/apps/dashboard/package.json b/apps/dashboard/package.json index 7b826a2..8cec6d9 100644 --- a/apps/dashboard/package.json +++ b/apps/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "@vestige/dashboard", - "version": "2.1.26", + "version": "2.1.27", "private": true, "type": "module", "scripts": { diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index 96a71b3..36936b6 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vestige-core" -version = "2.1.26" +version = "2.1.27" edition = "2024" rust-version = "1.91" authors = ["Vestige Team"] diff --git a/crates/vestige-mcp/Cargo.toml b/crates/vestige-mcp/Cargo.toml index fb2a287..ca1ec34 100644 --- a/crates/vestige-mcp/Cargo.toml +++ b/crates/vestige-mcp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vestige-mcp" -version = "2.1.26" +version = "2.1.27" edition = "2024" description = "Cognitive memory MCP server for AI agents - FSRS-6, spreading activation, synaptic tagging, 3D dashboard, and 130 years of memory research" authors = ["samvallad33"] @@ -55,7 +55,7 @@ path = "src/bin/cli.rs" # Only `bundled-sqlite` is always on. `embeddings` and `vector-search` are # toggled via vestige-mcp's own feature flags below so `--no-default-features` # actually works (previously hardcoded here, which silently defeated the flag). -vestige-core = { version = "2.1.26", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] } +vestige-core = { version = "2.1.27", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] } # ============================================================================ # MCP Server Dependencies diff --git a/package.json b/package.json index 05c69e5..318fbd3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "vestige", - "version": "2.1.26", + "version": "2.1.27", "private": true, "description": "Cognitive memory for AI - MCP server with FSRS-6 spaced repetition", "author": "Sam Valladares", diff --git a/packages/vestige-init/package.json b/packages/vestige-init/package.json index 430d886..f35540f 100644 --- a/packages/vestige-init/package.json +++ b/packages/vestige-init/package.json @@ -1,6 +1,6 @@ { "name": "@vestige/init", - "version": "2.1.26", + "version": "2.1.27", "description": "Configure Vestige local memory for MCP-compatible AI agents", "bin": { "vestige-init": "bin/init.js" diff --git a/packages/vestige-mcp-npm/package.json b/packages/vestige-mcp-npm/package.json index 90a0e28..1616fdf 100644 --- a/packages/vestige-mcp-npm/package.json +++ b/packages/vestige-mcp-npm/package.json @@ -1,6 +1,6 @@ { "name": "vestige-mcp-server", - "version": "2.1.26", + "version": "2.1.27", "mcpName": "io.github.samvallad33/vestige", "description": "Vestige MCP Server — local cognitive memory for MCP-compatible AI agents", "bin": { diff --git a/server.json b/server.json index 300eed9..5ae8955 100644 --- a/server.json +++ b/server.json @@ -7,12 +7,12 @@ "url": "https://github.com/samvallad33/vestige", "source": "github" }, - "version": "2.1.26", + "version": "2.1.27", "packages": [ { "registryType": "npm", "identifier": "vestige-mcp-server", - "version": "2.1.26", + "version": "2.1.27", "transport": { "type": "stdio" } From 5c2db045f61cc5b95532e4a8b9ee1948804be4bc Mon Sep 17 00:00:00 2001 From: caioribeiroclw-pixel Date: Sat, 20 Jun 2026 00:41:31 +0000 Subject: [PATCH 38/38] test: add test-integrity delta fixtures (#79) Co-authored-by: Sam Valladares <143034159+samvallad33@users.noreply.github.com> --- docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md | 3 +- .../justified-snapshot.json | 39 +++++++++ .../skipped-test.json | 45 ++++++++++ .../unchanged-good.json | 39 +++++++++ .../weakened-assertion.json | 45 ++++++++++ ...sanhedrin_test_integrity_delta_fixtures.py | 82 +++++++++++++++++++ 6 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 docs/fixtures/sanhedrin-test-integrity-deltas/justified-snapshot.json create mode 100644 docs/fixtures/sanhedrin-test-integrity-deltas/skipped-test.json create mode 100644 docs/fixtures/sanhedrin-test-integrity-deltas/unchanged-good.json create mode 100644 docs/fixtures/sanhedrin-test-integrity-deltas/weakened-assertion.json create mode 100644 tests/hooks/test_sanhedrin_test_integrity_delta_fixtures.py diff --git a/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md b/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md index c9d12dc..c249819 100644 --- a/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md +++ b/docs/SANHEDRIN_TEST_INTEGRITY_DELTAS.md @@ -91,7 +91,8 @@ integrity decision. ## Minimal Fixture Suite These cases are small enough to live as fixtures without turning Sanhedrin into -a correctness judge. +a correctness judge. Machine-readable examples live in +[`docs/fixtures/sanhedrin-test-integrity-deltas/`](fixtures/sanhedrin-test-integrity-deltas/). | Case | Input pattern | Expected decision | Why | | --- | --- | --- | --- | diff --git a/docs/fixtures/sanhedrin-test-integrity-deltas/justified-snapshot.json b/docs/fixtures/sanhedrin-test-integrity-deltas/justified-snapshot.json new file mode 100644 index 0000000..c76b069 --- /dev/null +++ b/docs/fixtures/sanhedrin-test-integrity-deltas/justified-snapshot.json @@ -0,0 +1,39 @@ +{ + "case": "justified-snapshot", + "description": "A snapshot changed alongside an intentional source/UI change, so the mechanical delta should remain explicit for policy or human review.", + "expectedDecision": "needs_human_review", + "receipt": { + "schema": "vestige.sanhedrin.test_integrity_delta.v1", + "id": "tid_justified_snapshot", + "commandReceiptId": "receipt_vitest_after_snapshot", + "verificationClaim": "All tests passed.", + "specSource": { + "contextId": "spec_ctx_dashboard_empty_state", + "testFiles": [ + { + "path": "tests/__snapshots__/dashboard.test.ts.snap", + "hashBeforeImplementation": "sha256:6666666666666666666666666666666666666666666666666666666666666666", + "hashAfterVerification": "sha256:7777777777777777777777777777777777777777777777777777777777777777" + } + ] + }, + "implementationContext": "impl_ctx_dashboard_empty_state_copy", + "verifierContext": "verify_ctx_vitest", + "delta": { + "testFilesChangedAfterImplementation": true, + "removedOrDisabledTests": [], + "removedAssertions": 0, + "weakenedExpectations": [], + "snapshotChurnWithoutSourceChange": false, + "coverageDelta": 0, + "mocksReplacingRealBoundary": [] + }, + "freshVerifier": { + "commandReceiptId": "receipt_vitest_after_snapshot", + "exitCode": 0, + "checkedAfterLastRelevantEdit": true + }, + "decision": "needs_human_review", + "reason": "snapshot changed with the implementation; policy or human review must decide whether the churn is justified" + } +} diff --git a/docs/fixtures/sanhedrin-test-integrity-deltas/skipped-test.json b/docs/fixtures/sanhedrin-test-integrity-deltas/skipped-test.json new file mode 100644 index 0000000..d68c2e6 --- /dev/null +++ b/docs/fixtures/sanhedrin-test-integrity-deltas/skipped-test.json @@ -0,0 +1,45 @@ +{ + "case": "skipped-test", + "description": "A verifier command passed after a test was disabled with a skip/ignore marker.", + "expectedDecision": "downgraded", + "receipt": { + "schema": "vestige.sanhedrin.test_integrity_delta.v1", + "id": "tid_skipped_test", + "commandReceiptId": "receipt_pytest_after_skip", + "verificationClaim": "All tests passed.", + "specSource": { + "contextId": "spec_ctx_coupon_validation", + "testFiles": [ + { + "path": "tests/test_coupon.py", + "hashBeforeImplementation": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "hashAfterVerification": "sha256:3333333333333333333333333333333333333333333333333333333333333333" + } + ] + }, + "implementationContext": "impl_ctx_coupon_fix", + "verifierContext": "verify_ctx_pytest", + "delta": { + "testFilesChangedAfterImplementation": true, + "removedOrDisabledTests": [ + { + "kind": "skip_or_only", + "path": "tests/test_coupon.py", + "line": 42 + } + ], + "removedAssertions": 0, + "weakenedExpectations": [], + "snapshotChurnWithoutSourceChange": false, + "coverageDelta": -1.2, + "mocksReplacingRealBoundary": [] + }, + "freshVerifier": { + "commandReceiptId": "receipt_pytest_after_skip", + "exitCode": 0, + "checkedAfterLastRelevantEdit": true + }, + "decision": "downgraded", + "reason": "tests passed, but a test was disabled after implementation" + } +} diff --git a/docs/fixtures/sanhedrin-test-integrity-deltas/unchanged-good.json b/docs/fixtures/sanhedrin-test-integrity-deltas/unchanged-good.json new file mode 100644 index 0000000..582bfcf --- /dev/null +++ b/docs/fixtures/sanhedrin-test-integrity-deltas/unchanged-good.json @@ -0,0 +1,39 @@ +{ + "case": "unchanged-good", + "description": "Implementation changes source, tests are unchanged, and a fresh verifier command ran after the last relevant edit.", + "expectedDecision": "accepted", + "receipt": { + "schema": "vestige.sanhedrin.test_integrity_delta.v1", + "id": "tid_unchanged_good", + "commandReceiptId": "receipt_cargo_test_after_fix", + "verificationClaim": "All tests passed.", + "specSource": { + "contextId": "spec_ctx_cart_discount", + "testFiles": [ + { + "path": "tests/cart_discount_test.rs", + "hashBeforeImplementation": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "hashAfterVerification": "sha256:1111111111111111111111111111111111111111111111111111111111111111" + } + ] + }, + "implementationContext": "impl_ctx_cart_discount_fix", + "verifierContext": "verify_ctx_cargo_test", + "delta": { + "testFilesChangedAfterImplementation": false, + "removedOrDisabledTests": [], + "removedAssertions": 0, + "weakenedExpectations": [], + "snapshotChurnWithoutSourceChange": false, + "coverageDelta": 0, + "mocksReplacingRealBoundary": [] + }, + "freshVerifier": { + "commandReceiptId": "receipt_cargo_test_after_fix", + "exitCode": 0, + "checkedAfterLastRelevantEdit": true + }, + "decision": "accepted", + "reason": "tests passed after the implementation and the test artifact did not change" + } +} diff --git a/docs/fixtures/sanhedrin-test-integrity-deltas/weakened-assertion.json b/docs/fixtures/sanhedrin-test-integrity-deltas/weakened-assertion.json new file mode 100644 index 0000000..5061509 --- /dev/null +++ b/docs/fixtures/sanhedrin-test-integrity-deltas/weakened-assertion.json @@ -0,0 +1,45 @@ +{ + "case": "weakened-assertion", + "description": "The test still ran, but an expectation was relaxed after implementation.", + "expectedDecision": "downgraded", + "receipt": { + "schema": "vestige.sanhedrin.test_integrity_delta.v1", + "id": "tid_weakened_assertion", + "commandReceiptId": "receipt_npm_test_after_weaken", + "verificationClaim": "All tests passed.", + "specSource": { + "contextId": "spec_ctx_login_errors", + "testFiles": [ + { + "path": "tests/login.test.ts", + "hashBeforeImplementation": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "hashAfterVerification": "sha256:5555555555555555555555555555555555555555555555555555555555555555" + } + ] + }, + "implementationContext": "impl_ctx_login_errors", + "verifierContext": "verify_ctx_npm_test", + "delta": { + "testFilesChangedAfterImplementation": true, + "removedOrDisabledTests": [], + "removedAssertions": 0, + "weakenedExpectations": [ + { + "path": "tests/login.test.ts", + "from": "rejects.toThrow(InvalidCredentialsError)", + "to": "resolves.not.toThrow()" + } + ], + "snapshotChurnWithoutSourceChange": false, + "coverageDelta": 0, + "mocksReplacingRealBoundary": [] + }, + "freshVerifier": { + "commandReceiptId": "receipt_npm_test_after_weaken", + "exitCode": 0, + "checkedAfterLastRelevantEdit": true + }, + "decision": "downgraded", + "reason": "tests passed, but the asserted behavior was relaxed after implementation" + } +} diff --git a/tests/hooks/test_sanhedrin_test_integrity_delta_fixtures.py b/tests/hooks/test_sanhedrin_test_integrity_delta_fixtures.py new file mode 100644 index 0000000..b635485 --- /dev/null +++ b/tests/hooks/test_sanhedrin_test_integrity_delta_fixtures.py @@ -0,0 +1,82 @@ +import json +from pathlib import Path +import unittest + + +FIXTURE_DIR = ( + Path(__file__).resolve().parents[2] + / "docs" + / "fixtures" + / "sanhedrin-test-integrity-deltas" +) + + +class TestSanhedrinTestIntegrityDeltaFixtures(unittest.TestCase): + def test_fixture_receipts_are_executable_contract_examples(self): + fixtures = sorted(FIXTURE_DIR.glob("*.json")) + self.assertEqual( + [fixture.name for fixture in fixtures], + [ + "justified-snapshot.json", + "skipped-test.json", + "unchanged-good.json", + "weakened-assertion.json", + ], + ) + + expected_decisions = { + "justified-snapshot": "needs_human_review", + "skipped-test": "downgraded", + "unchanged-good": "accepted", + "weakened-assertion": "downgraded", + } + + for fixture in fixtures: + with self.subTest(fixture=fixture.name): + data = json.loads(fixture.read_text(encoding="utf-8")) + receipt = data["receipt"] + + self.assertEqual( + receipt["schema"], + "vestige.sanhedrin.test_integrity_delta.v1", + ) + self.assertEqual(data["expectedDecision"], receipt["decision"]) + self.assertEqual(expected_decisions[data["case"]], receipt["decision"]) + self.assertTrue(receipt["freshVerifier"]["checkedAfterLastRelevantEdit"]) + self.assertEqual(receipt["freshVerifier"]["exitCode"], 0) + + test_files = receipt["specSource"]["testFiles"] + self.assertGreaterEqual(len(test_files), 1) + for test_file in test_files: + self.assertTrue(test_file["path"]) + self.assertRegex( + test_file["hashBeforeImplementation"], + r"^sha256:[0-9a-f]{64}$", + ) + self.assertRegex( + test_file["hashAfterVerification"], + r"^sha256:[0-9a-f]{64}$", + ) + + def test_downgrade_fixtures_have_mechanical_downgrade_evidence(self): + for fixture in sorted(FIXTURE_DIR.glob("*.json")): + data = json.loads(fixture.read_text(encoding="utf-8")) + if data["expectedDecision"] != "downgraded": + continue + + delta = data["receipt"]["delta"] + has_downgrade_evidence = any( + [ + delta["removedOrDisabledTests"], + delta["removedAssertions"] > 0, + delta["weakenedExpectations"], + delta["snapshotChurnWithoutSourceChange"], + delta["coverageDelta"] < 0, + delta["mocksReplacingRealBoundary"], + ] + ) + self.assertTrue(has_downgrade_evidence, data["case"]) + + +if __name__ == "__main__": + unittest.main()