diff --git a/.github/workflows/guard-no-private-cloud.yml b/.github/workflows/guard-no-private-cloud.yml new file mode 100644 index 0000000..515e959 --- /dev/null +++ b/.github/workflows/guard-no-private-cloud.yml @@ -0,0 +1,25 @@ +name: Guard — No Private Cloud Code + +# Fails if private Vestige Cloud *service* code (billing, sync-key/namespace +# mapping, Lemon Squeezy webhooks, transactional email) ever lands in this +# public repo. The public cloud *client* is allowed and does not trip this. +on: + push: + branches: [main, feat/cloud-sync-mvp] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + guard: + name: No private cloud service code + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Scan for private cloud service markers + run: ./scripts/check-no-private-cloud.sh diff --git a/Cargo.lock b/Cargo.lock index cfbbd44..1d225e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "ahash" version = "0.8.12" @@ -143,6 +153,18 @@ dependencies = [ "syn", ] +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.2.17", + "password-hash", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -317,6 +339,15 @@ dependencies = [ "core2", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "blake3" version = "1.8.4" @@ -527,6 +558,30 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.44" @@ -568,6 +623,17 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + [[package]] name = "clap" version = "4.6.0" @@ -821,6 +887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1025,6 +1092,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1379,6 +1447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -2229,6 +2298,15 @@ dependencies = [ "libc", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "interpolate_name" version = "0.2.4" @@ -3012,6 +3090,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.3.3" @@ -3137,6 +3221,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.15" @@ -3223,6 +3318,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -3404,7 +3510,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -3435,7 +3541,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -3445,7 +3551,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -3627,6 +3742,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", @@ -4626,6 +4742,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -4773,8 +4899,10 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "vestige-core" version = "2.1.27" dependencies = [ + "argon2", "blake3", "candle-core", + "chacha20poly1305", "chrono", "criterion", "directories", diff --git a/crates/vestige-core/Cargo.toml b/crates/vestige-core/Cargo.toml index d95fc2d..6e642ee 100644 --- a/crates/vestige-core/Cargo.toml +++ b/crates/vestige-core/Cargo.toml @@ -63,6 +63,14 @@ qwen3-reranker = ["qwen3-embeddings"] # the default local-first build never links an HTTP client. connectors = ["dep:reqwest"] +# Hosted managed-sync backend (Vestige Cloud). Adds the HTTP `PortableSyncBackend` +# (`HttpPortableSyncBackend`) that pull-merge-pushes the portable archive to a +# hosted blob endpoint over HTTPS with a per-user sync key. Like `connectors`, +# this is the ONLY thing that links an HTTP client — the default local-first +# build stays network-free. Uses `reqwest`'s blocking client because the +# `PortableSyncBackend` trait methods are synchronous. +cloud-sync = ["dep:reqwest", "reqwest/blocking", "dep:chacha20poly1305", "dep:argon2"] + # Metal GPU acceleration on Apple Silicon (significantly faster inference) metal = ["fastembed/metal"] @@ -157,6 +165,15 @@ blake3 = "1" # `connectors` feature — the default local-first build does not link reqwest. reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } +# ============================================================================ +# OPTIONAL: Zero-knowledge client-side encryption for Vestige Cloud sync +# ============================================================================ +# XChaCha20-Poly1305 AEAD + Argon2id passphrase KDF. The archive is encrypted on +# the client before upload, so the hosted service only ever stores ciphertext. +# Behind the `cloud-sync` feature — the default local-first build links neither. +chacha20poly1305 = { version = "0.10", optional = true } +argon2 = { version = "0.5", optional = true } + [dev-dependencies] tempfile = "3" criterion = { version = "0.5", features = ["html_reports"] } diff --git a/crates/vestige-core/src/lib.rs b/crates/vestige-core/src/lib.rs index 667eab4..f6f12b4 100644 --- a/crates/vestige-core/src/lib.rs +++ b/crates/vestige-core/src/lib.rs @@ -198,6 +198,7 @@ pub use storage::{ PortableArchive, PortableImportMode, PortableImportReport, + PortableSyncReport, ReconcileReport, Result, SchedulingState, diff --git a/crates/vestige-core/src/storage/cloud_crypto.rs b/crates/vestige-core/src/storage/cloud_crypto.rs new file mode 100644 index 0000000..5d88045 --- /dev/null +++ b/crates/vestige-core/src/storage/cloud_crypto.rs @@ -0,0 +1,164 @@ +//! Zero-knowledge client-side encryption for Vestige Cloud sync. +//! +//! The portable archive is encrypted on the client **before** it is uploaded +//! and decrypted **after** it is downloaded, so the hosted service only ever +//! stores ciphertext. The encryption passphrase is supplied by the user +//! (`VESTIGE_CLOUD_ENCRYPTION_KEY`) and is **never** sent to the server — it is +//! independent of the bearer sync key. This is what makes the "we hold no keys" +//! guarantee literally true: a server breach yields only random noise. +//! +//! Construction: +//! - KDF: Argon2id over (passphrase, random 16-byte salt) → 32-byte key. +//! - AEAD: XChaCha20-Poly1305 (192-bit nonce) over the archive bytes. +//! - Envelope (all non-secret framing prepended to the ciphertext): +//! `MAGIC(8) | VERSION(1) | salt(16) | nonce(24) | ciphertext+tag` +//! +//! Tradeoff (by design, and a selling point): if the user loses the passphrase, +//! the synced data is unrecoverable. We cannot reset it — we never have it. + +use argon2::Argon2; +use chacha20poly1305::aead::rand_core::RngCore; +use chacha20poly1305::aead::{Aead, AeadCore, KeyInit, OsRng}; +use chacha20poly1305::{Key, XChaCha20Poly1305, XNonce}; + +use super::sqlite::{Result, StorageError}; + +/// Magic marker identifying a Vestige zero-knowledge envelope. +const MAGIC: &[u8; 8] = b"VSTGENC1"; +const VERSION: u8 = 1; +const SALT_LEN: usize = 16; +const NONCE_LEN: usize = 24; // XChaCha20-Poly1305 nonce is 192-bit. +const KEY_LEN: usize = 32; +const HEADER_LEN: usize = MAGIC.len() + 1 + SALT_LEN + NONCE_LEN; + +/// Derive a 32-byte key from the passphrase and salt using Argon2id (defaults). +fn derive_key(passphrase: &[u8], salt: &[u8]) -> Result<[u8; KEY_LEN]> { + let mut key = [0u8; KEY_LEN]; + Argon2::default() + .hash_password_into(passphrase, salt, &mut key) + .map_err(|e| StorageError::Init(format!("key derivation failed: {e}")))?; + Ok(key) +} + +/// Encrypt `plaintext` under `passphrase`, returning the self-describing envelope. +/// +/// A fresh random salt and nonce are generated per call, so re-encrypting the +/// same archive yields different ciphertext (no deterministic leakage). +pub fn encrypt(passphrase: &str, plaintext: &[u8]) -> Result> { + let mut salt = [0u8; SALT_LEN]; + OsRng.fill_bytes(&mut salt); + + let key_bytes = derive_key(passphrase.as_bytes(), &salt)?; + let cipher = XChaCha20Poly1305::new(Key::from_slice(&key_bytes)); + let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng); + + let ciphertext = cipher + .encrypt(&nonce, plaintext) + .map_err(|_| StorageError::Init("encryption failed".to_string()))?; + + let mut out = Vec::with_capacity(HEADER_LEN + ciphertext.len()); + out.extend_from_slice(MAGIC); + out.push(VERSION); + out.extend_from_slice(&salt); + out.extend_from_slice(nonce.as_slice()); + out.extend_from_slice(&ciphertext); + Ok(out) +} + +/// True if `bytes` start with the Vestige encryption magic. Lets the sync +/// backend distinguish an encrypted envelope from a legacy plaintext archive. +pub fn is_encrypted(bytes: &[u8]) -> bool { + bytes.len() >= MAGIC.len() && &bytes[..MAGIC.len()] == MAGIC +} + +/// Decrypt a Vestige envelope under `passphrase`. Fails on a wrong passphrase or +/// any tampering (the Poly1305 tag is verified). +pub fn decrypt(passphrase: &str, envelope: &[u8]) -> Result> { + if envelope.len() < HEADER_LEN { + return Err(StorageError::Init( + "cloud archive is too short to be a valid encrypted envelope".to_string(), + )); + } + if &envelope[..MAGIC.len()] != MAGIC { + return Err(StorageError::Init( + "cloud archive is not a Vestige encrypted envelope".to_string(), + )); + } + let version = envelope[MAGIC.len()]; + if version != VERSION { + return Err(StorageError::Init(format!( + "unsupported cloud encryption version {version}" + ))); + } + + let salt_start = MAGIC.len() + 1; + let nonce_start = salt_start + SALT_LEN; + let ct_start = nonce_start + NONCE_LEN; + let salt = &envelope[salt_start..nonce_start]; + let nonce = XNonce::from_slice(&envelope[nonce_start..ct_start]); + let ciphertext = &envelope[ct_start..]; + + let key_bytes = derive_key(passphrase.as_bytes(), salt)?; + let cipher = XChaCha20Poly1305::new(Key::from_slice(&key_bytes)); + + cipher.decrypt(nonce, ciphertext).map_err(|_| { + StorageError::Init( + "cloud decryption failed: wrong VESTIGE_CLOUD_ENCRYPTION_KEY or corrupted data" + .to_string(), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let pass = "correct horse battery staple"; + let msg = b"the entire cognitive graph, in plaintext, before upload"; + let env = encrypt(pass, msg).unwrap(); + assert!(is_encrypted(&env)); + assert_ne!(&env[..], &msg[..], "envelope must not contain plaintext"); + let back = decrypt(pass, &env).unwrap(); + assert_eq!(back, msg); + } + + #[test] + fn wrong_passphrase_fails() { + let env = encrypt("right-pass", b"secret").unwrap(); + assert!(decrypt("wrong-pass", &env).is_err()); + } + + #[test] + fn tamper_is_detected() { + let mut env = encrypt("pass", b"important memory").unwrap(); + // Flip a byte in the ciphertext region. + let last = env.len() - 1; + env[last] ^= 0xff; + assert!(decrypt("pass", &env).is_err(), "AEAD must reject tampering"); + } + + #[test] + fn ciphertext_is_nondeterministic() { + // Same input encrypted twice → different envelopes (random salt+nonce). + let a = encrypt("p", b"x").unwrap(); + let b = encrypt("p", b"x").unwrap(); + assert_ne!(a, b); + // Both still decrypt correctly. + assert_eq!(decrypt("p", &a).unwrap(), b"x"); + assert_eq!(decrypt("p", &b).unwrap(), b"x"); + } + + #[test] + fn plaintext_is_not_misdetected_as_envelope() { + assert!(!is_encrypted(b"{\"archiveFormat\":\"vestige.portable.v1\"}")); + assert!(!is_encrypted(b"")); + } + + #[test] + fn rejects_short_or_foreign_envelope() { + assert!(decrypt("p", b"too short").is_err()); + assert!(decrypt("p", &[0u8; 100]).is_err()); + } +} diff --git a/crates/vestige-core/src/storage/cloud_sync.rs b/crates/vestige-core/src/storage/cloud_sync.rs new file mode 100644 index 0000000..e30c525 --- /dev/null +++ b/crates/vestige-core/src/storage/cloud_sync.rs @@ -0,0 +1,391 @@ +//! Hosted managed-sync backend (Vestige Cloud). +//! +//! This module is only compiled with the `cloud-sync` feature. It provides +//! [`HttpPortableSyncBackend`], an HTTP implementation of the +//! [`PortableSyncBackend`](super::sqlite::PortableSyncBackend) trait that +//! pull-merge-pushes the portable archive to a hosted blob endpoint. +//! +//! The merge/conflict engine is unchanged: this backend only moves bytes. The +//! authoritative `key -> namespace` mapping and per-user isolation live in the +//! hosted service; the client just presents an opaque sync key as a bearer +//! token. The default local-first build never links an HTTP client. +//! +//! ## Concurrency +//! +//! Two devices can each pull → merge → push. To avoid a lost update in the +//! GET↔PUT window, the backend uses optimistic concurrency: it captures the +//! object `ETag` on read and sends it as `If-Match` on write. The generic +//! [`sync_portable_archive`](super::sqlite::SqliteMemoryStore::sync_portable_archive) +//! driver calls `read_archive` then `write_archive` exactly once, so the ETag +//! captured during the pull is the precondition for the push. A +//! `412 Precondition Failed` means another device wrote in between; the caller +//! re-runs sync (the merge is idempotent and converges by `updated_at`). + +use std::cell::RefCell; +use std::time::Duration; + +use reqwest::blocking::Client; +use reqwest::header::{AUTHORIZATION, ETAG, IF_MATCH}; +use reqwest::StatusCode; + +use super::portable::PortableArchive; +use super::sqlite::{PortableSyncBackend, Result, StorageError}; + +/// Default request timeout for cloud sync HTTP calls. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(60); + +/// Blob path on the hosted service. One opaque blob per sync key (the service +/// derives the namespace from the key), so the client uses a fixed path. +const BLOB_PATH: &str = "/v1/blob"; + +/// HTTP-backed portable sync backend for Vestige Cloud. +/// +/// Mirrors the shape of +/// [`FilePortableSyncBackend`](super::sqlite::FilePortableSyncBackend) but reads +/// and writes the archive over HTTPS with a per-user bearer key. +#[derive(Debug)] +pub struct HttpPortableSyncBackend { + /// Base endpoint, e.g. `https://sync.vestige.dev`. No trailing slash. + endpoint: String, + /// Per-user sync key, presented as `Authorization: Bearer `. + sync_key: String, + /// Optional zero-knowledge passphrase. When set, the archive is encrypted + /// before upload and decrypted after download — the server never sees + /// plaintext, and this passphrase is never sent to the server. + encryption_key: Option, + /// Blocking HTTP client (the trait is synchronous). + client: Client, + /// ETag captured on the most recent successful read, used as the `If-Match` + /// precondition on the next write. `None` until the first read, or when the + /// remote had no archive yet. + last_etag: RefCell>, +} + +impl HttpPortableSyncBackend { + /// Build a cloud sync backend for `endpoint` authenticated with `sync_key`, + /// with no client-side encryption (plaintext upload). + /// + /// A trailing slash on `endpoint` is trimmed so URL joining is predictable. + pub fn new(endpoint: impl Into, sync_key: impl Into) -> Result { + Self::new_with_encryption(endpoint, sync_key, None) + } + + /// Build a cloud sync backend with optional zero-knowledge encryption. + /// + /// When `encryption_key` is `Some`, the portable archive is encrypted with + /// XChaCha20-Poly1305 (Argon2id-derived key) before upload and decrypted on + /// download. The passphrase never leaves this process. + pub fn new_with_encryption( + endpoint: impl Into, + sync_key: impl Into, + encryption_key: Option, + ) -> Result { + let endpoint = endpoint.into().trim_end_matches('/').to_string(); + let sync_key = sync_key.into(); + if endpoint.is_empty() { + return Err(StorageError::Init( + "cloud sync endpoint is empty (set VESTIGE_CLOUD_ENDPOINT)".to_string(), + )); + } + if sync_key.is_empty() { + return Err(StorageError::Init( + "cloud sync key is empty (set VESTIGE_CLOUD_SYNC_KEY)".to_string(), + )); + } + let encryption_key = encryption_key.filter(|k| !k.is_empty()); + let client = Client::builder() + .timeout(REQUEST_TIMEOUT) + .user_agent(concat!("vestige-cloud-sync/", env!("CARGO_PKG_VERSION"))) + .build() + .map_err(|e| StorageError::Init(format!("failed to build HTTP client: {e}")))?; + Ok(Self { + endpoint, + sync_key, + encryption_key, + client, + last_etag: RefCell::new(None), + }) + } + + /// Whether this backend encrypts client-side (zero-knowledge). + pub fn is_encrypted(&self) -> bool { + self.encryption_key.is_some() + } + + /// Full blob URL for this backend. + fn blob_url(&self) -> String { + format!("{}{}", self.endpoint, BLOB_PATH) + } +} + +impl PortableSyncBackend for HttpPortableSyncBackend { + fn label(&self) -> String { + format!("cloud:{}", self.endpoint) + } + + fn read_archive(&self) -> Result> { + let resp = self + .client + .get(self.blob_url()) + .header(AUTHORIZATION, format!("Bearer {}", self.sync_key)) + .send() + .map_err(|e| StorageError::Init(format!("cloud sync read failed: {e}")))?; + + match resp.status() { + StatusCode::NOT_FOUND => { + // No remote archive yet — first sync for this key. + *self.last_etag.borrow_mut() = None; + Ok(None) + } + StatusCode::OK => { + // Capture the ETag for the matching If-Match write. + let etag = resp + .headers() + .get(ETAG) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + *self.last_etag.borrow_mut() = etag; + + let bytes = resp + .bytes() + .map_err(|e| StorageError::Init(format!("cloud sync read body failed: {e}")))?; + + // Decrypt if this is a zero-knowledge envelope. If a passphrase + // is configured but the remote is still plaintext (legacy), parse + // it directly — the next push will encrypt it (transparent upgrade). + let plaintext: std::borrow::Cow<'_, [u8]> = + if super::cloud_crypto::is_encrypted(&bytes) { + let pass = self.encryption_key.as_deref().ok_or_else(|| { + StorageError::Init( + "remote archive is encrypted but VESTIGE_CLOUD_ENCRYPTION_KEY is \ + not set on this device" + .to_string(), + ) + })?; + std::borrow::Cow::Owned(super::cloud_crypto::decrypt(pass, &bytes)?) + } else { + std::borrow::Cow::Borrowed(&bytes) + }; + + let archive: PortableArchive = + serde_json::from_slice(&plaintext).map_err(|e| { + StorageError::Init(format!("failed to parse cloud sync archive: {e}")) + })?; + Ok(Some(archive)) + } + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(StorageError::Init( + "cloud sync rejected the sync key (401/403). Check your subscription and \ + VESTIGE_CLOUD_SYNC_KEY." + .to_string(), + )), + other => Err(StorageError::Init(format!( + "cloud sync read returned unexpected status {other}" + ))), + } + } + + fn write_archive(&self, archive: &PortableArchive) -> Result<()> { + let plaintext = serde_json::to_vec(archive) + .map_err(|e| StorageError::Init(format!("failed to serialize archive: {e}")))?; + + // Zero-knowledge: encrypt before upload when a passphrase is set, so the + // server only ever stores ciphertext. Content type reflects the payload. + let (body, content_type) = match self.encryption_key.as_deref() { + Some(pass) => ( + super::cloud_crypto::encrypt(pass, &plaintext)?, + "application/octet-stream", + ), + None => (plaintext, "application/json"), + }; + + let mut req = self + .client + .put(self.blob_url()) + .header(AUTHORIZATION, format!("Bearer {}", self.sync_key)) + .header(reqwest::header::CONTENT_TYPE, content_type) + .body(body); + + // Optimistic concurrency: only overwrite the object we pulled. If the + // remote had no archive, require that it still doesn't exist (`If-Match: *` + // would require existence, so we omit the header to allow first create). + if let Some(etag) = self.last_etag.borrow_mut().take() { + req = req.header(IF_MATCH, etag); + } + + let resp = req + .send() + .map_err(|e| StorageError::Init(format!("cloud sync write failed: {e}")))?; + + match resp.status() { + StatusCode::OK | StatusCode::CREATED | StatusCode::NO_CONTENT => Ok(()), + StatusCode::PRECONDITION_FAILED => Err(StorageError::Init( + "cloud sync conflict: another device updated your memory in between. \ + Run `vestige sync --cloud` again to merge and retry." + .to_string(), + )), + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(StorageError::Init( + "cloud sync rejected the sync key (401/403). Check your subscription and \ + VESTIGE_CLOUD_SYNC_KEY." + .to_string(), + )), + StatusCode::PAYLOAD_TOO_LARGE => Err(StorageError::Init( + "cloud sync archive too large for the hosted plan limit".to_string(), + )), + other => Err(StorageError::Init(format!( + "cloud sync write returned unexpected status {other}" + ))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::portable::{PortableArchive, PortableTable, PORTABLE_ARCHIVE_FORMAT}; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::mpsc; + use std::thread; + + fn sample_archive() -> PortableArchive { + PortableArchive { + archive_format: PORTABLE_ARCHIVE_FORMAT.to_string(), + vestige_version: "test".to_string(), + schema_version: 1, + exported_at: chrono::Utc::now(), + mode: "exact".to_string(), + tables: vec![PortableTable { + name: "knowledge_nodes".to_string(), + columns: vec!["id".to_string()], + rows: vec![], + }], + } + } + + /// A captured request the mock observed, surfaced to the test thread. + #[derive(Debug, Default, Clone)] + struct CapturedRequest { + method: String, + authorization: Option, + if_match: Option, + } + + /// Minimal one-shot HTTP mock. `responder` builds the raw HTTP response + /// string for the request line + headers it parsed. Returns the bound base + /// URL and a receiver for the captured request. + fn spawn_mock(responder: F) -> (String, mpsc::Receiver) + where + F: Fn(&CapturedRequest) -> String + Send + 'static, + { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock"); + let addr = listener.local_addr().expect("addr"); + let (tx, rx) = mpsc::channel(); + thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut buf = [0u8; 8192]; + let n = stream.read(&mut buf).unwrap_or(0); + let text = String::from_utf8_lossy(&buf[..n]); + let mut cap = CapturedRequest::default(); + for (i, line) in text.lines().enumerate() { + if i == 0 { + cap.method = line.split_whitespace().next().unwrap_or("").to_string(); + } else if let Some(v) = line.strip_prefix("authorization: ") { + cap.authorization = Some(v.trim().to_string()); + } else if let Some(v) = line.strip_prefix("if-match: ") { + cap.if_match = Some(v.trim().to_string()); + } + } + let response = responder(&cap); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + let _ = tx.send(cap); + } + }); + (format!("http://{addr}"), rx) + } + + fn http_response(status: &str, extra_headers: &str, body: &str) -> String { + format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\n{extra_headers}Connection: close\r\n\r\n{body}", + body.len() + ) + } + + #[test] + fn new_rejects_empty_endpoint_and_key() { + assert!(HttpPortableSyncBackend::new("", "key").is_err()); + assert!(HttpPortableSyncBackend::new("https://x", "").is_err()); + assert!(HttpPortableSyncBackend::new("https://x", "key").is_ok()); + } + + #[test] + fn endpoint_trailing_slash_trimmed() { + let be = HttpPortableSyncBackend::new("https://sync.example/", "k").unwrap(); + assert_eq!(be.blob_url(), "https://sync.example/v1/blob"); + } + + #[test] + fn read_404_returns_none() { + let (base, rx) = spawn_mock(|_| http_response("404 Not Found", "", "")); + let be = HttpPortableSyncBackend::new(base, "secret").unwrap(); + let got = be.read_archive().expect("read ok"); + assert!(got.is_none()); + let cap = rx.recv().unwrap(); + assert_eq!(cap.method, "GET"); + assert_eq!(cap.authorization.as_deref(), Some("Bearer secret")); + } + + #[test] + fn read_200_parses_and_captures_etag() { + let archive = sample_archive(); + let body = serde_json::to_string(&archive).unwrap(); + let (base, _rx) = spawn_mock(move |_| { + http_response("200 OK", "ETag: \"v1-abc\"\r\n", &body) + }); + let be = HttpPortableSyncBackend::new(base, "secret").unwrap(); + let got = be.read_archive().expect("read ok").expect("some archive"); + assert_eq!(got.archive_format, PORTABLE_ARCHIVE_FORMAT); + // ETag captured for the next If-Match write. + assert_eq!(be.last_etag.borrow().as_deref(), Some("\"v1-abc\"")); + } + + #[test] + fn read_401_is_error() { + let (base, _rx) = spawn_mock(|_| http_response("401 Unauthorized", "", "")); + let be = HttpPortableSyncBackend::new(base, "bad").unwrap(); + assert!(be.read_archive().is_err()); + } + + #[test] + fn write_sends_if_match_when_etag_present() { + // Seed an etag as if a prior read happened. + let (base, rx) = spawn_mock(|_| http_response("200 OK", "", "")); + let be = HttpPortableSyncBackend::new(base, "secret").unwrap(); + *be.last_etag.borrow_mut() = Some("\"v1-abc\"".to_string()); + be.write_archive(&sample_archive()).expect("write ok"); + let cap = rx.recv().unwrap(); + assert_eq!(cap.method, "PUT"); + assert_eq!(cap.authorization.as_deref(), Some("Bearer secret")); + assert_eq!(cap.if_match.as_deref(), Some("\"v1-abc\"")); + } + + #[test] + fn write_omits_if_match_for_first_create() { + let (base, rx) = spawn_mock(|_| http_response("201 Created", "", "")); + let be = HttpPortableSyncBackend::new(base, "secret").unwrap(); + // No prior read → no etag → no If-Match (allow create). + be.write_archive(&sample_archive()).expect("write ok"); + let cap = rx.recv().unwrap(); + assert_eq!(cap.method, "PUT"); + assert!(cap.if_match.is_none()); + } + + #[test] + fn write_412_is_conflict_error() { + let (base, _rx) = spawn_mock(|_| http_response("412 Precondition Failed", "", "")); + let be = HttpPortableSyncBackend::new(base, "secret").unwrap(); + *be.last_etag.borrow_mut() = Some("\"stale\"".to_string()); + let err = be.write_archive(&sample_archive()).unwrap_err(); + assert!(err.to_string().contains("conflict")); + } +} diff --git a/crates/vestige-core/src/storage/mod.rs b/crates/vestige-core/src/storage/mod.rs index d821259..84bb2de 100644 --- a/crates/vestige-core/src/storage/mod.rs +++ b/crates/vestige-core/src/storage/mod.rs @@ -2,12 +2,19 @@ //! //! Backend-agnostic memory store abstraction plus SQLite reference impl. +#[cfg(feature = "cloud-sync")] +mod cloud_crypto; +#[cfg(feature = "cloud-sync")] +mod cloud_sync; mod memory_store; mod migrations; mod portable; mod sqlite; mod trace_store; +#[cfg(feature = "cloud-sync")] +pub use cloud_sync::HttpPortableSyncBackend; + pub use memory_store::{ ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord, MemoryStore, MemoryStoreError, MemoryStoreResult, MemoryStoreSend, ModelSignature, diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index cf7f941..05fe6aa 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -6390,6 +6390,29 @@ impl SqliteMemoryStore { self.sync_portable_archive(&backend) } + /// Synchronize this database with the hosted Vestige Cloud managed-sync + /// service. `endpoint` is the base URL (e.g. `https://sync.vestige.dev`) and + /// `sync_key` is the per-user key issued at purchase. Pull-merge-push is + /// identical to file sync — only the transport differs. + /// + /// When `encryption_key` is `Some`, the archive is encrypted client-side + /// (XChaCha20-Poly1305) before upload, so the server only stores ciphertext + /// (zero-knowledge). The passphrase never leaves this process. + #[cfg(feature = "cloud-sync")] + pub fn sync_portable_archive_cloud( + &self, + endpoint: &str, + sync_key: &str, + encryption_key: Option, + ) -> Result { + let backend = super::cloud_sync::HttpPortableSyncBackend::new_with_encryption( + endpoint, + sync_key, + encryption_key, + )?; + self.sync_portable_archive(&backend) + } + fn merge_portable_table( tx: &rusqlite::Transaction<'_>, table_name: &str, diff --git a/crates/vestige-mcp/Cargo.toml b/crates/vestige-mcp/Cargo.toml index ca1ec34..87f2a5f 100644 --- a/crates/vestige-mcp/Cargo.toml +++ b/crates/vestige-mcp/Cargo.toml @@ -10,13 +10,18 @@ categories = ["command-line-utilities", "database"] repository = "https://github.com/samvallad33/vestige" [features] -default = ["embeddings", "ort-download", "vector-search", "connectors"] +default = ["embeddings", "ort-download", "vector-search", "connectors", "cloud-sync"] embeddings = ["vestige-core/embeddings"] vector-search = ["vestige-core/vector-search"] # External-source connectors (#57): GitHub Issues / Redmine indexing via the # `source_sync` MCP tool. On by default so the tool works out of the box; turn # off for a build with no HTTP client. connectors = ["vestige-core/connectors"] +# Hosted managed-sync (Vestige Cloud): `vestige sync --cloud` pushes/pulls the +# portable archive to the hosted blob service. On by default so the command +# works out of the box; the binary already links an HTTP client via +# `connectors`, so this adds no new dependency cost. +cloud-sync = ["vestige-core/cloud-sync"] # Default ort backend: downloads prebuilt ONNX Runtime at build time. # Fails on targets without prebuilts (notably x86_64-apple-darwin). ort-download = ["embeddings", "vestige-core/ort-download"] diff --git a/crates/vestige-mcp/src/bin/cli.rs b/crates/vestige-mcp/src/bin/cli.rs index b79dd55..555dc65 100644 --- a/crates/vestige-mcp/src/bin/cli.rs +++ b/crates/vestige-mcp/src/bin/cli.rs @@ -182,10 +182,20 @@ enum Commands { merge: bool, }, - /// Two-way sync with a file-backed portable archive + /// Two-way sync with a file-backed portable archive, or Vestige Cloud Sync { - /// Sync archive path, often in Dropbox/iCloud/Syncthing/Git - archive: PathBuf, + /// Sync archive path, often in Dropbox/iCloud/Syncthing/Git. + /// Omit when using --cloud. + archive: Option, + /// Sync with the hosted Vestige Cloud managed-sync service instead of a + /// file. Requires a sync key (VESTIGE_CLOUD_SYNC_KEY) and endpoint + /// (--endpoint or VESTIGE_CLOUD_ENDPOINT). + #[arg(long)] + cloud: bool, + /// Vestige Cloud base endpoint (e.g. https://sync.vestige.dev). + /// Defaults to the VESTIGE_CLOUD_ENDPOINT env var. + #[arg(long)] + endpoint: Option, }, /// Garbage collect stale memories below retention threshold @@ -287,7 +297,11 @@ fn main() -> anyhow::Result<()> { } => run_export(output, format, tags, since), Commands::PortableExport { output } => run_portable_export(output), Commands::PortableImport { input, merge } => run_portable_import(input, merge), - Commands::Sync { archive } => run_sync(archive), + Commands::Sync { + archive, + cloud, + endpoint, + } => run_sync(archive, cloud, endpoint), Commands::Gc { min_retention, max_age_days, @@ -2193,14 +2207,92 @@ fn run_portable_import(input: PathBuf, merge: bool) -> anyhow::Result<()> { } /// Run file-backed two-way sync. -fn run_sync(archive: PathBuf) -> anyhow::Result<()> { +fn run_sync( + archive: Option, + cloud: bool, + endpoint: Option, +) -> anyhow::Result<()> { + if cloud { + run_sync_cloud(endpoint) + } else { + let archive = archive.ok_or_else(|| { + anyhow::anyhow!( + "no sync target: pass an archive path for file sync, or --cloud for Vestige Cloud" + ) + })?; + run_sync_file(archive) + } +} + +fn run_sync_file(archive: PathBuf) -> anyhow::Result<()> { println!("{}", "=== Vestige File Sync ===".cyan().bold()); println!(); println!("{}: {}", "Archive".white().bold(), archive.display()); let storage = open_storage()?; let report = storage.sync_portable_archive_file(&archive)?; + print_sync_report(&report); + Ok(()) +} +#[cfg(feature = "cloud-sync")] +fn run_sync_cloud(endpoint: Option) -> anyhow::Result<()> { + let endpoint = endpoint + .or_else(|| std::env::var("VESTIGE_CLOUD_ENDPOINT").ok()) + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| { + anyhow::anyhow!( + "no cloud endpoint: pass --endpoint or set VESTIGE_CLOUD_ENDPOINT \ + (e.g. https://sync.vestige.dev)" + ) + })?; + let sync_key = std::env::var("VESTIGE_CLOUD_SYNC_KEY") + .ok() + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| { + anyhow::anyhow!( + "no sync key: set VESTIGE_CLOUD_SYNC_KEY (issued when you subscribe to \ + Vestige Cloud)" + ) + })?; + + // Optional zero-knowledge encryption passphrase. Never sent to the server. + let encryption_key = std::env::var("VESTIGE_CLOUD_ENCRYPTION_KEY") + .ok() + .filter(|s| !s.trim().is_empty()); + + println!("{}", "=== Vestige Cloud Sync ===".cyan().bold()); + println!(); + println!("{}: {}", "Endpoint".white().bold(), endpoint); + if encryption_key.is_some() { + println!( + "{}: {}", + "Encryption".white().bold(), + "zero-knowledge (XChaCha20-Poly1305) — your data is encrypted before upload".green() + ); + } else { + println!( + "{}: {}", + "Encryption".white().bold(), + "OFF — set VESTIGE_CLOUD_ENCRYPTION_KEY for zero-knowledge sync".yellow() + ); + } + + let storage = open_storage()?; + let report = storage.sync_portable_archive_cloud(&endpoint, &sync_key, encryption_key)?; + print_sync_report(&report); + Ok(()) +} + +#[cfg(not(feature = "cloud-sync"))] +fn run_sync_cloud(_endpoint: Option) -> anyhow::Result<()> { + anyhow::bail!( + "this build was compiled without the `cloud-sync` feature; rebuild with \ + --features cloud-sync to use Vestige Cloud" + ) +} + +fn print_sync_report(report: &vestige_core::PortableSyncReport) { if let Some(pull) = &report.pull { println!("{}", "Pull: merged remote archive".yellow()); println!( @@ -2228,8 +2320,6 @@ fn run_sync(archive: PathBuf) -> anyhow::Result<()> { .green() .bold() ); - - Ok(()) } /// Run garbage collection command diff --git a/scripts/check-no-private-cloud.sh b/scripts/check-no-private-cloud.sh new file mode 100755 index 0000000..d666dc5 --- /dev/null +++ b/scripts/check-no-private-cloud.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# check-no-private-cloud.sh — Fail if private Vestige Cloud *service* code leaks +# into this public repository. +# +# Vestige Cloud is split: +# - PUBLIC client (this repo): a thin HTTP sync backend that only moves +# encrypted bytes and presents an opaque bearer token. +# Legitimate. Reads VESTIGE_CLOUD_ENDPOINT / _SYNC_KEY / +# _ENCRYPTION_KEY env vars on the client side. +# - PRIVATE service (separate repo, no public remote): the hosted blob +# service that owns sync-key -> namespace mapping, per-user +# isolation, Lemon Squeezy billing webhooks, and +# transactional email. This MUST NEVER be committed here. +# +# This guard scans only tracked files (git grep) for distinctive *service* +# signatures — module headers, billing/provider internals, and server-side +# auth/namespace mapping — chosen so the legitimate public client does NOT +# match. It deliberately does NOT match the VESTIGE_CLOUD_* client env-var +# prefix, which the public client uses legitimately. +set -u + +cd "$(git rev-parse --show-toplevel)" || { + echo "check-no-private-cloud: not inside a git repository" >&2 + exit 2 +} + +# Distinctive private-service signatures. Each is a fixed string (grep -F via +# -e with --fixed-strings) that appears in the private vestige-cloud service +# and must never appear in this public repo. Keep these specific. +PATTERNS=( + # Service crate identity / entrypoint + 'name = "vestige-cloud"' + 'Vestige Cloud — hosted managed-sync blob service' + # Service module headers + 'Sync-key store and authentication' + 'Blob storage for the managed-sync service' + 'Lemon Squeezy webhook handling' + 'Transactional email delivery via Resend' + # Billing / provider internals (server-only) + 'LEMONSQUEEZY_WEBHOOK_SECRET' + 'lemonsqueezy' + # Server-side sync-key -> namespace mapping (the authoritative mapping that + # by design lives ONLY in the hosted service, never the client) + 'sync_keys SET key_hash' +) + +# Files this guard itself lives in / references the patterns must be excluded, +# or it would always flag itself. Exclude this script and any allowlist doc. +EXCLUDES=( + ':(exclude)scripts/check-no-private-cloud.sh' + ':(exclude).github/workflows/guard-no-private-cloud.yml' +) + +violations=0 +report="" + +for pat in "${PATTERNS[@]}"; do + # -I skip binary, -n line numbers, -F fixed string, -i case-insensitive. + if hits=$(git grep -Ini -F -e "$pat" -- "${EXCLUDES[@]}" 2>/dev/null); then + if [ -n "$hits" ]; then + violations=$((violations + 1)) + report+=$'\n'" ✗ private-service marker found: \"$pat\""$'\n' + report+="$(printf '%s\n' "$hits" | sed 's/^/ /')"$'\n' + fi + fi +done + +if [ "$violations" -ne 0 ]; then + echo "════════════════════════════════════════════════════════════════════" + echo " PRIVATE CLOUD SERVICE CODE DETECTED IN PUBLIC REPO" + echo "════════════════════════════════════════════════════════════════════" + echo "$report" + echo "════════════════════════════════════════════════════════════════════" + echo " The hosted Vestige Cloud service (billing, namespace mapping," + echo " per-user isolation) must live ONLY in the private repo, never here." + echo " Remove the file(s) above from this repo. If a match is a false" + echo " positive, refine the pattern in scripts/check-no-private-cloud.sh." + echo "════════════════════════════════════════════════════════════════════" + exit 1 +fi + +echo "check-no-private-cloud: OK — no private Vestige Cloud service code in public repo" +exit 0