mirror of
https://github.com/samvallad33/vestige.git
synced 2026-07-26 23:51:02 +02:00
merge: cloud sync pro — encrypted managed sync (lane 5/5)
This commit is contained in:
commit
335a42f341
11 changed files with 945 additions and 11 deletions
|
|
@ -63,6 +63,14 @@ qwen3-reranker = ["qwen3-embeddings"]
|
|||
# the default local-first build never links an HTTP client.
|
||||
connectors = ["dep:reqwest"]
|
||||
|
||||
# Hosted managed-sync backend (Vestige Cloud). Adds the HTTP `PortableSyncBackend`
|
||||
# (`HttpPortableSyncBackend`) that pull-merge-pushes the portable archive to a
|
||||
# hosted blob endpoint over HTTPS with a per-user sync key. Like `connectors`,
|
||||
# this is the ONLY thing that links an HTTP client — the default local-first
|
||||
# build stays network-free. Uses `reqwest`'s blocking client because the
|
||||
# `PortableSyncBackend` trait methods are synchronous.
|
||||
cloud-sync = ["dep:reqwest", "reqwest/blocking", "dep:chacha20poly1305", "dep:argon2"]
|
||||
|
||||
# Metal GPU acceleration on Apple Silicon (significantly faster inference)
|
||||
metal = ["fastembed/metal"]
|
||||
|
||||
|
|
@ -157,6 +165,15 @@ blake3 = "1"
|
|||
# `connectors` feature — the default local-first build does not link reqwest.
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true }
|
||||
|
||||
# ============================================================================
|
||||
# OPTIONAL: Zero-knowledge client-side encryption for Vestige Cloud sync
|
||||
# ============================================================================
|
||||
# XChaCha20-Poly1305 AEAD + Argon2id passphrase KDF. The archive is encrypted on
|
||||
# the client before upload, so the hosted service only ever stores ciphertext.
|
||||
# Behind the `cloud-sync` feature — the default local-first build links neither.
|
||||
chacha20poly1305 = { version = "0.10", optional = true }
|
||||
argon2 = { version = "0.5", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
criterion = { version = "0.5", features = ["html_reports"] }
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ pub use storage::{
|
|||
PortableArchive,
|
||||
PortableImportMode,
|
||||
PortableImportReport,
|
||||
PortableSyncReport,
|
||||
ReconcileReport,
|
||||
Result,
|
||||
SchedulingState,
|
||||
|
|
|
|||
164
crates/vestige-core/src/storage/cloud_crypto.rs
Normal file
164
crates/vestige-core/src/storage/cloud_crypto.rs
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
//! Zero-knowledge client-side encryption for Vestige Cloud sync.
|
||||
//!
|
||||
//! The portable archive is encrypted on the client **before** it is uploaded
|
||||
//! and decrypted **after** it is downloaded, so the hosted service only ever
|
||||
//! stores ciphertext. The encryption passphrase is supplied by the user
|
||||
//! (`VESTIGE_CLOUD_ENCRYPTION_KEY`) and is **never** sent to the server — it is
|
||||
//! independent of the bearer sync key. This is what makes the "we hold no keys"
|
||||
//! guarantee literally true: a server breach yields only random noise.
|
||||
//!
|
||||
//! Construction:
|
||||
//! - KDF: Argon2id over (passphrase, random 16-byte salt) → 32-byte key.
|
||||
//! - AEAD: XChaCha20-Poly1305 (192-bit nonce) over the archive bytes.
|
||||
//! - Envelope (all non-secret framing prepended to the ciphertext):
|
||||
//! `MAGIC(8) | VERSION(1) | salt(16) | nonce(24) | ciphertext+tag`
|
||||
//!
|
||||
//! Tradeoff (by design, and a selling point): if the user loses the passphrase,
|
||||
//! the synced data is unrecoverable. We cannot reset it — we never have it.
|
||||
|
||||
use argon2::Argon2;
|
||||
use chacha20poly1305::aead::rand_core::RngCore;
|
||||
use chacha20poly1305::aead::{Aead, AeadCore, KeyInit, OsRng};
|
||||
use chacha20poly1305::{Key, XChaCha20Poly1305, XNonce};
|
||||
|
||||
use super::sqlite::{Result, StorageError};
|
||||
|
||||
/// Magic marker identifying a Vestige zero-knowledge envelope.
|
||||
const MAGIC: &[u8; 8] = b"VSTGENC1";
|
||||
const VERSION: u8 = 1;
|
||||
const SALT_LEN: usize = 16;
|
||||
const NONCE_LEN: usize = 24; // XChaCha20-Poly1305 nonce is 192-bit.
|
||||
const KEY_LEN: usize = 32;
|
||||
const HEADER_LEN: usize = MAGIC.len() + 1 + SALT_LEN + NONCE_LEN;
|
||||
|
||||
/// Derive a 32-byte key from the passphrase and salt using Argon2id (defaults).
|
||||
fn derive_key(passphrase: &[u8], salt: &[u8]) -> Result<[u8; KEY_LEN]> {
|
||||
let mut key = [0u8; KEY_LEN];
|
||||
Argon2::default()
|
||||
.hash_password_into(passphrase, salt, &mut key)
|
||||
.map_err(|e| StorageError::Init(format!("key derivation failed: {e}")))?;
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
/// Encrypt `plaintext` under `passphrase`, returning the self-describing envelope.
|
||||
///
|
||||
/// A fresh random salt and nonce are generated per call, so re-encrypting the
|
||||
/// same archive yields different ciphertext (no deterministic leakage).
|
||||
pub fn encrypt(passphrase: &str, plaintext: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut salt = [0u8; SALT_LEN];
|
||||
OsRng.fill_bytes(&mut salt);
|
||||
|
||||
let key_bytes = derive_key(passphrase.as_bytes(), &salt)?;
|
||||
let cipher = XChaCha20Poly1305::new(Key::from_slice(&key_bytes));
|
||||
let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng);
|
||||
|
||||
let ciphertext = cipher
|
||||
.encrypt(&nonce, plaintext)
|
||||
.map_err(|_| StorageError::Init("encryption failed".to_string()))?;
|
||||
|
||||
let mut out = Vec::with_capacity(HEADER_LEN + ciphertext.len());
|
||||
out.extend_from_slice(MAGIC);
|
||||
out.push(VERSION);
|
||||
out.extend_from_slice(&salt);
|
||||
out.extend_from_slice(nonce.as_slice());
|
||||
out.extend_from_slice(&ciphertext);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// True if `bytes` start with the Vestige encryption magic. Lets the sync
|
||||
/// backend distinguish an encrypted envelope from a legacy plaintext archive.
|
||||
pub fn is_encrypted(bytes: &[u8]) -> bool {
|
||||
bytes.len() >= MAGIC.len() && &bytes[..MAGIC.len()] == MAGIC
|
||||
}
|
||||
|
||||
/// Decrypt a Vestige envelope under `passphrase`. Fails on a wrong passphrase or
|
||||
/// any tampering (the Poly1305 tag is verified).
|
||||
pub fn decrypt(passphrase: &str, envelope: &[u8]) -> Result<Vec<u8>> {
|
||||
if envelope.len() < HEADER_LEN {
|
||||
return Err(StorageError::Init(
|
||||
"cloud archive is too short to be a valid encrypted envelope".to_string(),
|
||||
));
|
||||
}
|
||||
if &envelope[..MAGIC.len()] != MAGIC {
|
||||
return Err(StorageError::Init(
|
||||
"cloud archive is not a Vestige encrypted envelope".to_string(),
|
||||
));
|
||||
}
|
||||
let version = envelope[MAGIC.len()];
|
||||
if version != VERSION {
|
||||
return Err(StorageError::Init(format!(
|
||||
"unsupported cloud encryption version {version}"
|
||||
)));
|
||||
}
|
||||
|
||||
let salt_start = MAGIC.len() + 1;
|
||||
let nonce_start = salt_start + SALT_LEN;
|
||||
let ct_start = nonce_start + NONCE_LEN;
|
||||
let salt = &envelope[salt_start..nonce_start];
|
||||
let nonce = XNonce::from_slice(&envelope[nonce_start..ct_start]);
|
||||
let ciphertext = &envelope[ct_start..];
|
||||
|
||||
let key_bytes = derive_key(passphrase.as_bytes(), salt)?;
|
||||
let cipher = XChaCha20Poly1305::new(Key::from_slice(&key_bytes));
|
||||
|
||||
cipher.decrypt(nonce, ciphertext).map_err(|_| {
|
||||
StorageError::Init(
|
||||
"cloud decryption failed: wrong VESTIGE_CLOUD_ENCRYPTION_KEY or corrupted data"
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trip() {
|
||||
let pass = "correct horse battery staple";
|
||||
let msg = b"the entire cognitive graph, in plaintext, before upload";
|
||||
let env = encrypt(pass, msg).unwrap();
|
||||
assert!(is_encrypted(&env));
|
||||
assert_ne!(&env[..], &msg[..], "envelope must not contain plaintext");
|
||||
let back = decrypt(pass, &env).unwrap();
|
||||
assert_eq!(back, msg);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrong_passphrase_fails() {
|
||||
let env = encrypt("right-pass", b"secret").unwrap();
|
||||
assert!(decrypt("wrong-pass", &env).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tamper_is_detected() {
|
||||
let mut env = encrypt("pass", b"important memory").unwrap();
|
||||
// Flip a byte in the ciphertext region.
|
||||
let last = env.len() - 1;
|
||||
env[last] ^= 0xff;
|
||||
assert!(decrypt("pass", &env).is_err(), "AEAD must reject tampering");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ciphertext_is_nondeterministic() {
|
||||
// Same input encrypted twice → different envelopes (random salt+nonce).
|
||||
let a = encrypt("p", b"x").unwrap();
|
||||
let b = encrypt("p", b"x").unwrap();
|
||||
assert_ne!(a, b);
|
||||
// Both still decrypt correctly.
|
||||
assert_eq!(decrypt("p", &a).unwrap(), b"x");
|
||||
assert_eq!(decrypt("p", &b).unwrap(), b"x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plaintext_is_not_misdetected_as_envelope() {
|
||||
assert!(!is_encrypted(b"{\"archiveFormat\":\"vestige.portable.v1\"}"));
|
||||
assert!(!is_encrypted(b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_short_or_foreign_envelope() {
|
||||
assert!(decrypt("p", b"too short").is_err());
|
||||
assert!(decrypt("p", &[0u8; 100]).is_err());
|
||||
}
|
||||
}
|
||||
391
crates/vestige-core/src/storage/cloud_sync.rs
Normal file
391
crates/vestige-core/src/storage/cloud_sync.rs
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
//! Hosted managed-sync backend (Vestige Cloud).
|
||||
//!
|
||||
//! This module is only compiled with the `cloud-sync` feature. It provides
|
||||
//! [`HttpPortableSyncBackend`], an HTTP implementation of the
|
||||
//! [`PortableSyncBackend`](super::sqlite::PortableSyncBackend) trait that
|
||||
//! pull-merge-pushes the portable archive to a hosted blob endpoint.
|
||||
//!
|
||||
//! The merge/conflict engine is unchanged: this backend only moves bytes. The
|
||||
//! authoritative `key -> namespace` mapping and per-user isolation live in the
|
||||
//! hosted service; the client just presents an opaque sync key as a bearer
|
||||
//! token. The default local-first build never links an HTTP client.
|
||||
//!
|
||||
//! ## Concurrency
|
||||
//!
|
||||
//! Two devices can each pull → merge → push. To avoid a lost update in the
|
||||
//! GET↔PUT window, the backend uses optimistic concurrency: it captures the
|
||||
//! object `ETag` on read and sends it as `If-Match` on write. The generic
|
||||
//! [`sync_portable_archive`](super::sqlite::SqliteMemoryStore::sync_portable_archive)
|
||||
//! driver calls `read_archive` then `write_archive` exactly once, so the ETag
|
||||
//! captured during the pull is the precondition for the push. A
|
||||
//! `412 Precondition Failed` means another device wrote in between; the caller
|
||||
//! re-runs sync (the merge is idempotent and converges by `updated_at`).
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header::{AUTHORIZATION, ETAG, IF_MATCH};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use super::portable::PortableArchive;
|
||||
use super::sqlite::{PortableSyncBackend, Result, StorageError};
|
||||
|
||||
/// Default request timeout for cloud sync HTTP calls.
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Blob path on the hosted service. One opaque blob per sync key (the service
|
||||
/// derives the namespace from the key), so the client uses a fixed path.
|
||||
const BLOB_PATH: &str = "/v1/blob";
|
||||
|
||||
/// HTTP-backed portable sync backend for Vestige Cloud.
|
||||
///
|
||||
/// Mirrors the shape of
|
||||
/// [`FilePortableSyncBackend`](super::sqlite::FilePortableSyncBackend) but reads
|
||||
/// and writes the archive over HTTPS with a per-user bearer key.
|
||||
#[derive(Debug)]
|
||||
pub struct HttpPortableSyncBackend {
|
||||
/// Base endpoint, e.g. `https://sync.vestige.dev`. No trailing slash.
|
||||
endpoint: String,
|
||||
/// Per-user sync key, presented as `Authorization: Bearer <key>`.
|
||||
sync_key: String,
|
||||
/// Optional zero-knowledge passphrase. When set, the archive is encrypted
|
||||
/// before upload and decrypted after download — the server never sees
|
||||
/// plaintext, and this passphrase is never sent to the server.
|
||||
encryption_key: Option<String>,
|
||||
/// Blocking HTTP client (the trait is synchronous).
|
||||
client: Client,
|
||||
/// ETag captured on the most recent successful read, used as the `If-Match`
|
||||
/// precondition on the next write. `None` until the first read, or when the
|
||||
/// remote had no archive yet.
|
||||
last_etag: RefCell<Option<String>>,
|
||||
}
|
||||
|
||||
impl HttpPortableSyncBackend {
|
||||
/// Build a cloud sync backend for `endpoint` authenticated with `sync_key`,
|
||||
/// with no client-side encryption (plaintext upload).
|
||||
///
|
||||
/// A trailing slash on `endpoint` is trimmed so URL joining is predictable.
|
||||
pub fn new(endpoint: impl Into<String>, sync_key: impl Into<String>) -> Result<Self> {
|
||||
Self::new_with_encryption(endpoint, sync_key, None)
|
||||
}
|
||||
|
||||
/// Build a cloud sync backend with optional zero-knowledge encryption.
|
||||
///
|
||||
/// When `encryption_key` is `Some`, the portable archive is encrypted with
|
||||
/// XChaCha20-Poly1305 (Argon2id-derived key) before upload and decrypted on
|
||||
/// download. The passphrase never leaves this process.
|
||||
pub fn new_with_encryption(
|
||||
endpoint: impl Into<String>,
|
||||
sync_key: impl Into<String>,
|
||||
encryption_key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
let endpoint = endpoint.into().trim_end_matches('/').to_string();
|
||||
let sync_key = sync_key.into();
|
||||
if endpoint.is_empty() {
|
||||
return Err(StorageError::Init(
|
||||
"cloud sync endpoint is empty (set VESTIGE_CLOUD_ENDPOINT)".to_string(),
|
||||
));
|
||||
}
|
||||
if sync_key.is_empty() {
|
||||
return Err(StorageError::Init(
|
||||
"cloud sync key is empty (set VESTIGE_CLOUD_SYNC_KEY)".to_string(),
|
||||
));
|
||||
}
|
||||
let encryption_key = encryption_key.filter(|k| !k.is_empty());
|
||||
let client = Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.user_agent(concat!("vestige-cloud-sync/", env!("CARGO_PKG_VERSION")))
|
||||
.build()
|
||||
.map_err(|e| StorageError::Init(format!("failed to build HTTP client: {e}")))?;
|
||||
Ok(Self {
|
||||
endpoint,
|
||||
sync_key,
|
||||
encryption_key,
|
||||
client,
|
||||
last_etag: RefCell::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether this backend encrypts client-side (zero-knowledge).
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
self.encryption_key.is_some()
|
||||
}
|
||||
|
||||
/// Full blob URL for this backend.
|
||||
fn blob_url(&self) -> String {
|
||||
format!("{}{}", self.endpoint, BLOB_PATH)
|
||||
}
|
||||
}
|
||||
|
||||
impl PortableSyncBackend for HttpPortableSyncBackend {
|
||||
fn label(&self) -> String {
|
||||
format!("cloud:{}", self.endpoint)
|
||||
}
|
||||
|
||||
fn read_archive(&self) -> Result<Option<PortableArchive>> {
|
||||
let resp = self
|
||||
.client
|
||||
.get(self.blob_url())
|
||||
.header(AUTHORIZATION, format!("Bearer {}", self.sync_key))
|
||||
.send()
|
||||
.map_err(|e| StorageError::Init(format!("cloud sync read failed: {e}")))?;
|
||||
|
||||
match resp.status() {
|
||||
StatusCode::NOT_FOUND => {
|
||||
// No remote archive yet — first sync for this key.
|
||||
*self.last_etag.borrow_mut() = None;
|
||||
Ok(None)
|
||||
}
|
||||
StatusCode::OK => {
|
||||
// Capture the ETag for the matching If-Match write.
|
||||
let etag = resp
|
||||
.headers()
|
||||
.get(ETAG)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
*self.last_etag.borrow_mut() = etag;
|
||||
|
||||
let bytes = resp
|
||||
.bytes()
|
||||
.map_err(|e| StorageError::Init(format!("cloud sync read body failed: {e}")))?;
|
||||
|
||||
// Decrypt if this is a zero-knowledge envelope. If a passphrase
|
||||
// is configured but the remote is still plaintext (legacy), parse
|
||||
// it directly — the next push will encrypt it (transparent upgrade).
|
||||
let plaintext: std::borrow::Cow<'_, [u8]> =
|
||||
if super::cloud_crypto::is_encrypted(&bytes) {
|
||||
let pass = self.encryption_key.as_deref().ok_or_else(|| {
|
||||
StorageError::Init(
|
||||
"remote archive is encrypted but VESTIGE_CLOUD_ENCRYPTION_KEY is \
|
||||
not set on this device"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
std::borrow::Cow::Owned(super::cloud_crypto::decrypt(pass, &bytes)?)
|
||||
} else {
|
||||
std::borrow::Cow::Borrowed(&bytes)
|
||||
};
|
||||
|
||||
let archive: PortableArchive =
|
||||
serde_json::from_slice(&plaintext).map_err(|e| {
|
||||
StorageError::Init(format!("failed to parse cloud sync archive: {e}"))
|
||||
})?;
|
||||
Ok(Some(archive))
|
||||
}
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(StorageError::Init(
|
||||
"cloud sync rejected the sync key (401/403). Check your subscription and \
|
||||
VESTIGE_CLOUD_SYNC_KEY."
|
||||
.to_string(),
|
||||
)),
|
||||
other => Err(StorageError::Init(format!(
|
||||
"cloud sync read returned unexpected status {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_archive(&self, archive: &PortableArchive) -> Result<()> {
|
||||
let plaintext = serde_json::to_vec(archive)
|
||||
.map_err(|e| StorageError::Init(format!("failed to serialize archive: {e}")))?;
|
||||
|
||||
// Zero-knowledge: encrypt before upload when a passphrase is set, so the
|
||||
// server only ever stores ciphertext. Content type reflects the payload.
|
||||
let (body, content_type) = match self.encryption_key.as_deref() {
|
||||
Some(pass) => (
|
||||
super::cloud_crypto::encrypt(pass, &plaintext)?,
|
||||
"application/octet-stream",
|
||||
),
|
||||
None => (plaintext, "application/json"),
|
||||
};
|
||||
|
||||
let mut req = self
|
||||
.client
|
||||
.put(self.blob_url())
|
||||
.header(AUTHORIZATION, format!("Bearer {}", self.sync_key))
|
||||
.header(reqwest::header::CONTENT_TYPE, content_type)
|
||||
.body(body);
|
||||
|
||||
// Optimistic concurrency: only overwrite the object we pulled. If the
|
||||
// remote had no archive, require that it still doesn't exist (`If-Match: *`
|
||||
// would require existence, so we omit the header to allow first create).
|
||||
if let Some(etag) = self.last_etag.borrow_mut().take() {
|
||||
req = req.header(IF_MATCH, etag);
|
||||
}
|
||||
|
||||
let resp = req
|
||||
.send()
|
||||
.map_err(|e| StorageError::Init(format!("cloud sync write failed: {e}")))?;
|
||||
|
||||
match resp.status() {
|
||||
StatusCode::OK | StatusCode::CREATED | StatusCode::NO_CONTENT => Ok(()),
|
||||
StatusCode::PRECONDITION_FAILED => Err(StorageError::Init(
|
||||
"cloud sync conflict: another device updated your memory in between. \
|
||||
Run `vestige sync --cloud` again to merge and retry."
|
||||
.to_string(),
|
||||
)),
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(StorageError::Init(
|
||||
"cloud sync rejected the sync key (401/403). Check your subscription and \
|
||||
VESTIGE_CLOUD_SYNC_KEY."
|
||||
.to_string(),
|
||||
)),
|
||||
StatusCode::PAYLOAD_TOO_LARGE => Err(StorageError::Init(
|
||||
"cloud sync archive too large for the hosted plan limit".to_string(),
|
||||
)),
|
||||
other => Err(StorageError::Init(format!(
|
||||
"cloud sync write returned unexpected status {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use super::super::portable::{PortableArchive, PortableTable, PORTABLE_ARCHIVE_FORMAT};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
|
||||
fn sample_archive() -> PortableArchive {
|
||||
PortableArchive {
|
||||
archive_format: PORTABLE_ARCHIVE_FORMAT.to_string(),
|
||||
vestige_version: "test".to_string(),
|
||||
schema_version: 1,
|
||||
exported_at: chrono::Utc::now(),
|
||||
mode: "exact".to_string(),
|
||||
tables: vec![PortableTable {
|
||||
name: "knowledge_nodes".to_string(),
|
||||
columns: vec!["id".to_string()],
|
||||
rows: vec![],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// A captured request the mock observed, surfaced to the test thread.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct CapturedRequest {
|
||||
method: String,
|
||||
authorization: Option<String>,
|
||||
if_match: Option<String>,
|
||||
}
|
||||
|
||||
/// Minimal one-shot HTTP mock. `responder` builds the raw HTTP response
|
||||
/// string for the request line + headers it parsed. Returns the bound base
|
||||
/// URL and a receiver for the captured request.
|
||||
fn spawn_mock<F>(responder: F) -> (String, mpsc::Receiver<CapturedRequest>)
|
||||
where
|
||||
F: Fn(&CapturedRequest) -> String + Send + 'static,
|
||||
{
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
let (tx, rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
if let Ok((mut stream, _)) = listener.accept() {
|
||||
let mut buf = [0u8; 8192];
|
||||
let n = stream.read(&mut buf).unwrap_or(0);
|
||||
let text = String::from_utf8_lossy(&buf[..n]);
|
||||
let mut cap = CapturedRequest::default();
|
||||
for (i, line) in text.lines().enumerate() {
|
||||
if i == 0 {
|
||||
cap.method = line.split_whitespace().next().unwrap_or("").to_string();
|
||||
} else if let Some(v) = line.strip_prefix("authorization: ") {
|
||||
cap.authorization = Some(v.trim().to_string());
|
||||
} else if let Some(v) = line.strip_prefix("if-match: ") {
|
||||
cap.if_match = Some(v.trim().to_string());
|
||||
}
|
||||
}
|
||||
let response = responder(&cap);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
let _ = stream.flush();
|
||||
let _ = tx.send(cap);
|
||||
}
|
||||
});
|
||||
(format!("http://{addr}"), rx)
|
||||
}
|
||||
|
||||
fn http_response(status: &str, extra_headers: &str, body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Length: {}\r\n{extra_headers}Connection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_rejects_empty_endpoint_and_key() {
|
||||
assert!(HttpPortableSyncBackend::new("", "key").is_err());
|
||||
assert!(HttpPortableSyncBackend::new("https://x", "").is_err());
|
||||
assert!(HttpPortableSyncBackend::new("https://x", "key").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_trailing_slash_trimmed() {
|
||||
let be = HttpPortableSyncBackend::new("https://sync.example/", "k").unwrap();
|
||||
assert_eq!(be.blob_url(), "https://sync.example/v1/blob");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_404_returns_none() {
|
||||
let (base, rx) = spawn_mock(|_| http_response("404 Not Found", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
let got = be.read_archive().expect("read ok");
|
||||
assert!(got.is_none());
|
||||
let cap = rx.recv().unwrap();
|
||||
assert_eq!(cap.method, "GET");
|
||||
assert_eq!(cap.authorization.as_deref(), Some("Bearer secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_200_parses_and_captures_etag() {
|
||||
let archive = sample_archive();
|
||||
let body = serde_json::to_string(&archive).unwrap();
|
||||
let (base, _rx) = spawn_mock(move |_| {
|
||||
http_response("200 OK", "ETag: \"v1-abc\"\r\n", &body)
|
||||
});
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
let got = be.read_archive().expect("read ok").expect("some archive");
|
||||
assert_eq!(got.archive_format, PORTABLE_ARCHIVE_FORMAT);
|
||||
// ETag captured for the next If-Match write.
|
||||
assert_eq!(be.last_etag.borrow().as_deref(), Some("\"v1-abc\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_401_is_error() {
|
||||
let (base, _rx) = spawn_mock(|_| http_response("401 Unauthorized", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "bad").unwrap();
|
||||
assert!(be.read_archive().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_sends_if_match_when_etag_present() {
|
||||
// Seed an etag as if a prior read happened.
|
||||
let (base, rx) = spawn_mock(|_| http_response("200 OK", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
*be.last_etag.borrow_mut() = Some("\"v1-abc\"".to_string());
|
||||
be.write_archive(&sample_archive()).expect("write ok");
|
||||
let cap = rx.recv().unwrap();
|
||||
assert_eq!(cap.method, "PUT");
|
||||
assert_eq!(cap.authorization.as_deref(), Some("Bearer secret"));
|
||||
assert_eq!(cap.if_match.as_deref(), Some("\"v1-abc\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_omits_if_match_for_first_create() {
|
||||
let (base, rx) = spawn_mock(|_| http_response("201 Created", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
// No prior read → no etag → no If-Match (allow create).
|
||||
be.write_archive(&sample_archive()).expect("write ok");
|
||||
let cap = rx.recv().unwrap();
|
||||
assert_eq!(cap.method, "PUT");
|
||||
assert!(cap.if_match.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_412_is_conflict_error() {
|
||||
let (base, _rx) = spawn_mock(|_| http_response("412 Precondition Failed", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
*be.last_etag.borrow_mut() = Some("\"stale\"".to_string());
|
||||
let err = be.write_archive(&sample_archive()).unwrap_err();
|
||||
assert!(err.to_string().contains("conflict"));
|
||||
}
|
||||
}
|
||||
|
|
@ -2,12 +2,19 @@
|
|||
//!
|
||||
//! Backend-agnostic memory store abstraction plus SQLite reference impl.
|
||||
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
mod cloud_crypto;
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
mod cloud_sync;
|
||||
mod memory_store;
|
||||
mod migrations;
|
||||
mod portable;
|
||||
mod sqlite;
|
||||
mod trace_store;
|
||||
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
pub use cloud_sync::HttpPortableSyncBackend;
|
||||
|
||||
pub use memory_store::{
|
||||
ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord,
|
||||
MemoryStore, MemoryStoreError, MemoryStoreResult, MemoryStoreSend, ModelSignature,
|
||||
|
|
|
|||
|
|
@ -6390,6 +6390,29 @@ impl SqliteMemoryStore {
|
|||
self.sync_portable_archive(&backend)
|
||||
}
|
||||
|
||||
/// Synchronize this database with the hosted Vestige Cloud managed-sync
|
||||
/// service. `endpoint` is the base URL (e.g. `https://sync.vestige.dev`) and
|
||||
/// `sync_key` is the per-user key issued at purchase. Pull-merge-push is
|
||||
/// identical to file sync — only the transport differs.
|
||||
///
|
||||
/// When `encryption_key` is `Some`, the archive is encrypted client-side
|
||||
/// (XChaCha20-Poly1305) before upload, so the server only stores ciphertext
|
||||
/// (zero-knowledge). The passphrase never leaves this process.
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
pub fn sync_portable_archive_cloud(
|
||||
&self,
|
||||
endpoint: &str,
|
||||
sync_key: &str,
|
||||
encryption_key: Option<String>,
|
||||
) -> Result<PortableSyncReport> {
|
||||
let backend = super::cloud_sync::HttpPortableSyncBackend::new_with_encryption(
|
||||
endpoint,
|
||||
sync_key,
|
||||
encryption_key,
|
||||
)?;
|
||||
self.sync_portable_archive(&backend)
|
||||
}
|
||||
|
||||
fn merge_portable_table(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
table_name: &str,
|
||||
|
|
|
|||
|
|
@ -10,13 +10,18 @@ categories = ["command-line-utilities", "database"]
|
|||
repository = "https://github.com/samvallad33/vestige"
|
||||
|
||||
[features]
|
||||
default = ["embeddings", "ort-download", "vector-search", "connectors"]
|
||||
default = ["embeddings", "ort-download", "vector-search", "connectors", "cloud-sync"]
|
||||
embeddings = ["vestige-core/embeddings"]
|
||||
vector-search = ["vestige-core/vector-search"]
|
||||
# External-source connectors (#57): GitHub Issues / Redmine indexing via the
|
||||
# `source_sync` MCP tool. On by default so the tool works out of the box; turn
|
||||
# off for a build with no HTTP client.
|
||||
connectors = ["vestige-core/connectors"]
|
||||
# Hosted managed-sync (Vestige Cloud): `vestige sync --cloud` pushes/pulls the
|
||||
# portable archive to the hosted blob service. On by default so the command
|
||||
# works out of the box; the binary already links an HTTP client via
|
||||
# `connectors`, so this adds no new dependency cost.
|
||||
cloud-sync = ["vestige-core/cloud-sync"]
|
||||
# Default ort backend: downloads prebuilt ONNX Runtime at build time.
|
||||
# Fails on targets without prebuilts (notably x86_64-apple-darwin).
|
||||
ort-download = ["embeddings", "vestige-core/ort-download"]
|
||||
|
|
|
|||
|
|
@ -182,10 +182,20 @@ enum Commands {
|
|||
merge: bool,
|
||||
},
|
||||
|
||||
/// Two-way sync with a file-backed portable archive
|
||||
/// Two-way sync with a file-backed portable archive, or Vestige Cloud
|
||||
Sync {
|
||||
/// Sync archive path, often in Dropbox/iCloud/Syncthing/Git
|
||||
archive: PathBuf,
|
||||
/// Sync archive path, often in Dropbox/iCloud/Syncthing/Git.
|
||||
/// Omit when using --cloud.
|
||||
archive: Option<PathBuf>,
|
||||
/// Sync with the hosted Vestige Cloud managed-sync service instead of a
|
||||
/// file. Requires a sync key (VESTIGE_CLOUD_SYNC_KEY) and endpoint
|
||||
/// (--endpoint or VESTIGE_CLOUD_ENDPOINT).
|
||||
#[arg(long)]
|
||||
cloud: bool,
|
||||
/// Vestige Cloud base endpoint (e.g. https://sync.vestige.dev).
|
||||
/// Defaults to the VESTIGE_CLOUD_ENDPOINT env var.
|
||||
#[arg(long)]
|
||||
endpoint: Option<String>,
|
||||
},
|
||||
|
||||
/// Garbage collect stale memories below retention threshold
|
||||
|
|
@ -287,7 +297,11 @@ fn main() -> anyhow::Result<()> {
|
|||
} => run_export(output, format, tags, since),
|
||||
Commands::PortableExport { output } => run_portable_export(output),
|
||||
Commands::PortableImport { input, merge } => run_portable_import(input, merge),
|
||||
Commands::Sync { archive } => run_sync(archive),
|
||||
Commands::Sync {
|
||||
archive,
|
||||
cloud,
|
||||
endpoint,
|
||||
} => run_sync(archive, cloud, endpoint),
|
||||
Commands::Gc {
|
||||
min_retention,
|
||||
max_age_days,
|
||||
|
|
@ -2193,14 +2207,92 @@ fn run_portable_import(input: PathBuf, merge: bool) -> anyhow::Result<()> {
|
|||
}
|
||||
|
||||
/// Run file-backed two-way sync.
|
||||
fn run_sync(archive: PathBuf) -> anyhow::Result<()> {
|
||||
fn run_sync(
|
||||
archive: Option<PathBuf>,
|
||||
cloud: bool,
|
||||
endpoint: Option<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
if cloud {
|
||||
run_sync_cloud(endpoint)
|
||||
} else {
|
||||
let archive = archive.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"no sync target: pass an archive path for file sync, or --cloud for Vestige Cloud"
|
||||
)
|
||||
})?;
|
||||
run_sync_file(archive)
|
||||
}
|
||||
}
|
||||
|
||||
fn run_sync_file(archive: PathBuf) -> anyhow::Result<()> {
|
||||
println!("{}", "=== Vestige File Sync ===".cyan().bold());
|
||||
println!();
|
||||
println!("{}: {}", "Archive".white().bold(), archive.display());
|
||||
|
||||
let storage = open_storage()?;
|
||||
let report = storage.sync_portable_archive_file(&archive)?;
|
||||
print_sync_report(&report);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
fn run_sync_cloud(endpoint: Option<String>) -> anyhow::Result<()> {
|
||||
let endpoint = endpoint
|
||||
.or_else(|| std::env::var("VESTIGE_CLOUD_ENDPOINT").ok())
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"no cloud endpoint: pass --endpoint or set VESTIGE_CLOUD_ENDPOINT \
|
||||
(e.g. https://sync.vestige.dev)"
|
||||
)
|
||||
})?;
|
||||
let sync_key = std::env::var("VESTIGE_CLOUD_SYNC_KEY")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"no sync key: set VESTIGE_CLOUD_SYNC_KEY (issued when you subscribe to \
|
||||
Vestige Cloud)"
|
||||
)
|
||||
})?;
|
||||
|
||||
// Optional zero-knowledge encryption passphrase. Never sent to the server.
|
||||
let encryption_key = std::env::var("VESTIGE_CLOUD_ENCRYPTION_KEY")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
|
||||
println!("{}", "=== Vestige Cloud Sync ===".cyan().bold());
|
||||
println!();
|
||||
println!("{}: {}", "Endpoint".white().bold(), endpoint);
|
||||
if encryption_key.is_some() {
|
||||
println!(
|
||||
"{}: {}",
|
||||
"Encryption".white().bold(),
|
||||
"zero-knowledge (XChaCha20-Poly1305) — your data is encrypted before upload".green()
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
"{}: {}",
|
||||
"Encryption".white().bold(),
|
||||
"OFF — set VESTIGE_CLOUD_ENCRYPTION_KEY for zero-knowledge sync".yellow()
|
||||
);
|
||||
}
|
||||
|
||||
let storage = open_storage()?;
|
||||
let report = storage.sync_portable_archive_cloud(&endpoint, &sync_key, encryption_key)?;
|
||||
print_sync_report(&report);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "cloud-sync"))]
|
||||
fn run_sync_cloud(_endpoint: Option<String>) -> anyhow::Result<()> {
|
||||
anyhow::bail!(
|
||||
"this build was compiled without the `cloud-sync` feature; rebuild with \
|
||||
--features cloud-sync to use Vestige Cloud"
|
||||
)
|
||||
}
|
||||
|
||||
fn print_sync_report(report: &vestige_core::PortableSyncReport) {
|
||||
if let Some(pull) = &report.pull {
|
||||
println!("{}", "Pull: merged remote archive".yellow());
|
||||
println!(
|
||||
|
|
@ -2228,8 +2320,6 @@ fn run_sync(archive: PathBuf) -> anyhow::Result<()> {
|
|||
.green()
|
||||
.bold()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run garbage collection command
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue