mirror of
https://github.com/samvallad33/vestige.git
synced 2026-07-24 23:41:01 +02:00
merge: cloud sync pro — encrypted managed sync (lane 5/5)
This commit is contained in:
Sam Valladares
commit
335a42f341
11 changed files with 945 additions and 11 deletions
25
.github/workflows/guard-no-private-cloud.yml
vendored
Normal file
25
.github/workflows/guard-no-private-cloud.yml
vendored
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
name: Guard — No Private Cloud Code
|
||||
|
||||
# Fails if private Vestige Cloud *service* code (billing, sync-key/namespace
|
||||
# mapping, Lemon Squeezy webhooks, transactional email) ever lands in this
|
||||
# public repo. The public cloud *client* is allowed and does not trip this.
|
||||
on:
|
||||
push:
|
||||
branches: [main, feat/cloud-sync-mvp]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
guard:
|
||||
name: No private cloud service code
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Scan for private cloud service markers
|
||||
run: ./scripts/check-no-private-cloud.sh
|
||||
134
Cargo.lock
generated
134
Cargo.lock
generated
|
|
@ -8,6 +8,16 @@ version = "2.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aead"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
|
|
@ -143,6 +153,18 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "argon2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"blake2",
|
||||
"cpufeatures 0.2.17",
|
||||
"password-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayref"
|
||||
version = "0.3.9"
|
||||
|
|
@ -317,6 +339,15 @@ dependencies = [
|
|||
"core2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake2"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake3"
|
||||
version = "1.8.4"
|
||||
|
|
@ -527,6 +558,30 @@ version = "0.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chacha20"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher",
|
||||
"cpufeatures 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chacha20poly1305"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"chacha20",
|
||||
"cipher",
|
||||
"poly1305",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.44"
|
||||
|
|
@ -568,6 +623,17 @@ dependencies = [
|
|||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"inout",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.0"
|
||||
|
|
@ -821,6 +887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"rand_core 0.6.4",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
|
|
@ -1025,6 +1092,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1379,6 +1447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2229,6 +2298,15 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "interpolate_name"
|
||||
version = "0.2.4"
|
||||
|
|
@ -3012,6 +3090,12 @@ version = "11.1.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "open"
|
||||
version = "5.3.3"
|
||||
|
|
@ -3137,6 +3221,17 @@ dependencies = [
|
|||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "password-hash"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.15"
|
||||
|
|
@ -3223,6 +3318,17 @@ dependencies = [
|
|||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "poly1305"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
|
||||
dependencies = [
|
||||
"cpufeatures 0.2.17",
|
||||
"opaque-debug",
|
||||
"universal-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.13.1"
|
||||
|
|
@ -3404,7 +3510,7 @@ dependencies = [
|
|||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3435,7 +3541,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3445,7 +3551,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3627,6 +3742,7 @@ dependencies = [
|
|||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
|
|
@ -4626,6 +4742,16 @@ version = "0.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
|
||||
|
||||
[[package]]
|
||||
name = "universal-hash"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
|
|
@ -4773,8 +4899,10 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
|||
name = "vestige-core"
|
||||
version = "2.1.27"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"blake3",
|
||||
"candle-core",
|
||||
"chacha20poly1305",
|
||||
"chrono",
|
||||
"criterion",
|
||||
"directories",
|
||||
|
|
|
|||
|
|
@ -63,6 +63,14 @@ qwen3-reranker = ["qwen3-embeddings"]
|
|||
# the default local-first build never links an HTTP client.
|
||||
connectors = ["dep:reqwest"]
|
||||
|
||||
# Hosted managed-sync backend (Vestige Cloud). Adds the HTTP `PortableSyncBackend`
|
||||
# (`HttpPortableSyncBackend`) that pull-merge-pushes the portable archive to a
|
||||
# hosted blob endpoint over HTTPS with a per-user sync key. Like `connectors`,
|
||||
# this is the ONLY thing that links an HTTP client — the default local-first
|
||||
# build stays network-free. Uses `reqwest`'s blocking client because the
|
||||
# `PortableSyncBackend` trait methods are synchronous.
|
||||
cloud-sync = ["dep:reqwest", "reqwest/blocking", "dep:chacha20poly1305", "dep:argon2"]
|
||||
|
||||
# Metal GPU acceleration on Apple Silicon (significantly faster inference)
|
||||
metal = ["fastembed/metal"]
|
||||
|
||||
|
|
@ -157,6 +165,15 @@ blake3 = "1"
|
|||
# `connectors` feature — the default local-first build does not link reqwest.
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true }
|
||||
|
||||
# ============================================================================
|
||||
# OPTIONAL: Zero-knowledge client-side encryption for Vestige Cloud sync
|
||||
# ============================================================================
|
||||
# XChaCha20-Poly1305 AEAD + Argon2id passphrase KDF. The archive is encrypted on
|
||||
# the client before upload, so the hosted service only ever stores ciphertext.
|
||||
# Behind the `cloud-sync` feature — the default local-first build links neither.
|
||||
chacha20poly1305 = { version = "0.10", optional = true }
|
||||
argon2 = { version = "0.5", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
criterion = { version = "0.5", features = ["html_reports"] }
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ pub use storage::{
|
|||
PortableArchive,
|
||||
PortableImportMode,
|
||||
PortableImportReport,
|
||||
PortableSyncReport,
|
||||
ReconcileReport,
|
||||
Result,
|
||||
SchedulingState,
|
||||
|
|
|
|||
164
crates/vestige-core/src/storage/cloud_crypto.rs
Normal file
164
crates/vestige-core/src/storage/cloud_crypto.rs
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
//! Zero-knowledge client-side encryption for Vestige Cloud sync.
|
||||
//!
|
||||
//! The portable archive is encrypted on the client **before** it is uploaded
|
||||
//! and decrypted **after** it is downloaded, so the hosted service only ever
|
||||
//! stores ciphertext. The encryption passphrase is supplied by the user
|
||||
//! (`VESTIGE_CLOUD_ENCRYPTION_KEY`) and is **never** sent to the server — it is
|
||||
//! independent of the bearer sync key. This is what makes the "we hold no keys"
|
||||
//! guarantee literally true: a server breach yields only random noise.
|
||||
//!
|
||||
//! Construction:
|
||||
//! - KDF: Argon2id over (passphrase, random 16-byte salt) → 32-byte key.
|
||||
//! - AEAD: XChaCha20-Poly1305 (192-bit nonce) over the archive bytes.
|
||||
//! - Envelope (all non-secret framing prepended to the ciphertext):
|
||||
//! `MAGIC(8) | VERSION(1) | salt(16) | nonce(24) | ciphertext+tag`
|
||||
//!
|
||||
//! Tradeoff (by design, and a selling point): if the user loses the passphrase,
|
||||
//! the synced data is unrecoverable. We cannot reset it — we never have it.
|
||||
|
||||
use argon2::Argon2;
|
||||
use chacha20poly1305::aead::rand_core::RngCore;
|
||||
use chacha20poly1305::aead::{Aead, AeadCore, KeyInit, OsRng};
|
||||
use chacha20poly1305::{Key, XChaCha20Poly1305, XNonce};
|
||||
|
||||
use super::sqlite::{Result, StorageError};
|
||||
|
||||
/// Magic marker identifying a Vestige zero-knowledge envelope.
|
||||
const MAGIC: &[u8; 8] = b"VSTGENC1";
|
||||
const VERSION: u8 = 1;
|
||||
const SALT_LEN: usize = 16;
|
||||
const NONCE_LEN: usize = 24; // XChaCha20-Poly1305 nonce is 192-bit.
|
||||
const KEY_LEN: usize = 32;
|
||||
const HEADER_LEN: usize = MAGIC.len() + 1 + SALT_LEN + NONCE_LEN;
|
||||
|
||||
/// Derive a 32-byte key from the passphrase and salt using Argon2id (defaults).
|
||||
fn derive_key(passphrase: &[u8], salt: &[u8]) -> Result<[u8; KEY_LEN]> {
|
||||
let mut key = [0u8; KEY_LEN];
|
||||
Argon2::default()
|
||||
.hash_password_into(passphrase, salt, &mut key)
|
||||
.map_err(|e| StorageError::Init(format!("key derivation failed: {e}")))?;
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
/// Encrypt `plaintext` under `passphrase`, returning the self-describing envelope.
|
||||
///
|
||||
/// A fresh random salt and nonce are generated per call, so re-encrypting the
|
||||
/// same archive yields different ciphertext (no deterministic leakage).
|
||||
pub fn encrypt(passphrase: &str, plaintext: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut salt = [0u8; SALT_LEN];
|
||||
OsRng.fill_bytes(&mut salt);
|
||||
|
||||
let key_bytes = derive_key(passphrase.as_bytes(), &salt)?;
|
||||
let cipher = XChaCha20Poly1305::new(Key::from_slice(&key_bytes));
|
||||
let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng);
|
||||
|
||||
let ciphertext = cipher
|
||||
.encrypt(&nonce, plaintext)
|
||||
.map_err(|_| StorageError::Init("encryption failed".to_string()))?;
|
||||
|
||||
let mut out = Vec::with_capacity(HEADER_LEN + ciphertext.len());
|
||||
out.extend_from_slice(MAGIC);
|
||||
out.push(VERSION);
|
||||
out.extend_from_slice(&salt);
|
||||
out.extend_from_slice(nonce.as_slice());
|
||||
out.extend_from_slice(&ciphertext);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// True if `bytes` start with the Vestige encryption magic. Lets the sync
|
||||
/// backend distinguish an encrypted envelope from a legacy plaintext archive.
|
||||
pub fn is_encrypted(bytes: &[u8]) -> bool {
|
||||
bytes.len() >= MAGIC.len() && &bytes[..MAGIC.len()] == MAGIC
|
||||
}
|
||||
|
||||
/// Decrypt a Vestige envelope under `passphrase`. Fails on a wrong passphrase or
|
||||
/// any tampering (the Poly1305 tag is verified).
|
||||
pub fn decrypt(passphrase: &str, envelope: &[u8]) -> Result<Vec<u8>> {
|
||||
if envelope.len() < HEADER_LEN {
|
||||
return Err(StorageError::Init(
|
||||
"cloud archive is too short to be a valid encrypted envelope".to_string(),
|
||||
));
|
||||
}
|
||||
if &envelope[..MAGIC.len()] != MAGIC {
|
||||
return Err(StorageError::Init(
|
||||
"cloud archive is not a Vestige encrypted envelope".to_string(),
|
||||
));
|
||||
}
|
||||
let version = envelope[MAGIC.len()];
|
||||
if version != VERSION {
|
||||
return Err(StorageError::Init(format!(
|
||||
"unsupported cloud encryption version {version}"
|
||||
)));
|
||||
}
|
||||
|
||||
let salt_start = MAGIC.len() + 1;
|
||||
let nonce_start = salt_start + SALT_LEN;
|
||||
let ct_start = nonce_start + NONCE_LEN;
|
||||
let salt = &envelope[salt_start..nonce_start];
|
||||
let nonce = XNonce::from_slice(&envelope[nonce_start..ct_start]);
|
||||
let ciphertext = &envelope[ct_start..];
|
||||
|
||||
let key_bytes = derive_key(passphrase.as_bytes(), salt)?;
|
||||
let cipher = XChaCha20Poly1305::new(Key::from_slice(&key_bytes));
|
||||
|
||||
cipher.decrypt(nonce, ciphertext).map_err(|_| {
|
||||
StorageError::Init(
|
||||
"cloud decryption failed: wrong VESTIGE_CLOUD_ENCRYPTION_KEY or corrupted data"
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn round_trip() {
|
||||
let pass = "correct horse battery staple";
|
||||
let msg = b"the entire cognitive graph, in plaintext, before upload";
|
||||
let env = encrypt(pass, msg).unwrap();
|
||||
assert!(is_encrypted(&env));
|
||||
assert_ne!(&env[..], &msg[..], "envelope must not contain plaintext");
|
||||
let back = decrypt(pass, &env).unwrap();
|
||||
assert_eq!(back, msg);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrong_passphrase_fails() {
|
||||
let env = encrypt("right-pass", b"secret").unwrap();
|
||||
assert!(decrypt("wrong-pass", &env).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tamper_is_detected() {
|
||||
let mut env = encrypt("pass", b"important memory").unwrap();
|
||||
// Flip a byte in the ciphertext region.
|
||||
let last = env.len() - 1;
|
||||
env[last] ^= 0xff;
|
||||
assert!(decrypt("pass", &env).is_err(), "AEAD must reject tampering");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ciphertext_is_nondeterministic() {
|
||||
// Same input encrypted twice → different envelopes (random salt+nonce).
|
||||
let a = encrypt("p", b"x").unwrap();
|
||||
let b = encrypt("p", b"x").unwrap();
|
||||
assert_ne!(a, b);
|
||||
// Both still decrypt correctly.
|
||||
assert_eq!(decrypt("p", &a).unwrap(), b"x");
|
||||
assert_eq!(decrypt("p", &b).unwrap(), b"x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plaintext_is_not_misdetected_as_envelope() {
|
||||
assert!(!is_encrypted(b"{\"archiveFormat\":\"vestige.portable.v1\"}"));
|
||||
assert!(!is_encrypted(b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_short_or_foreign_envelope() {
|
||||
assert!(decrypt("p", b"too short").is_err());
|
||||
assert!(decrypt("p", &[0u8; 100]).is_err());
|
||||
}
|
||||
}
|
||||
391
crates/vestige-core/src/storage/cloud_sync.rs
Normal file
391
crates/vestige-core/src/storage/cloud_sync.rs
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
//! Hosted managed-sync backend (Vestige Cloud).
|
||||
//!
|
||||
//! This module is only compiled with the `cloud-sync` feature. It provides
|
||||
//! [`HttpPortableSyncBackend`], an HTTP implementation of the
|
||||
//! [`PortableSyncBackend`](super::sqlite::PortableSyncBackend) trait that
|
||||
//! pull-merge-pushes the portable archive to a hosted blob endpoint.
|
||||
//!
|
||||
//! The merge/conflict engine is unchanged: this backend only moves bytes. The
|
||||
//! authoritative `key -> namespace` mapping and per-user isolation live in the
|
||||
//! hosted service; the client just presents an opaque sync key as a bearer
|
||||
//! token. The default local-first build never links an HTTP client.
|
||||
//!
|
||||
//! ## Concurrency
|
||||
//!
|
||||
//! Two devices can each pull → merge → push. To avoid a lost update in the
|
||||
//! GET↔PUT window, the backend uses optimistic concurrency: it captures the
|
||||
//! object `ETag` on read and sends it as `If-Match` on write. The generic
|
||||
//! [`sync_portable_archive`](super::sqlite::SqliteMemoryStore::sync_portable_archive)
|
||||
//! driver calls `read_archive` then `write_archive` exactly once, so the ETag
|
||||
//! captured during the pull is the precondition for the push. A
|
||||
//! `412 Precondition Failed` means another device wrote in between; the caller
|
||||
//! re-runs sync (the merge is idempotent and converges by `updated_at`).
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header::{AUTHORIZATION, ETAG, IF_MATCH};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use super::portable::PortableArchive;
|
||||
use super::sqlite::{PortableSyncBackend, Result, StorageError};
|
||||
|
||||
/// Default request timeout for cloud sync HTTP calls.
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Blob path on the hosted service. One opaque blob per sync key (the service
|
||||
/// derives the namespace from the key), so the client uses a fixed path.
|
||||
const BLOB_PATH: &str = "/v1/blob";
|
||||
|
||||
/// HTTP-backed portable sync backend for Vestige Cloud.
|
||||
///
|
||||
/// Mirrors the shape of
|
||||
/// [`FilePortableSyncBackend`](super::sqlite::FilePortableSyncBackend) but reads
|
||||
/// and writes the archive over HTTPS with a per-user bearer key.
|
||||
#[derive(Debug)]
|
||||
pub struct HttpPortableSyncBackend {
|
||||
/// Base endpoint, e.g. `https://sync.vestige.dev`. No trailing slash.
|
||||
endpoint: String,
|
||||
/// Per-user sync key, presented as `Authorization: Bearer <key>`.
|
||||
sync_key: String,
|
||||
/// Optional zero-knowledge passphrase. When set, the archive is encrypted
|
||||
/// before upload and decrypted after download — the server never sees
|
||||
/// plaintext, and this passphrase is never sent to the server.
|
||||
encryption_key: Option<String>,
|
||||
/// Blocking HTTP client (the trait is synchronous).
|
||||
client: Client,
|
||||
/// ETag captured on the most recent successful read, used as the `If-Match`
|
||||
/// precondition on the next write. `None` until the first read, or when the
|
||||
/// remote had no archive yet.
|
||||
last_etag: RefCell<Option<String>>,
|
||||
}
|
||||
|
||||
impl HttpPortableSyncBackend {
|
||||
/// Build a cloud sync backend for `endpoint` authenticated with `sync_key`,
|
||||
/// with no client-side encryption (plaintext upload).
|
||||
///
|
||||
/// A trailing slash on `endpoint` is trimmed so URL joining is predictable.
|
||||
pub fn new(endpoint: impl Into<String>, sync_key: impl Into<String>) -> Result<Self> {
|
||||
Self::new_with_encryption(endpoint, sync_key, None)
|
||||
}
|
||||
|
||||
/// Build a cloud sync backend with optional zero-knowledge encryption.
|
||||
///
|
||||
/// When `encryption_key` is `Some`, the portable archive is encrypted with
|
||||
/// XChaCha20-Poly1305 (Argon2id-derived key) before upload and decrypted on
|
||||
/// download. The passphrase never leaves this process.
|
||||
pub fn new_with_encryption(
|
||||
endpoint: impl Into<String>,
|
||||
sync_key: impl Into<String>,
|
||||
encryption_key: Option<String>,
|
||||
) -> Result<Self> {
|
||||
let endpoint = endpoint.into().trim_end_matches('/').to_string();
|
||||
let sync_key = sync_key.into();
|
||||
if endpoint.is_empty() {
|
||||
return Err(StorageError::Init(
|
||||
"cloud sync endpoint is empty (set VESTIGE_CLOUD_ENDPOINT)".to_string(),
|
||||
));
|
||||
}
|
||||
if sync_key.is_empty() {
|
||||
return Err(StorageError::Init(
|
||||
"cloud sync key is empty (set VESTIGE_CLOUD_SYNC_KEY)".to_string(),
|
||||
));
|
||||
}
|
||||
let encryption_key = encryption_key.filter(|k| !k.is_empty());
|
||||
let client = Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.user_agent(concat!("vestige-cloud-sync/", env!("CARGO_PKG_VERSION")))
|
||||
.build()
|
||||
.map_err(|e| StorageError::Init(format!("failed to build HTTP client: {e}")))?;
|
||||
Ok(Self {
|
||||
endpoint,
|
||||
sync_key,
|
||||
encryption_key,
|
||||
client,
|
||||
last_etag: RefCell::new(None),
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether this backend encrypts client-side (zero-knowledge).
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
self.encryption_key.is_some()
|
||||
}
|
||||
|
||||
/// Full blob URL for this backend.
|
||||
fn blob_url(&self) -> String {
|
||||
format!("{}{}", self.endpoint, BLOB_PATH)
|
||||
}
|
||||
}
|
||||
|
||||
impl PortableSyncBackend for HttpPortableSyncBackend {
|
||||
fn label(&self) -> String {
|
||||
format!("cloud:{}", self.endpoint)
|
||||
}
|
||||
|
||||
fn read_archive(&self) -> Result<Option<PortableArchive>> {
|
||||
let resp = self
|
||||
.client
|
||||
.get(self.blob_url())
|
||||
.header(AUTHORIZATION, format!("Bearer {}", self.sync_key))
|
||||
.send()
|
||||
.map_err(|e| StorageError::Init(format!("cloud sync read failed: {e}")))?;
|
||||
|
||||
match resp.status() {
|
||||
StatusCode::NOT_FOUND => {
|
||||
// No remote archive yet — first sync for this key.
|
||||
*self.last_etag.borrow_mut() = None;
|
||||
Ok(None)
|
||||
}
|
||||
StatusCode::OK => {
|
||||
// Capture the ETag for the matching If-Match write.
|
||||
let etag = resp
|
||||
.headers()
|
||||
.get(ETAG)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
*self.last_etag.borrow_mut() = etag;
|
||||
|
||||
let bytes = resp
|
||||
.bytes()
|
||||
.map_err(|e| StorageError::Init(format!("cloud sync read body failed: {e}")))?;
|
||||
|
||||
// Decrypt if this is a zero-knowledge envelope. If a passphrase
|
||||
// is configured but the remote is still plaintext (legacy), parse
|
||||
// it directly — the next push will encrypt it (transparent upgrade).
|
||||
let plaintext: std::borrow::Cow<'_, [u8]> =
|
||||
if super::cloud_crypto::is_encrypted(&bytes) {
|
||||
let pass = self.encryption_key.as_deref().ok_or_else(|| {
|
||||
StorageError::Init(
|
||||
"remote archive is encrypted but VESTIGE_CLOUD_ENCRYPTION_KEY is \
|
||||
not set on this device"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
std::borrow::Cow::Owned(super::cloud_crypto::decrypt(pass, &bytes)?)
|
||||
} else {
|
||||
std::borrow::Cow::Borrowed(&bytes)
|
||||
};
|
||||
|
||||
let archive: PortableArchive =
|
||||
serde_json::from_slice(&plaintext).map_err(|e| {
|
||||
StorageError::Init(format!("failed to parse cloud sync archive: {e}"))
|
||||
})?;
|
||||
Ok(Some(archive))
|
||||
}
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(StorageError::Init(
|
||||
"cloud sync rejected the sync key (401/403). Check your subscription and \
|
||||
VESTIGE_CLOUD_SYNC_KEY."
|
||||
.to_string(),
|
||||
)),
|
||||
other => Err(StorageError::Init(format!(
|
||||
"cloud sync read returned unexpected status {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_archive(&self, archive: &PortableArchive) -> Result<()> {
|
||||
let plaintext = serde_json::to_vec(archive)
|
||||
.map_err(|e| StorageError::Init(format!("failed to serialize archive: {e}")))?;
|
||||
|
||||
// Zero-knowledge: encrypt before upload when a passphrase is set, so the
|
||||
// server only ever stores ciphertext. Content type reflects the payload.
|
||||
let (body, content_type) = match self.encryption_key.as_deref() {
|
||||
Some(pass) => (
|
||||
super::cloud_crypto::encrypt(pass, &plaintext)?,
|
||||
"application/octet-stream",
|
||||
),
|
||||
None => (plaintext, "application/json"),
|
||||
};
|
||||
|
||||
let mut req = self
|
||||
.client
|
||||
.put(self.blob_url())
|
||||
.header(AUTHORIZATION, format!("Bearer {}", self.sync_key))
|
||||
.header(reqwest::header::CONTENT_TYPE, content_type)
|
||||
.body(body);
|
||||
|
||||
// Optimistic concurrency: only overwrite the object we pulled. If the
|
||||
// remote had no archive, require that it still doesn't exist (`If-Match: *`
|
||||
// would require existence, so we omit the header to allow first create).
|
||||
if let Some(etag) = self.last_etag.borrow_mut().take() {
|
||||
req = req.header(IF_MATCH, etag);
|
||||
}
|
||||
|
||||
let resp = req
|
||||
.send()
|
||||
.map_err(|e| StorageError::Init(format!("cloud sync write failed: {e}")))?;
|
||||
|
||||
match resp.status() {
|
||||
StatusCode::OK | StatusCode::CREATED | StatusCode::NO_CONTENT => Ok(()),
|
||||
StatusCode::PRECONDITION_FAILED => Err(StorageError::Init(
|
||||
"cloud sync conflict: another device updated your memory in between. \
|
||||
Run `vestige sync --cloud` again to merge and retry."
|
||||
.to_string(),
|
||||
)),
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(StorageError::Init(
|
||||
"cloud sync rejected the sync key (401/403). Check your subscription and \
|
||||
VESTIGE_CLOUD_SYNC_KEY."
|
||||
.to_string(),
|
||||
)),
|
||||
StatusCode::PAYLOAD_TOO_LARGE => Err(StorageError::Init(
|
||||
"cloud sync archive too large for the hosted plan limit".to_string(),
|
||||
)),
|
||||
other => Err(StorageError::Init(format!(
|
||||
"cloud sync write returned unexpected status {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use super::super::portable::{PortableArchive, PortableTable, PORTABLE_ARCHIVE_FORMAT};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
|
||||
fn sample_archive() -> PortableArchive {
|
||||
PortableArchive {
|
||||
archive_format: PORTABLE_ARCHIVE_FORMAT.to_string(),
|
||||
vestige_version: "test".to_string(),
|
||||
schema_version: 1,
|
||||
exported_at: chrono::Utc::now(),
|
||||
mode: "exact".to_string(),
|
||||
tables: vec![PortableTable {
|
||||
name: "knowledge_nodes".to_string(),
|
||||
columns: vec!["id".to_string()],
|
||||
rows: vec![],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
/// A captured request the mock observed, surfaced to the test thread.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct CapturedRequest {
|
||||
method: String,
|
||||
authorization: Option<String>,
|
||||
if_match: Option<String>,
|
||||
}
|
||||
|
||||
/// Minimal one-shot HTTP mock. `responder` builds the raw HTTP response
|
||||
/// string for the request line + headers it parsed. Returns the bound base
|
||||
/// URL and a receiver for the captured request.
|
||||
fn spawn_mock<F>(responder: F) -> (String, mpsc::Receiver<CapturedRequest>)
|
||||
where
|
||||
F: Fn(&CapturedRequest) -> String + Send + 'static,
|
||||
{
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
let (tx, rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
if let Ok((mut stream, _)) = listener.accept() {
|
||||
let mut buf = [0u8; 8192];
|
||||
let n = stream.read(&mut buf).unwrap_or(0);
|
||||
let text = String::from_utf8_lossy(&buf[..n]);
|
||||
let mut cap = CapturedRequest::default();
|
||||
for (i, line) in text.lines().enumerate() {
|
||||
if i == 0 {
|
||||
cap.method = line.split_whitespace().next().unwrap_or("").to_string();
|
||||
} else if let Some(v) = line.strip_prefix("authorization: ") {
|
||||
cap.authorization = Some(v.trim().to_string());
|
||||
} else if let Some(v) = line.strip_prefix("if-match: ") {
|
||||
cap.if_match = Some(v.trim().to_string());
|
||||
}
|
||||
}
|
||||
let response = responder(&cap);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
let _ = stream.flush();
|
||||
let _ = tx.send(cap);
|
||||
}
|
||||
});
|
||||
(format!("http://{addr}"), rx)
|
||||
}
|
||||
|
||||
fn http_response(status: &str, extra_headers: &str, body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Length: {}\r\n{extra_headers}Connection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_rejects_empty_endpoint_and_key() {
|
||||
assert!(HttpPortableSyncBackend::new("", "key").is_err());
|
||||
assert!(HttpPortableSyncBackend::new("https://x", "").is_err());
|
||||
assert!(HttpPortableSyncBackend::new("https://x", "key").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_trailing_slash_trimmed() {
|
||||
let be = HttpPortableSyncBackend::new("https://sync.example/", "k").unwrap();
|
||||
assert_eq!(be.blob_url(), "https://sync.example/v1/blob");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_404_returns_none() {
|
||||
let (base, rx) = spawn_mock(|_| http_response("404 Not Found", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
let got = be.read_archive().expect("read ok");
|
||||
assert!(got.is_none());
|
||||
let cap = rx.recv().unwrap();
|
||||
assert_eq!(cap.method, "GET");
|
||||
assert_eq!(cap.authorization.as_deref(), Some("Bearer secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_200_parses_and_captures_etag() {
|
||||
let archive = sample_archive();
|
||||
let body = serde_json::to_string(&archive).unwrap();
|
||||
let (base, _rx) = spawn_mock(move |_| {
|
||||
http_response("200 OK", "ETag: \"v1-abc\"\r\n", &body)
|
||||
});
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
let got = be.read_archive().expect("read ok").expect("some archive");
|
||||
assert_eq!(got.archive_format, PORTABLE_ARCHIVE_FORMAT);
|
||||
// ETag captured for the next If-Match write.
|
||||
assert_eq!(be.last_etag.borrow().as_deref(), Some("\"v1-abc\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_401_is_error() {
|
||||
let (base, _rx) = spawn_mock(|_| http_response("401 Unauthorized", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "bad").unwrap();
|
||||
assert!(be.read_archive().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_sends_if_match_when_etag_present() {
|
||||
// Seed an etag as if a prior read happened.
|
||||
let (base, rx) = spawn_mock(|_| http_response("200 OK", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
*be.last_etag.borrow_mut() = Some("\"v1-abc\"".to_string());
|
||||
be.write_archive(&sample_archive()).expect("write ok");
|
||||
let cap = rx.recv().unwrap();
|
||||
assert_eq!(cap.method, "PUT");
|
||||
assert_eq!(cap.authorization.as_deref(), Some("Bearer secret"));
|
||||
assert_eq!(cap.if_match.as_deref(), Some("\"v1-abc\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_omits_if_match_for_first_create() {
|
||||
let (base, rx) = spawn_mock(|_| http_response("201 Created", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
// No prior read → no etag → no If-Match (allow create).
|
||||
be.write_archive(&sample_archive()).expect("write ok");
|
||||
let cap = rx.recv().unwrap();
|
||||
assert_eq!(cap.method, "PUT");
|
||||
assert!(cap.if_match.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_412_is_conflict_error() {
|
||||
let (base, _rx) = spawn_mock(|_| http_response("412 Precondition Failed", "", ""));
|
||||
let be = HttpPortableSyncBackend::new(base, "secret").unwrap();
|
||||
*be.last_etag.borrow_mut() = Some("\"stale\"".to_string());
|
||||
let err = be.write_archive(&sample_archive()).unwrap_err();
|
||||
assert!(err.to_string().contains("conflict"));
|
||||
}
|
||||
}
|
||||
|
|
@ -2,12 +2,19 @@
|
|||
//!
|
||||
//! Backend-agnostic memory store abstraction plus SQLite reference impl.
|
||||
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
mod cloud_crypto;
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
mod cloud_sync;
|
||||
mod memory_store;
|
||||
mod migrations;
|
||||
mod portable;
|
||||
mod sqlite;
|
||||
mod trace_store;
|
||||
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
pub use cloud_sync::HttpPortableSyncBackend;
|
||||
|
||||
pub use memory_store::{
|
||||
ClassificationResult, Domain, HealthStatus, LocalMemoryStore, MemoryEdge, MemoryRecord,
|
||||
MemoryStore, MemoryStoreError, MemoryStoreResult, MemoryStoreSend, ModelSignature,
|
||||
|
|
|
|||
|
|
@ -6390,6 +6390,29 @@ impl SqliteMemoryStore {
|
|||
self.sync_portable_archive(&backend)
|
||||
}
|
||||
|
||||
/// Synchronize this database with the hosted Vestige Cloud managed-sync
|
||||
/// service. `endpoint` is the base URL (e.g. `https://sync.vestige.dev`) and
|
||||
/// `sync_key` is the per-user key issued at purchase. Pull-merge-push is
|
||||
/// identical to file sync — only the transport differs.
|
||||
///
|
||||
/// When `encryption_key` is `Some`, the archive is encrypted client-side
|
||||
/// (XChaCha20-Poly1305) before upload, so the server only stores ciphertext
|
||||
/// (zero-knowledge). The passphrase never leaves this process.
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
pub fn sync_portable_archive_cloud(
|
||||
&self,
|
||||
endpoint: &str,
|
||||
sync_key: &str,
|
||||
encryption_key: Option<String>,
|
||||
) -> Result<PortableSyncReport> {
|
||||
let backend = super::cloud_sync::HttpPortableSyncBackend::new_with_encryption(
|
||||
endpoint,
|
||||
sync_key,
|
||||
encryption_key,
|
||||
)?;
|
||||
self.sync_portable_archive(&backend)
|
||||
}
|
||||
|
||||
fn merge_portable_table(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
table_name: &str,
|
||||
|
|
|
|||
|
|
@ -10,13 +10,18 @@ categories = ["command-line-utilities", "database"]
|
|||
repository = "https://github.com/samvallad33/vestige"
|
||||
|
||||
[features]
|
||||
default = ["embeddings", "ort-download", "vector-search", "connectors"]
|
||||
default = ["embeddings", "ort-download", "vector-search", "connectors", "cloud-sync"]
|
||||
embeddings = ["vestige-core/embeddings"]
|
||||
vector-search = ["vestige-core/vector-search"]
|
||||
# External-source connectors (#57): GitHub Issues / Redmine indexing via the
|
||||
# `source_sync` MCP tool. On by default so the tool works out of the box; turn
|
||||
# off for a build with no HTTP client.
|
||||
connectors = ["vestige-core/connectors"]
|
||||
# Hosted managed-sync (Vestige Cloud): `vestige sync --cloud` pushes/pulls the
|
||||
# portable archive to the hosted blob service. On by default so the command
|
||||
# works out of the box; the binary already links an HTTP client via
|
||||
# `connectors`, so this adds no new dependency cost.
|
||||
cloud-sync = ["vestige-core/cloud-sync"]
|
||||
# Default ort backend: downloads prebuilt ONNX Runtime at build time.
|
||||
# Fails on targets without prebuilts (notably x86_64-apple-darwin).
|
||||
ort-download = ["embeddings", "vestige-core/ort-download"]
|
||||
|
|
|
|||
|
|
@ -182,10 +182,20 @@ enum Commands {
|
|||
merge: bool,
|
||||
},
|
||||
|
||||
/// Two-way sync with a file-backed portable archive
|
||||
/// Two-way sync with a file-backed portable archive, or Vestige Cloud
|
||||
Sync {
|
||||
/// Sync archive path, often in Dropbox/iCloud/Syncthing/Git
|
||||
archive: PathBuf,
|
||||
/// Sync archive path, often in Dropbox/iCloud/Syncthing/Git.
|
||||
/// Omit when using --cloud.
|
||||
archive: Option<PathBuf>,
|
||||
/// Sync with the hosted Vestige Cloud managed-sync service instead of a
|
||||
/// file. Requires a sync key (VESTIGE_CLOUD_SYNC_KEY) and endpoint
|
||||
/// (--endpoint or VESTIGE_CLOUD_ENDPOINT).
|
||||
#[arg(long)]
|
||||
cloud: bool,
|
||||
/// Vestige Cloud base endpoint (e.g. https://sync.vestige.dev).
|
||||
/// Defaults to the VESTIGE_CLOUD_ENDPOINT env var.
|
||||
#[arg(long)]
|
||||
endpoint: Option<String>,
|
||||
},
|
||||
|
||||
/// Garbage collect stale memories below retention threshold
|
||||
|
|
@ -287,7 +297,11 @@ fn main() -> anyhow::Result<()> {
|
|||
} => run_export(output, format, tags, since),
|
||||
Commands::PortableExport { output } => run_portable_export(output),
|
||||
Commands::PortableImport { input, merge } => run_portable_import(input, merge),
|
||||
Commands::Sync { archive } => run_sync(archive),
|
||||
Commands::Sync {
|
||||
archive,
|
||||
cloud,
|
||||
endpoint,
|
||||
} => run_sync(archive, cloud, endpoint),
|
||||
Commands::Gc {
|
||||
min_retention,
|
||||
max_age_days,
|
||||
|
|
@ -2193,14 +2207,92 @@ fn run_portable_import(input: PathBuf, merge: bool) -> anyhow::Result<()> {
|
|||
}
|
||||
|
||||
/// Run file-backed two-way sync.
|
||||
fn run_sync(archive: PathBuf) -> anyhow::Result<()> {
|
||||
fn run_sync(
|
||||
archive: Option<PathBuf>,
|
||||
cloud: bool,
|
||||
endpoint: Option<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
if cloud {
|
||||
run_sync_cloud(endpoint)
|
||||
} else {
|
||||
let archive = archive.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"no sync target: pass an archive path for file sync, or --cloud for Vestige Cloud"
|
||||
)
|
||||
})?;
|
||||
run_sync_file(archive)
|
||||
}
|
||||
}
|
||||
|
||||
fn run_sync_file(archive: PathBuf) -> anyhow::Result<()> {
|
||||
println!("{}", "=== Vestige File Sync ===".cyan().bold());
|
||||
println!();
|
||||
println!("{}: {}", "Archive".white().bold(), archive.display());
|
||||
|
||||
let storage = open_storage()?;
|
||||
let report = storage.sync_portable_archive_file(&archive)?;
|
||||
print_sync_report(&report);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "cloud-sync")]
|
||||
fn run_sync_cloud(endpoint: Option<String>) -> anyhow::Result<()> {
|
||||
let endpoint = endpoint
|
||||
.or_else(|| std::env::var("VESTIGE_CLOUD_ENDPOINT").ok())
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"no cloud endpoint: pass --endpoint or set VESTIGE_CLOUD_ENDPOINT \
|
||||
(e.g. https://sync.vestige.dev)"
|
||||
)
|
||||
})?;
|
||||
let sync_key = std::env::var("VESTIGE_CLOUD_SYNC_KEY")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"no sync key: set VESTIGE_CLOUD_SYNC_KEY (issued when you subscribe to \
|
||||
Vestige Cloud)"
|
||||
)
|
||||
})?;
|
||||
|
||||
// Optional zero-knowledge encryption passphrase. Never sent to the server.
|
||||
let encryption_key = std::env::var("VESTIGE_CLOUD_ENCRYPTION_KEY")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
|
||||
println!("{}", "=== Vestige Cloud Sync ===".cyan().bold());
|
||||
println!();
|
||||
println!("{}: {}", "Endpoint".white().bold(), endpoint);
|
||||
if encryption_key.is_some() {
|
||||
println!(
|
||||
"{}: {}",
|
||||
"Encryption".white().bold(),
|
||||
"zero-knowledge (XChaCha20-Poly1305) — your data is encrypted before upload".green()
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
"{}: {}",
|
||||
"Encryption".white().bold(),
|
||||
"OFF — set VESTIGE_CLOUD_ENCRYPTION_KEY for zero-knowledge sync".yellow()
|
||||
);
|
||||
}
|
||||
|
||||
let storage = open_storage()?;
|
||||
let report = storage.sync_portable_archive_cloud(&endpoint, &sync_key, encryption_key)?;
|
||||
print_sync_report(&report);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "cloud-sync"))]
|
||||
fn run_sync_cloud(_endpoint: Option<String>) -> anyhow::Result<()> {
|
||||
anyhow::bail!(
|
||||
"this build was compiled without the `cloud-sync` feature; rebuild with \
|
||||
--features cloud-sync to use Vestige Cloud"
|
||||
)
|
||||
}
|
||||
|
||||
fn print_sync_report(report: &vestige_core::PortableSyncReport) {
|
||||
if let Some(pull) = &report.pull {
|
||||
println!("{}", "Pull: merged remote archive".yellow());
|
||||
println!(
|
||||
|
|
@ -2228,8 +2320,6 @@ fn run_sync(archive: PathBuf) -> anyhow::Result<()> {
|
|||
.green()
|
||||
.bold()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run garbage collection command
|
||||
|
|
|
|||
83
scripts/check-no-private-cloud.sh
Executable file
83
scripts/check-no-private-cloud.sh
Executable file
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/env bash
|
||||
# check-no-private-cloud.sh — Fail if private Vestige Cloud *service* code leaks
|
||||
# into this public repository.
|
||||
#
|
||||
# Vestige Cloud is split:
|
||||
# - PUBLIC client (this repo): a thin HTTP sync backend that only moves
|
||||
# encrypted bytes and presents an opaque bearer token.
|
||||
# Legitimate. Reads VESTIGE_CLOUD_ENDPOINT / _SYNC_KEY /
|
||||
# _ENCRYPTION_KEY env vars on the client side.
|
||||
# - PRIVATE service (separate repo, no public remote): the hosted blob
|
||||
# service that owns sync-key -> namespace mapping, per-user
|
||||
# isolation, Lemon Squeezy billing webhooks, and
|
||||
# transactional email. This MUST NEVER be committed here.
|
||||
#
|
||||
# This guard scans only tracked files (git grep) for distinctive *service*
|
||||
# signatures — module headers, billing/provider internals, and server-side
|
||||
# auth/namespace mapping — chosen so the legitimate public client does NOT
|
||||
# match. It deliberately does NOT match the VESTIGE_CLOUD_* client env-var
|
||||
# prefix, which the public client uses legitimately.
|
||||
set -u
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)" || {
|
||||
echo "check-no-private-cloud: not inside a git repository" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Distinctive private-service signatures. Each is a fixed string (grep -F via
|
||||
# -e with --fixed-strings) that appears in the private vestige-cloud service
|
||||
# and must never appear in this public repo. Keep these specific.
|
||||
PATTERNS=(
|
||||
# Service crate identity / entrypoint
|
||||
'name = "vestige-cloud"'
|
||||
'Vestige Cloud — hosted managed-sync blob service'
|
||||
# Service module headers
|
||||
'Sync-key store and authentication'
|
||||
'Blob storage for the managed-sync service'
|
||||
'Lemon Squeezy webhook handling'
|
||||
'Transactional email delivery via Resend'
|
||||
# Billing / provider internals (server-only)
|
||||
'LEMONSQUEEZY_WEBHOOK_SECRET'
|
||||
'lemonsqueezy'
|
||||
# Server-side sync-key -> namespace mapping (the authoritative mapping that
|
||||
# by design lives ONLY in the hosted service, never the client)
|
||||
'sync_keys SET key_hash'
|
||||
)
|
||||
|
||||
# Files this guard itself lives in / references the patterns must be excluded,
|
||||
# or it would always flag itself. Exclude this script and any allowlist doc.
|
||||
EXCLUDES=(
|
||||
':(exclude)scripts/check-no-private-cloud.sh'
|
||||
':(exclude).github/workflows/guard-no-private-cloud.yml'
|
||||
)
|
||||
|
||||
violations=0
|
||||
report=""
|
||||
|
||||
for pat in "${PATTERNS[@]}"; do
|
||||
# -I skip binary, -n line numbers, -F fixed string, -i case-insensitive.
|
||||
if hits=$(git grep -Ini -F -e "$pat" -- "${EXCLUDES[@]}" 2>/dev/null); then
|
||||
if [ -n "$hits" ]; then
|
||||
violations=$((violations + 1))
|
||||
report+=$'\n'" ✗ private-service marker found: \"$pat\""$'\n'
|
||||
report+="$(printf '%s\n' "$hits" | sed 's/^/ /')"$'\n'
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$violations" -ne 0 ]; then
|
||||
echo "════════════════════════════════════════════════════════════════════"
|
||||
echo " PRIVATE CLOUD SERVICE CODE DETECTED IN PUBLIC REPO"
|
||||
echo "════════════════════════════════════════════════════════════════════"
|
||||
echo "$report"
|
||||
echo "════════════════════════════════════════════════════════════════════"
|
||||
echo " The hosted Vestige Cloud service (billing, namespace mapping,"
|
||||
echo " per-user isolation) must live ONLY in the private repo, never here."
|
||||
echo " Remove the file(s) above from this repo. If a match is a false"
|
||||
echo " positive, refine the pattern in scripts/check-no-private-cloud.sh."
|
||||
echo "════════════════════════════════════════════════════════════════════"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "check-no-private-cloud: OK — no private Vestige Cloud service code in public repo"
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue