diff --git a/surfsense_backend/app/schemas/obsidian_plugin.py b/surfsense_backend/app/schemas/obsidian_plugin.py index 8eb863745..0b9f0f3bf 100644 --- a/surfsense_backend/app/schemas/obsidian_plugin.py +++ b/surfsense_backend/app/schemas/obsidian_plugin.py @@ -39,6 +39,11 @@ class NotePayload(_PluginBase): aliases: list[str] = Field(default_factory=list) content_hash: str = Field(..., description="Plugin-computed SHA-256 of the raw content") + size: int | None = Field( + default=None, + ge=0, + description="Byte size of the local file (mtime+size short-circuit signal). Optional for forward compatibility.", + ) mtime: datetime ctime: datetime @@ -68,6 +73,10 @@ class DeleteBatchRequest(_PluginBase): class ManifestEntry(_PluginBase): hash: str mtime: datetime + size: int | None = Field( + default=None, + description="Byte size last seen by the server. Enables mtime+size short-circuit; absent when not yet recorded.", + ) class ManifestResponse(_PluginBase): diff --git a/surfsense_backend/app/services/obsidian_plugin_indexer.py b/surfsense_backend/app/services/obsidian_plugin_indexer.py index 385c8e013..7aadf90c8 100644 --- a/surfsense_backend/app/services/obsidian_plugin_indexer.py +++ b/surfsense_backend/app/services/obsidian_plugin_indexer.py @@ -15,9 +15,9 @@ Responsibilities: - ``delete_note`` — soft delete with a tombstone in ``document_metadata`` so reconciliation can distinguish "user explicitly killed this in the UI" from "plugin hasn't synced yet". -- ``get_manifest`` — return ``{path: {hash, mtime}}`` for every non-deleted - note belonging to a vault, used by the plugin's reconcile pass on - ``onload``. +- ``get_manifest`` — return ``{path: {hash, mtime, size}}`` for every + non-deleted note belonging to a vault, used by the plugin's reconcile + pass on ``onload``. Design notes ------------ @@ -108,6 +108,7 @@ def _build_metadata( "embeds": payload.embeds, "aliases": payload.aliases, "plugin_content_hash": payload.content_hash, + "plugin_file_size": payload.size, "mtime": payload.mtime.isoformat(), "ctime": payload.ctime.isoformat(), "connector_id": connector_id, @@ -360,8 +361,8 @@ async def get_manifest( connector: SearchSourceConnector, vault_id: str, ) -> ManifestResponse: - """Return ``{path: {hash, mtime}}`` for every non-deleted note in this - vault. + """Return ``{path: {hash, mtime, size}}`` for every non-deleted note in + this vault. The plugin compares this against its local vault on every ``onload`` to catch up edits made while offline. Rows missing ``plugin_content_hash`` @@ -395,6 +396,8 @@ async def get_manifest( mtime = datetime.fromisoformat(mtime_raw) except ValueError: continue - items[path] = ManifestEntry(hash=plugin_hash, mtime=mtime) + size_raw = meta.get("plugin_file_size") + size = int(size_raw) if isinstance(size_raw, int) else None + items[path] = ManifestEntry(hash=plugin_hash, mtime=mtime, size=size) return ManifestResponse(vault_id=vault_id, items=items) diff --git a/surfsense_obsidian/src/main.ts b/surfsense_obsidian/src/main.ts index d8b9c8a62..de3665d36 100644 --- a/surfsense_obsidian/src/main.ts +++ b/surfsense_obsidian/src/main.ts @@ -1,4 +1,4 @@ -import { Notice, Plugin } from "obsidian"; +import { Notice, Platform, Plugin } from "obsidian"; import { SurfSenseApiClient } from "./api-client"; import { PersistentQueue } from "./queue"; import { SurfSenseSettingTab } from "./settings"; @@ -23,6 +23,7 @@ export default class SurfSensePlugin extends Plugin { serverCapabilities: string[] = []; private settingTab: SurfSenseSettingTab | null = null; private statusListeners = new Set<() => void>(); + private reconcileTimerId: number | null = null; async onload() { await this.loadSettings(); @@ -60,10 +61,13 @@ export default class SurfSensePlugin extends Plugin { this.serverCapabilities = [...caps]; this.notifyStatusChange(); }, + onReconcileBackoffChanged: () => { + this.restartReconcileTimer(); + }, }); this.queue.setFlushHandler(() => { - if (this.settings.syncMode !== "auto") return; + if (!this.shouldAutoSync()) return; void this.engine.flushQueue(); }); @@ -141,9 +145,20 @@ export default class SurfSensePlugin extends Plugin { }, }); + const onNetChange = () => { + if (this.shouldAutoSync()) void this.engine.flushQueue(); + }; + this.registerDomEvent(window, "online", onNetChange); + const conn = (navigator as unknown as { connection?: NetworkConnection }).connection; + if (conn && typeof conn.addEventListener === "function") { + conn.addEventListener("change", onNetChange); + this.register(() => conn.removeEventListener?.("change", onNetChange)); + } + // Wait for layout so the metadataCache is warm before reconcile. this.app.workspace.onLayoutReady(() => { void this.engine.start(); + this.restartReconcileTimer(); }); } @@ -160,6 +175,38 @@ export default class SurfSensePlugin extends Plugin { new StatusModal(this.app, this).open(); } + restartReconcileTimer(): void { + if (this.reconcileTimerId !== null) { + window.clearInterval(this.reconcileTimerId); + this.reconcileTimerId = null; + } + const minutes = this.settings.syncIntervalMinutes ?? 10; + if (minutes <= 0) return; + const baseMs = minutes * 60 * 1000; + // Idle vaults back off (×2 → ×4 → ×8); resets on the first edit or non-empty reconcile. + const effectiveMs = this.engine?.getReconcileBackoffMs(baseMs) ?? baseMs; + const id = window.setInterval( + () => { + if (!this.shouldAutoSync()) return; + void this.engine.maybeReconcile(); + }, + effectiveMs, + ); + this.reconcileTimerId = id; + this.registerInterval(id); + } + + /** Gate for background network activity; per-edit flush + periodic reconcile both consult this. */ + shouldAutoSync(): boolean { + if (!this.settings.wifiOnly) return true; + if (!Platform.isMobileApp) return true; + // navigator.connection is supported on Android Capacitor; undefined on iOS. + // When unavailable, behave permissively so iOS users aren't blocked outright. + const conn = (navigator as unknown as { connection?: NetworkConnection }).connection; + if (!conn || typeof conn.type !== "string") return true; + return conn.type === "wifi" || conn.type === "ethernet"; + } + onStatusChange(listener: () => void): void { this.statusListeners.add(listener); } @@ -199,6 +246,13 @@ export default class SurfSensePlugin extends Plugin { } } +/** Subset of the Network Information API used to detect WiFi vs cellular on Android. */ +interface NetworkConnection { + type?: string; + addEventListener?: (event: string, handler: () => void) => void; + removeEventListener?: (event: string, handler: () => void) => void; +} + function generateUuid(): string { const c = globalThis.crypto; if (c?.randomUUID) return c.randomUUID(); diff --git a/surfsense_obsidian/src/payload.ts b/surfsense_obsidian/src/payload.ts index 86b889f89..3294d62df 100644 --- a/surfsense_obsidian/src/payload.ts +++ b/surfsense_obsidian/src/payload.ts @@ -53,6 +53,7 @@ export async function buildNotePayload( embeds, aliases, content_hash: contentHash, + size: file.stat.size, mtime: file.stat.mtime, ctime: file.stat.ctime, }; diff --git a/surfsense_obsidian/src/settings.ts b/surfsense_obsidian/src/settings.ts index 0b753a2bb..8cd8c1edb 100644 --- a/surfsense_obsidian/src/settings.ts +++ b/surfsense_obsidian/src/settings.ts @@ -1,6 +1,7 @@ import { type App, Notice, + Platform, PluginSettingTab, Setting, } from "obsidian"; @@ -121,18 +122,33 @@ export class SurfSenseSettingTab extends PluginSettingTab { new Setting(containerEl).setName("Vault").setHeading(); new Setting(containerEl) - .setName("Sync mode") - .setDesc("Auto syncs on every edit. Manual only syncs when you trigger it via the command palette.") - .addDropdown((drop) => - drop - .addOption("auto", "Auto") - .addOption("manual", "Manual") - .setValue(settings.syncMode) - .onChange(async (value) => { - this.plugin.settings.syncMode = value === "manual" ? "manual" : "auto"; - await this.plugin.saveSettings(); - }), - ); + .setName("Sync interval") + .setDesc( + "How often to check for changes made outside Obsidian. Set to off to only sync manually.", + ) + .addDropdown((drop) => { + const options: Array<[number, string]> = [ + [0, "Off"], + [5, "5 minutes"], + [10, "10 minutes"], + [15, "15 minutes"], + [30, "30 minutes"], + [60, "60 minutes"], + [120, "2 hours"], + [360, "6 hours"], + [720, "12 hours"], + [1440, "24 hours"], + ]; + for (const [value, label] of options) { + drop.addOption(String(value), label); + } + drop.setValue(String(settings.syncIntervalMinutes)); + drop.onChange(async (value) => { + this.plugin.settings.syncIntervalMinutes = Number(value); + await this.plugin.saveSettings(); + this.plugin.restartReconcileTimer(); + }); + }); this.renderFolderList( containerEl, @@ -184,6 +200,39 @@ export class SurfSenseSettingTab extends PluginSettingTab { }), ); + if (Platform.isMobileApp) { + new Setting(containerEl) + .setName("Sync only on WiFi") + .setDesc( + "Pause automatic syncing on cellular. Note: only Android can detect network type — on iOS this toggle has no effect.", + ) + .addToggle((toggle) => + toggle + .setValue(settings.wifiOnly) + .onChange(async (value) => { + this.plugin.settings.wifiOnly = value; + await this.plugin.saveSettings(); + }), + ); + } + + new Setting(containerEl) + .setName("Force sync") + .setDesc("Manually re-index the entire vault now.") + .addButton((btn) => + btn.setButtonText("Update").onClick(async () => { + btn.setDisabled(true); + try { + await this.plugin.engine.maybeReconcile(true); + new Notice("Surfsense: re-sync requested."); + } catch (err) { + this.handleApiError(err); + } finally { + btn.setDisabled(false); + } + }), + ); + new Setting(containerEl) .addButton((btn) => btn @@ -231,7 +280,7 @@ export class SurfSenseSettingTab extends PluginSettingTab { setting.addButton((btn) => btn - .setButtonText("Add Folder") + .setButtonText("Add folder") .setCta() .onClick(() => { new FolderSuggestModal( diff --git a/surfsense_obsidian/src/sync-engine.ts b/surfsense_obsidian/src/sync-engine.ts index 8f5fc67f6..8021273da 100644 --- a/surfsense_obsidian/src/sync-engine.ts +++ b/surfsense_obsidian/src/sync-engine.ts @@ -7,10 +7,11 @@ import { VaultNotRegisteredError, } from "./api-client"; import { isExcluded, isFolderFiltered } from "./excludes"; -import { buildNotePayload, computeContentHash } from "./payload"; +import { buildNotePayload } from "./payload"; import { type BatchResult, PersistentQueue } from "./queue"; import type { HealthResponse, + ManifestEntry, NotePayload, QueueItem, StatusKind, @@ -32,6 +33,8 @@ export interface SyncEngineDeps { saveSettings: (mut: (s: SyncEngineSettings) => void) => Promise; setStatus: (s: StatusState) => void; onCapabilities: (caps: string[]) => void; + /** Fired when the adaptive backoff multiplier may have changed; main.ts uses it to reschedule. */ + onReconcileBackoffChanged?: () => void; } export interface SyncEngineSettings { @@ -42,7 +45,6 @@ export interface SyncEngineSettings { excludeFolders: string[]; excludePatterns: string[]; includeAttachments: boolean; - syncMode: "auto" | "manual"; lastReconcileAt: number | null; lastSyncAt: number | null; filesSynced: number; @@ -57,11 +59,21 @@ export class SyncEngine { private readonly deps: SyncEngineDeps; private capabilities: string[] = []; private pendingMdEdits = new Map>(); + /** Consecutive reconciles that found no work; powers the adaptive interval. */ + private idleReconcileStreak = 0; + /** 2^streak is capped at this value (e.g. 8 → max ×8 backoff). */ + private readonly maxBackoffMultiplier = 8; constructor(deps: SyncEngineDeps) { this.deps = deps; } + /** Returns the next-tick interval given the user's base, scaled by the idle streak. */ + getReconcileBackoffMs(baseMs: number): number { + const multiplier = Math.min(2 ** this.idleReconcileStreak, this.maxBackoffMultiplier); + return baseMs * multiplier; + } + getCapabilities(): readonly string[] { return this.capabilities; } @@ -131,6 +143,7 @@ export class SyncEngine { if (!this.shouldTrack(file)) return; const settings = this.deps.getSettings(); if (this.isExcluded(file.path, settings)) return; + this.resetIdleStreak(); if (this.isMarkdown(file)) { this.scheduleMdUpsert(file.path); return; @@ -142,6 +155,7 @@ export class SyncEngine { if (!this.shouldTrack(file)) return; const settings = this.deps.getSettings(); if (this.isExcluded(file.path, settings)) return; + this.resetIdleStreak(); if (this.isMarkdown(file)) { // Defer to metadataCache.changed so payload fields are fresh. this.scheduleMdUpsert(file.path); @@ -152,6 +166,7 @@ export class SyncEngine { onDelete(file: TAbstractFile): void { if (!this.shouldTrack(file)) return; + this.resetIdleStreak(); this.deps.queue.enqueueDelete(file.path); void this.deps.saveSettings((s) => { s.tombstones[file.path] = Date.now(); @@ -160,6 +175,7 @@ export class SyncEngine { onRename(file: TAbstractFile, oldPath: string): void { if (!this.shouldTrack(file)) return; + this.resetIdleStreak(); const settings = this.deps.getSettings(); if (this.isExcluded(file.path, settings)) { this.deps.queue.enqueueDelete(oldPath); @@ -341,6 +357,7 @@ export class SyncEngine { embeds: [], aliases: [], content_hash: hash, + size: file.stat.size, mtime: file.stat.mtime, ctime: file.stat.ctime, is_binary: true, @@ -359,47 +376,59 @@ export class SyncEngine { this.setStatus("syncing", "Reconciling vault with server…"); try { const manifest = await this.deps.apiClient.getManifest(settings.vaultId); - const remote = manifest.entries ?? {}; - await this.diffAndQueue(settings, remote); + const remote = manifest.items ?? {}; + const enqueued = this.diffAndQueue(settings, remote); await this.deps.saveSettings((s) => { s.lastReconcileAt = Date.now(); s.tombstones = pruneTombstones(s.tombstones); }); + this.updateIdleStreak(enqueued); await this.flushQueue(); } catch (err) { this.classifyAndStatus(err, "Reconcile failed"); } } - private async diffAndQueue( + /** + * Compare local vault to server manifest and enqueue diffs. + * + * Performance: short-circuits on `mtime + size` for every file. We trust the + * pair as a "no change" signal because (a) content edits move mtime, and + * (b) same-mtime/different-content requires deliberate filesystem trickery. + * False positives (mtime moved, content identical) collapse to a no-op + * upsert on the server via its `content_hash` check. Net effect: zero disk + * reads on idle reconciles. + * + * Returns the number of items enqueued so the caller can drive the + * adaptive backoff. + */ + private diffAndQueue( settings: SyncEngineSettings, - remote: Record, - ): Promise { + remote: Record, + ): number { const localFiles = this.deps.app.vault.getFiles().filter((f) => { if (!this.shouldTrack(f)) return false; if (this.isExcluded(f.path, settings)) return false; return true; }); const localPaths = new Set(localFiles.map((f) => f.path)); + let enqueued = 0; - // Local-only or content-changed → upsert. for (const file of localFiles) { const remoteEntry = remote[file.path]; if (!remoteEntry) { this.deps.queue.enqueueUpsert(file.path); + enqueued++; continue; } - if (file.stat.mtime > remoteEntry.mtime + 1000) { - this.deps.queue.enqueueUpsert(file.path); - continue; - } - if (this.isMarkdown(file)) { - const content = await this.deps.app.vault.cachedRead(file); - const hash = await computeContentHash(content); - if (hash !== remoteEntry.hash) { - this.deps.queue.enqueueUpsert(file.path); - } - } + const remoteMtimeMs = toMillis(remoteEntry.mtime); + const mtimeMatches = file.stat.mtime <= remoteMtimeMs + 1000; + // Older server rows lack `size`; treat as "unknown" → fall through to upsert. + const sizeMatches = + typeof remoteEntry.size === "number" && file.stat.size === remoteEntry.size; + if (mtimeMatches && sizeMatches) continue; + this.deps.queue.enqueueUpsert(file.path); + enqueued++; } // Remote-only → delete, but only if NOT a fresh tombstone (which @@ -409,7 +438,28 @@ export class SyncEngine { const tombstone = settings.tombstones[path]; if (tombstone && Date.now() - tombstone < TOMBSTONE_TTL_MS) continue; this.deps.queue.enqueueDelete(path); + enqueued++; } + + return enqueued; + } + + /** Bump (idle) or reset (active) the streak; notify only when the cap-aware multiplier changes. */ + private updateIdleStreak(enqueued: number): void { + const previousStreak = this.idleReconcileStreak; + if (enqueued === 0) this.idleReconcileStreak++; + else this.idleReconcileStreak = 0; + const cap = Math.log2(this.maxBackoffMultiplier); + const cappedPrev = Math.min(previousStreak, cap); + const cappedNow = Math.min(this.idleReconcileStreak, cap); + if (cappedPrev !== cappedNow) this.deps.onReconcileBackoffChanged?.(); + } + + /** Vault edit happened — drop back to the base interval immediately. */ + private resetIdleStreak(): void { + if (this.idleReconcileStreak === 0) return; + this.idleReconcileStreak = 0; + this.deps.onReconcileBackoffChanged?.(); } // ---- status helpers --------------------------------------------------- @@ -514,6 +564,14 @@ function formatRelative(ts: number): string { return `${Math.round(diff / 86_400_000)}d ago`; } +/** Manifest mtimes are Pydantic-serialised ISO strings; vault stats are epoch ms. Normalise to ms. */ +function toMillis(value: number | string | Date): number { + if (typeof value === "number") return value; + if (value instanceof Date) return value.getTime(); + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : 0; +} + function pruneTombstones(tombstones: Record): Record { const out: Record = {}; const cutoff = Date.now() - TOMBSTONE_TTL_MS; diff --git a/surfsense_obsidian/src/types.ts b/surfsense_obsidian/src/types.ts index d3dd2a14e..6f7683830 100644 --- a/surfsense_obsidian/src/types.ts +++ b/surfsense_obsidian/src/types.ts @@ -7,7 +7,10 @@ export interface SurfsensePluginSettings { connectorId: number | null; /** UUID for the vault — lives here so Obsidian Sync replicates it across devices. */ vaultId: string; - syncMode: "auto" | "manual"; + /** 0 disables periodic reconcile (Force sync still works). */ + syncIntervalMinutes: number; + /** Mobile-only: pause auto-sync when on cellular. iOS can't detect network type, so the toggle is a no-op there. */ + wifiOnly: boolean; includeFolders: string[]; excludeFolders: string[]; excludePatterns: string[]; @@ -25,7 +28,8 @@ export const DEFAULT_SETTINGS: SurfsensePluginSettings = { searchSpaceId: null, connectorId: null, vaultId: "", - syncMode: "auto", + syncIntervalMinutes: 10, + wifiOnly: false, includeFolders: [], excludeFolders: [], excludePatterns: [".trash", "_attachments", "templates"], @@ -77,6 +81,8 @@ export interface NotePayload { embeds: string[]; aliases: string[]; content_hash: string; + /** Byte size of the local file; pairs with mtime for the reconcile short-circuit. */ + size: number; mtime: number; ctime: number; [key: string]: unknown; @@ -112,12 +118,14 @@ export interface HealthResponse { export interface ManifestEntry { hash: string; mtime: number; + /** Optional: byte size of stored content. Enables mtime+size short-circuit; falls back to upsert when missing. */ + size?: number; [key: string]: unknown; } export interface ManifestResponse { vault_id: string; - entries: Record; + items: Record; [key: string]: unknown; }