apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: add file size tracking and sync interval settings to Obsidian plugin for improved reconciliation and performance

Browse source
This commit is contained in:
Anish Sarkar 2026-04-20 23:48:51 +05:30
parent 28d3c628f1
commit 4d3406341d
7 changed files with 225 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

9
surfsense_backend/app/schemas/obsidian_plugin.py
View file

@ -39,6 +39,11 @@ class NotePayload(_PluginBase):
aliases: list[str] = Field(default_factory=list)
content_hash: str = Field(..., description="Plugin-computed SHA-256 of the raw content")
size: int | None = Field(
default=None,
ge=0,
description="Byte size of the local file (mtime+size short-circuit signal). Optional for forward compatibility.",
)
mtime: datetime
ctime: datetime
@ -68,6 +73,10 @@ class DeleteBatchRequest(_PluginBase):
class ManifestEntry(_PluginBase):
hash: str
mtime: datetime
size: int | None = Field(
default=None,
description="Byte size last seen by the server. Enables mtime+size short-circuit; absent when not yet recorded.",
)
class ManifestResponse(_PluginBase):

15
surfsense_backend/app/services/obsidian_plugin_indexer.py
View file

@ -15,9 +15,9 @@ Responsibilities:
- ``delete_note`` — soft delete with a tombstone in ``document_metadata``
so reconciliation can distinguish "user explicitly killed this in the UI"
from "plugin hasn't synced yet".
- ``get_manifest`` — return ``{path: {hash, mtime}}`` for every non-deleted
note belonging to a vault, used by the plugin's reconcile pass on
``onload``.
- ``get_manifest`` — return ``{path: {hash, mtime, size}}`` for every
non-deleted note belonging to a vault, used by the plugin's reconcile
pass on ``onload``.
Design notes
------------
@ -108,6 +108,7 @@ def _build_metadata(
"embeds": payload.embeds,
"aliases": payload.aliases,
"plugin_content_hash": payload.content_hash,
"plugin_file_size": payload.size,
"mtime": payload.mtime.isoformat(),
"ctime": payload.ctime.isoformat(),
"connector_id": connector_id,
@ -360,8 +361,8 @@ async def get_manifest(
connector: SearchSourceConnector,
vault_id: str,
) -> ManifestResponse:
"""Return ``{path: {hash, mtime}}`` for every non-deleted note in this
vault.
"""Return ``{path: {hash, mtime, size}}`` for every non-deleted note in
this vault.
The plugin compares this against its local vault on every ``onload`` to
catch up edits made while offline. Rows missing ``plugin_content_hash``
@ -395,6 +396,8 @@ async def get_manifest(
mtime = datetime.fromisoformat(mtime_raw)
except ValueError:
continue
items[path] = ManifestEntry(hash=plugin_hash, mtime=mtime)
size_raw = meta.get("plugin_file_size")
size = int(size_raw) if isinstance(size_raw, int) else None
items[path] = ManifestEntry(hash=plugin_hash, mtime=mtime, size=size)
return ManifestResponse(vault_id=vault_id, items=items)

58
surfsense_obsidian/src/main.ts
View file

@ -1,4 +1,4 @@
import { Notice, Plugin } from "obsidian";
import { Notice, Platform, Plugin } from "obsidian";
import { SurfSenseApiClient } from "./api-client";
import { PersistentQueue } from "./queue";
import { SurfSenseSettingTab } from "./settings";
@ -23,6 +23,7 @@ export default class SurfSensePlugin extends Plugin {
serverCapabilities: string[] = [];
private settingTab: SurfSenseSettingTab | null = null;
private statusListeners = new Set<() => void>();
private reconcileTimerId: number | null = null;
async onload() {
await this.loadSettings();
@ -60,10 +61,13 @@ export default class SurfSensePlugin extends Plugin {
this.serverCapabilities = [...caps];
this.notifyStatusChange();
},
onReconcileBackoffChanged: () => {
this.restartReconcileTimer();
},
});
this.queue.setFlushHandler(() => {
if (this.settings.syncMode !== "auto") return;
if (!this.shouldAutoSync()) return;
void this.engine.flushQueue();
});
@ -141,9 +145,20 @@ export default class SurfSensePlugin extends Plugin {
},
});
const onNetChange = () => {
if (this.shouldAutoSync()) void this.engine.flushQueue();
};
this.registerDomEvent(window, "online", onNetChange);
const conn = (navigator as unknown as { connection?: NetworkConnection }).connection;
if (conn && typeof conn.addEventListener === "function") {
conn.addEventListener("change", onNetChange);
this.register(() => conn.removeEventListener?.("change", onNetChange));
}
// Wait for layout so the metadataCache is warm before reconcile.
this.app.workspace.onLayoutReady(() => {
void this.engine.start();
this.restartReconcileTimer();
});
}
@ -160,6 +175,38 @@ export default class SurfSensePlugin extends Plugin {
new StatusModal(this.app, this).open();
}
restartReconcileTimer(): void {
if (this.reconcileTimerId !== null) {
window.clearInterval(this.reconcileTimerId);
this.reconcileTimerId = null;
}
const minutes = this.settings.syncIntervalMinutes ?? 10;
if (minutes <= 0) return;
const baseMs = minutes * 60 * 1000;
// Idle vaults back off (×2 → ×4 → ×8); resets on the first edit or non-empty reconcile.
const effectiveMs = this.engine?.getReconcileBackoffMs(baseMs) ?? baseMs;
const id = window.setInterval(
() => {
if (!this.shouldAutoSync()) return;
void this.engine.maybeReconcile();
},
effectiveMs,
);
this.reconcileTimerId = id;
this.registerInterval(id);
}
/** Gate for background network activity; per-edit flush + periodic reconcile both consult this. */
shouldAutoSync(): boolean {
if (!this.settings.wifiOnly) return true;
if (!Platform.isMobileApp) return true;
// navigator.connection is supported on Android Capacitor; undefined on iOS.
// When unavailable, behave permissively so iOS users aren't blocked outright.
const conn = (navigator as unknown as { connection?: NetworkConnection }).connection;
if (!conn || typeof conn.type !== "string") return true;
return conn.type === "wifi" || conn.type === "ethernet";
}
onStatusChange(listener: () => void): void {
this.statusListeners.add(listener);
}
@ -199,6 +246,13 @@ export default class SurfSensePlugin extends Plugin {
}
}
/** Subset of the Network Information API used to detect WiFi vs cellular on Android. */
interface NetworkConnection {
type?: string;
addEventListener?: (event: string, handler: () => void) => void;
removeEventListener?: (event: string, handler: () => void) => void;
}
function generateUuid(): string {
const c = globalThis.crypto;
if (c?.randomUUID) return c.randomUUID();

1
surfsense_obsidian/src/payload.ts
View file

@ -53,6 +53,7 @@ export async function buildNotePayload(
embeds,
aliases,
content_hash: contentHash,
size: file.stat.size,
mtime: file.stat.mtime,
ctime: file.stat.ctime,
};

75
surfsense_obsidian/src/settings.ts
View file

@ -1,6 +1,7 @@
import {
type App,
Notice,
Platform,
PluginSettingTab,
Setting,
} from "obsidian";
@ -121,18 +122,33 @@ export class SurfSenseSettingTab extends PluginSettingTab {
new Setting(containerEl).setName("Vault").setHeading();
new Setting(containerEl)
.setName("Sync mode")
.setDesc("Auto syncs on every edit. Manual only syncs when you trigger it via the command palette.")
.addDropdown((drop) =>
drop
.addOption("auto", "Auto")
.addOption("manual", "Manual")
.setValue(settings.syncMode)
.onChange(async (value) => {
this.plugin.settings.syncMode = value === "manual" ? "manual" : "auto";
await this.plugin.saveSettings();
}),
);
.setName("Sync interval")
.setDesc(
"How often to check for changes made outside Obsidian. Set to off to only sync manually.",
)
.addDropdown((drop) => {
const options: Array<[number, string]> = [
[0, "Off"],
[5, "5 minutes"],
[10, "10 minutes"],
[15, "15 minutes"],
[30, "30 minutes"],
[60, "60 minutes"],
[120, "2 hours"],
[360, "6 hours"],
[720, "12 hours"],
[1440, "24 hours"],
];
for (const [value, label] of options) {
drop.addOption(String(value), label);
}
drop.setValue(String(settings.syncIntervalMinutes));
drop.onChange(async (value) => {
this.plugin.settings.syncIntervalMinutes = Number(value);
await this.plugin.saveSettings();
this.plugin.restartReconcileTimer();
});
});
this.renderFolderList(
containerEl,
@ -184,6 +200,39 @@ export class SurfSenseSettingTab extends PluginSettingTab {
}),
);
if (Platform.isMobileApp) {
new Setting(containerEl)
.setName("Sync only on WiFi")
.setDesc(
"Pause automatic syncing on cellular. Note: only Android can detect network type — on iOS this toggle has no effect.",
)
.addToggle((toggle) =>
toggle
.setValue(settings.wifiOnly)
.onChange(async (value) => {
this.plugin.settings.wifiOnly = value;
await this.plugin.saveSettings();
}),
);
}
new Setting(containerEl)
.setName("Force sync")
.setDesc("Manually re-index the entire vault now.")
.addButton((btn) =>
btn.setButtonText("Update").onClick(async () => {
btn.setDisabled(true);
try {
await this.plugin.engine.maybeReconcile(true);
new Notice("Surfsense: re-sync requested.");
} catch (err) {
this.handleApiError(err);
} finally {
btn.setDisabled(false);
}
}),
);
new Setting(containerEl)
.addButton((btn) =>
btn
@ -231,7 +280,7 @@ export class SurfSenseSettingTab extends PluginSettingTab {
setting.addButton((btn) =>
btn
.setButtonText("Add Folder")
.setButtonText("Add folder")
.setCta()
.onClick(() => {
new FolderSuggestModal(

96
surfsense_obsidian/src/sync-engine.ts
View file

@ -7,10 +7,11 @@ import {
VaultNotRegisteredError,
} from "./api-client";
import { isExcluded, isFolderFiltered } from "./excludes";
import { buildNotePayload, computeContentHash } from "./payload";
import { buildNotePayload } from "./payload";
import { type BatchResult, PersistentQueue } from "./queue";
import type {
HealthResponse,
ManifestEntry,
NotePayload,
QueueItem,
StatusKind,
@ -32,6 +33,8 @@ export interface SyncEngineDeps {
saveSettings: (mut: (s: SyncEngineSettings) => void) => Promise<void>;
setStatus: (s: StatusState) => void;
onCapabilities: (caps: string[]) => void;
/** Fired when the adaptive backoff multiplier may have changed; main.ts uses it to reschedule. */
onReconcileBackoffChanged?: () => void;
}
export interface SyncEngineSettings {
@ -42,7 +45,6 @@ export interface SyncEngineSettings {
excludeFolders: string[];
excludePatterns: string[];
includeAttachments: boolean;
syncMode: "auto" | "manual";
lastReconcileAt: number | null;
lastSyncAt: number | null;
filesSynced: number;
@ -57,11 +59,21 @@ export class SyncEngine {
private readonly deps: SyncEngineDeps;
private capabilities: string[] = [];
private pendingMdEdits = new Map<string, ReturnType<typeof setTimeout>>();
/** Consecutive reconciles that found no work; powers the adaptive interval. */
private idleReconcileStreak = 0;
/** 2^streak is capped at this value (e.g. 8 → max ×8 backoff). */
private readonly maxBackoffMultiplier = 8;
constructor(deps: SyncEngineDeps) {
this.deps = deps;
}
/** Returns the next-tick interval given the user's base, scaled by the idle streak. */
getReconcileBackoffMs(baseMs: number): number {
const multiplier = Math.min(2 ** this.idleReconcileStreak, this.maxBackoffMultiplier);
return baseMs * multiplier;
}
getCapabilities(): readonly string[] {
return this.capabilities;
}
@ -131,6 +143,7 @@ export class SyncEngine {
if (!this.shouldTrack(file)) return;
const settings = this.deps.getSettings();
if (this.isExcluded(file.path, settings)) return;
this.resetIdleStreak();
if (this.isMarkdown(file)) {
this.scheduleMdUpsert(file.path);
return;
@ -142,6 +155,7 @@ export class SyncEngine {
if (!this.shouldTrack(file)) return;
const settings = this.deps.getSettings();
if (this.isExcluded(file.path, settings)) return;
this.resetIdleStreak();
if (this.isMarkdown(file)) {
// Defer to metadataCache.changed so payload fields are fresh.
this.scheduleMdUpsert(file.path);
@ -152,6 +166,7 @@ export class SyncEngine {
onDelete(file: TAbstractFile): void {
if (!this.shouldTrack(file)) return;
this.resetIdleStreak();
this.deps.queue.enqueueDelete(file.path);
void this.deps.saveSettings((s) => {
s.tombstones[file.path] = Date.now();
@ -160,6 +175,7 @@ export class SyncEngine {
onRename(file: TAbstractFile, oldPath: string): void {
if (!this.shouldTrack(file)) return;
this.resetIdleStreak();
const settings = this.deps.getSettings();
if (this.isExcluded(file.path, settings)) {
this.deps.queue.enqueueDelete(oldPath);
@ -341,6 +357,7 @@ export class SyncEngine {
embeds: [],
aliases: [],
content_hash: hash,
size: file.stat.size,
mtime: file.stat.mtime,
ctime: file.stat.ctime,
is_binary: true,
@ -359,47 +376,59 @@ export class SyncEngine {
this.setStatus("syncing", "Reconciling vault with server…");
try {
const manifest = await this.deps.apiClient.getManifest(settings.vaultId);
const remote = manifest.entries ?? {};
await this.diffAndQueue(settings, remote);
const remote = manifest.items ?? {};
const enqueued = this.diffAndQueue(settings, remote);
await this.deps.saveSettings((s) => {
s.lastReconcileAt = Date.now();
s.tombstones = pruneTombstones(s.tombstones);
});
this.updateIdleStreak(enqueued);
await this.flushQueue();
} catch (err) {
this.classifyAndStatus(err, "Reconcile failed");
}
}
private async diffAndQueue(
/**
* Compare local vault to server manifest and enqueue diffs.
*
* Performance: short-circuits on `mtime + size` for every file. We trust the
* pair as a "no change" signal because (a) content edits move mtime, and
* (b) same-mtime/different-content requires deliberate filesystem trickery.
* False positives (mtime moved, content identical) collapse to a no-op
* upsert on the server via its `content_hash` check. Net effect: zero disk
* reads on idle reconciles.
*
* Returns the number of items enqueued so the caller can drive the
* adaptive backoff.
*/
private diffAndQueue(
settings: SyncEngineSettings,
remote: Record<string, { hash: string; mtime: number }>,
): Promise<void> {
remote: Record<string, ManifestEntry>,
): number {
const localFiles = this.deps.app.vault.getFiles().filter((f) => {
if (!this.shouldTrack(f)) return false;
if (this.isExcluded(f.path, settings)) return false;
return true;
});
const localPaths = new Set(localFiles.map((f) => f.path));
let enqueued = 0;
// Local-only or content-changed → upsert.
for (const file of localFiles) {
const remoteEntry = remote[file.path];
if (!remoteEntry) {
this.deps.queue.enqueueUpsert(file.path);
enqueued++;
continue;
}
if (file.stat.mtime > remoteEntry.mtime + 1000) {
this.deps.queue.enqueueUpsert(file.path);
continue;
}
if (this.isMarkdown(file)) {
const content = await this.deps.app.vault.cachedRead(file);
const hash = await computeContentHash(content);
if (hash !== remoteEntry.hash) {
this.deps.queue.enqueueUpsert(file.path);
}
}
const remoteMtimeMs = toMillis(remoteEntry.mtime);
const mtimeMatches = file.stat.mtime <= remoteMtimeMs + 1000;
// Older server rows lack `size`; treat as "unknown" → fall through to upsert.
const sizeMatches =
typeof remoteEntry.size === "number" && file.stat.size === remoteEntry.size;
if (mtimeMatches && sizeMatches) continue;
this.deps.queue.enqueueUpsert(file.path);
enqueued++;
}
// Remote-only → delete, but only if NOT a fresh tombstone (which
@ -409,7 +438,28 @@ export class SyncEngine {
const tombstone = settings.tombstones[path];
if (tombstone && Date.now() - tombstone < TOMBSTONE_TTL_MS) continue;
this.deps.queue.enqueueDelete(path);
enqueued++;
}
return enqueued;
}
/** Bump (idle) or reset (active) the streak; notify only when the cap-aware multiplier changes. */
private updateIdleStreak(enqueued: number): void {
const previousStreak = this.idleReconcileStreak;
if (enqueued === 0) this.idleReconcileStreak++;
else this.idleReconcileStreak = 0;
const cap = Math.log2(this.maxBackoffMultiplier);
const cappedPrev = Math.min(previousStreak, cap);
const cappedNow = Math.min(this.idleReconcileStreak, cap);
if (cappedPrev !== cappedNow) this.deps.onReconcileBackoffChanged?.();
}
/** Vault edit happened — drop back to the base interval immediately. */
private resetIdleStreak(): void {
if (this.idleReconcileStreak === 0) return;
this.idleReconcileStreak = 0;
this.deps.onReconcileBackoffChanged?.();
}
// ---- status helpers ---------------------------------------------------
@ -514,6 +564,14 @@ function formatRelative(ts: number): string {
return `${Math.round(diff / 86_400_000)}d ago`;
}
/** Manifest mtimes are Pydantic-serialised ISO strings; vault stats are epoch ms. Normalise to ms. */
function toMillis(value: number | string | Date): number {
if (typeof value === "number") return value;
if (value instanceof Date) return value.getTime();
const parsed = Date.parse(value);
return Number.isFinite(parsed) ? parsed : 0;
}
function pruneTombstones(tombstones: Record<string, number>): Record<string, number> {
const out: Record<string, number> = {};
const cutoff = Date.now() - TOMBSTONE_TTL_MS;

14
surfsense_obsidian/src/types.ts
View file

@ -7,7 +7,10 @@ export interface SurfsensePluginSettings {
connectorId: number | null;
/** UUID for the vault — lives here so Obsidian Sync replicates it across devices. */
vaultId: string;
syncMode: "auto" | "manual";
/** 0 disables periodic reconcile (Force sync still works). */
syncIntervalMinutes: number;
/** Mobile-only: pause auto-sync when on cellular. iOS can't detect network type, so the toggle is a no-op there. */
wifiOnly: boolean;
includeFolders: string[];
excludeFolders: string[];
excludePatterns: string[];
@ -25,7 +28,8 @@ export const DEFAULT_SETTINGS: SurfsensePluginSettings = {
searchSpaceId: null,
connectorId: null,
vaultId: "",
syncMode: "auto",
syncIntervalMinutes: 10,
wifiOnly: false,
includeFolders: [],
excludeFolders: [],
excludePatterns: [".trash", "_attachments", "templates"],
@ -77,6 +81,8 @@ export interface NotePayload {
embeds: string[];
aliases: string[];
content_hash: string;
/** Byte size of the local file; pairs with mtime for the reconcile short-circuit. */
size: number;
mtime: number;
ctime: number;
[key: string]: unknown;
@ -112,12 +118,14 @@ export interface HealthResponse {
export interface ManifestEntry {
hash: string;
mtime: number;
/** Optional: byte size of stored content. Enables mtime+size short-circuit; falls back to upsert when missing. */
size?: number;
[key: string]: unknown;
}
export interface ManifestResponse {
vault_id: string;
entries: Record<string, ManifestEntry>;
items: Record<string, ManifestEntry>;
[key: string]: unknown;
}