diff --git a/surfsense_desktop/src/modules/folder-watcher.ts b/surfsense_desktop/src/modules/folder-watcher.ts index bfd2136c9..072ae7b3f 100644 --- a/surfsense_desktop/src/modules/folder-watcher.ts +++ b/surfsense_desktop/src/modules/folder-watcher.ts @@ -1,5 +1,5 @@ import { BrowserWindow, dialog } from 'electron'; -import chokidar from 'chokidar'; +import chokidar, { type FSWatcher } from 'chokidar'; import * as path from 'path'; import * as fs from 'fs'; import { IPC_CHANNELS } from '../ipc/channels'; @@ -16,13 +16,24 @@ export interface WatchedFolderConfig { interface WatcherEntry { config: WatchedFolderConfig; - watcher: chokidar.FSWatcher | null; + watcher: FSWatcher | null; } +type MtimeMap = Record; + const STORE_KEY = 'watchedFolders'; +const MTIME_TOLERANCE_S = 1.0; + let store: any = null; +let mtimeStore: any = null; let watchers: Map = new Map(); +/** + * In-memory cache of mtime maps, keyed by folder path. + * Persisted to electron-store on mutation. + */ +const mtimeMaps: Map = new Map(); + async function getStore() { if (!store) { const { default: Store } = await import('electron-store'); @@ -36,6 +47,73 @@ async function getStore() { return store; } +async function getMtimeStore() { + if (!mtimeStore) { + const { default: Store } = await import('electron-store'); + mtimeStore = new Store({ + name: 'folder-mtime-maps', + defaults: {} as Record, + }); + } + return mtimeStore; +} + +function loadMtimeMap(folderPath: string): MtimeMap { + return mtimeMaps.get(folderPath) ?? {}; +} + +function persistMtimeMap(folderPath: string) { + const map = mtimeMaps.get(folderPath) ?? {}; + getMtimeStore().then((s) => s.set(folderPath, map)); +} + +function walkFolderMtimes(config: WatchedFolderConfig): MtimeMap { + const root = config.path; + const result: MtimeMap = {}; + const excludes = new Set(config.excludePatterns); + + function walk(dir: string) { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + const name = entry.name; + + // Skip dotfiles/dotdirs and excluded names + if (name.startsWith('.') || excludes.has(name)) continue; + + const full = path.join(dir, name); + + if (entry.isDirectory()) { + walk(full); + } else if (entry.isFile()) { + if ( + config.fileExtensions && + config.fileExtensions.length > 0 + ) { + const ext = path.extname(name).toLowerCase(); + if (!config.fileExtensions.includes(ext)) continue; + } + + try { + const stat = fs.statSync(full); + const rel = path.relative(root, full); + result[rel] = stat.mtimeMs; + } catch { + // File may have been removed between readdir and stat + } + } + } + } + + walk(root); + return result; +} + function getMainWindow(): BrowserWindow | null { const windows = BrowserWindow.getAllWindows(); return windows.length > 0 ? windows[0] : null; @@ -48,11 +126,16 @@ function sendToRenderer(channel: string, data: any) { } } -function startWatcher(config: WatchedFolderConfig) { +async function startWatcher(config: WatchedFolderConfig) { if (watchers.has(config.path)) { return; } + // Load persisted mtime map into memory before starting the watcher + const ms = await getMtimeStore(); + const storedMap: MtimeMap = ms.get(config.path) ?? {}; + mtimeMaps.set(config.path, { ...storedMap }); + const ignored = [ /(^|[/\\])\../, // dotfiles by default ...config.excludePatterns.map((p) => `**/${p}/**`), @@ -60,7 +143,7 @@ function startWatcher(config: WatchedFolderConfig) { const watcher = chokidar.watch(config.path, { persistent: true, - ignoreInitial: false, + ignoreInitial: true, awaitWriteFinish: { stabilityThreshold: 500, pollInterval: 100, @@ -72,6 +155,58 @@ function startWatcher(config: WatchedFolderConfig) { watcher.on('ready', () => { ready = true; + + // Detect offline changes by diffing current filesystem against stored mtime map + const currentMap = walkFolderMtimes(config); + const storedSnapshot = loadMtimeMap(config.path); + const now = Date.now(); + + for (const [rel, currentMtime] of Object.entries(currentMap)) { + const storedMtime = storedSnapshot[rel]; + if (storedMtime === undefined) { + // New file added while app was closed + sendToRenderer(IPC_CHANNELS.FOLDER_SYNC_FILE_CHANGED, { + connectorId: config.connectorId, + searchSpaceId: config.searchSpaceId, + folderPath: config.path, + relativePath: rel, + fullPath: path.join(config.path, rel), + action: 'add', + timestamp: now, + }); + } else if (Math.abs(currentMtime - storedMtime) >= MTIME_TOLERANCE_S * 1000) { + // File modified while app was closed + sendToRenderer(IPC_CHANNELS.FOLDER_SYNC_FILE_CHANGED, { + connectorId: config.connectorId, + searchSpaceId: config.searchSpaceId, + folderPath: config.path, + relativePath: rel, + fullPath: path.join(config.path, rel), + action: 'change', + timestamp: now, + }); + } + } + + for (const rel of Object.keys(storedSnapshot)) { + if (!(rel in currentMap)) { + // File deleted while app was closed + sendToRenderer(IPC_CHANNELS.FOLDER_SYNC_FILE_CHANGED, { + connectorId: config.connectorId, + searchSpaceId: config.searchSpaceId, + folderPath: config.path, + relativePath: rel, + fullPath: path.join(config.path, rel), + action: 'unlink', + timestamp: now, + }); + } + } + + // Replace stored map with current filesystem state + mtimeMaps.set(config.path, currentMap); + persistMtimeMap(config.path); + sendToRenderer(IPC_CHANNELS.FOLDER_SYNC_WATCHER_READY, { connectorId: config.connectorId, folderPath: config.path, @@ -91,6 +226,21 @@ function startWatcher(config: WatchedFolderConfig) { if (!config.fileExtensions.includes(ext)) return; } + // Keep mtime map in sync with live changes + const map = mtimeMaps.get(config.path); + if (map) { + if (action === 'unlink') { + delete map[relativePath]; + } else { + try { + map[relativePath] = fs.statSync(filePath).mtimeMs; + } catch { + // File may have been removed between event and stat + } + } + persistMtimeMap(config.path); + } + sendToRenderer(IPC_CHANNELS.FOLDER_SYNC_FILE_CHANGED, { connectorId: config.connectorId, searchSpaceId: config.searchSpaceId, @@ -110,6 +260,7 @@ function startWatcher(config: WatchedFolderConfig) { } function stopWatcher(folderPath: string) { + persistMtimeMap(folderPath); const entry = watchers.get(folderPath); if (entry?.watcher) { entry.watcher.close(); @@ -144,7 +295,7 @@ export async function addWatchedFolder( s.set(STORE_KEY, folders); if (config.active) { - startWatcher(config); + await startWatcher(config); } return folders; @@ -160,6 +311,11 @@ export async function removeWatchedFolder( stopWatcher(folderPath); + // Clean up persisted mtime map for this folder + mtimeMaps.delete(folderPath); + const ms = await getMtimeStore(); + ms.delete(folderPath); + return updated; } @@ -190,9 +346,9 @@ export async function pauseWatcher(): Promise { } export async function resumeWatcher(): Promise { - for (const [folderPath, entry] of watchers) { + for (const [, entry] of watchers) { if (!entry.watcher && entry.config.active) { - startWatcher(entry.config); + await startWatcher(entry.config); } } } @@ -203,7 +359,7 @@ export async function registerFolderWatcher(): Promise { for (const config of folders) { if (config.active && fs.existsSync(config.path)) { - startWatcher(config); + await startWatcher(config); } } }