apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-27 09:46:25 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: implement upload-based folder indexing and synchronization features

Browse source
This commit is contained in:
Anish Sarkar 2026-04-08 15:46:52 +05:30
parent b3925654dd
commit 5f5954e932
13 changed files with 1273 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

104
surfsense_web/components/sources/FolderWatchDialog.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { X } from "lucide-react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import {
@ -13,8 +13,8 @@ import {
} from "@/components/ui/dialog";
import { Spinner } from "@/components/ui/spinner";
import { Switch } from "@/components/ui/switch";
import { documentsApiService } from "@/lib/apis/documents-api.service";
import { getSupportedExtensionsSet } from "@/lib/supported-extensions";
import { type FolderSyncProgress, uploadFolderScan } from "@/lib/folder-sync-upload";
export interface SelectedFolder {
path: string;
@ -29,7 +29,7 @@ interface FolderWatchDialogProps {
initialFolder?: SelectedFolder | null;
}
const DEFAULT_EXCLUDE_PATTERNS = [
export const DEFAULT_EXCLUDE_PATTERNS = [
".git",
"node_modules",
"__pycache__",
@ -48,6 +48,8 @@ export function FolderWatchDialog({
const [selectedFolder, setSelectedFolder] = useState<SelectedFolder | null>(null);
const [shouldSummarize, setShouldSummarize] = useState(false);
const [submitting, setSubmitting] = useState(false);
const [progress, setProgress] = useState<FolderSyncProgress | null>(null);
const abortRef = useRef<AbortController | null>(null);
useEffect(() => {
if (open && initialFolder) {
@ -68,29 +70,38 @@ export function FolderWatchDialog({
setSelectedFolder({ path: folderPath, name: folderName });
}, []);
const handleCancel = useCallback(() => {
abortRef.current?.abort();
}, []);
const handleSubmit = useCallback(async () => {
if (!selectedFolder) return;
const api = window.electronAPI;
if (!api) return;
const controller = new AbortController();
abortRef.current = controller;
setSubmitting(true);
try {
const result = await documentsApiService.folderIndex(searchSpaceId, {
folder_path: selectedFolder.path,
folder_name: selectedFolder.name,
search_space_id: searchSpaceId,
enable_summary: shouldSummarize,
file_extensions: supportedExtensions,
});
setProgress(null);
const rootFolderId = (result as { root_folder_id?: number })?.root_folder_id ?? null;
try {
const rootFolderId = await uploadFolderScan({
folderPath: selectedFolder.path,
folderName: selectedFolder.name,
searchSpaceId,
excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
fileExtensions: supportedExtensions,
enableSummary: shouldSummarize,
onProgress: setProgress,
signal: controller.signal,
});
await api.addWatchedFolder({
path: selectedFolder.path,
name: selectedFolder.name,
excludePatterns: DEFAULT_EXCLUDE_PATTERNS,
fileExtensions: supportedExtensions,
rootFolderId,
rootFolderId: rootFolderId ?? null,
searchSpaceId,
active: true,
});
@ -98,12 +109,19 @@ export function FolderWatchDialog({
toast.success(`Watching folder: ${selectedFolder.name}`);
setSelectedFolder(null);
setShouldSummarize(false);
setProgress(null);
onOpenChange(false);
onSuccess?.();
} catch (err) {
toast.error((err as Error)?.message || "Failed to watch folder");
if ((err as Error)?.name === "AbortError") {
toast.info("Folder sync cancelled. Partial progress was saved.");
} else {
toast.error((err as Error)?.message || "Failed to watch folder");
}
} finally {
abortRef.current = null;
setSubmitting(false);
setProgress(null);
}
}, [
selectedFolder,
@ -119,12 +137,31 @@ export function FolderWatchDialog({
if (!nextOpen && !submitting) {
setSelectedFolder(null);
setShouldSummarize(false);
setProgress(null);
}
onOpenChange(nextOpen);
},
[onOpenChange, submitting]
);
const progressLabel = useMemo(() => {
if (!progress) return null;
switch (progress.phase) {
case "listing":
return "Scanning folder...";
case "checking":
return `Checking ${progress.total} file(s)...`;
case "uploading":
return `Uploading ${progress.uploaded}/${progress.total} file(s)...`;
case "finalizing":
return "Finalizing...";
case "done":
return "Done!";
default:
return null;
}
}, [progress]);
return (
<Dialog open={open} onOpenChange={handleOpenChange}>
<DialogContent className="sm:max-w-md select-none">
@ -174,14 +211,39 @@ export function FolderWatchDialog({
<Switch checked={shouldSummarize} onCheckedChange={setShouldSummarize} />
</div>
<Button className="w-full relative" onClick={handleSubmit} disabled={submitting}>
<span className={submitting ? "invisible" : ""}>Start Folder Sync</span>
{submitting && (
<span className="absolute inset-0 flex items-center justify-center">
<Spinner size="sm" />
</span>
{progressLabel && (
<div className="rounded-lg bg-slate-400/5 dark:bg-white/5 px-3 py-2">
<p className="text-xs text-muted-foreground">{progressLabel}</p>
{progress && progress.phase === "uploading" && progress.total > 0 && (
<div className="mt-1.5 h-1.5 w-full rounded-full bg-muted overflow-hidden">
<div
className="h-full bg-primary rounded-full transition-[width] duration-300"
style={{ width: `${Math.round((progress.uploaded / progress.total) * 100)}%` }}
/>
</div>
)}
</div>
)}
<div className="flex gap-2">
{submitting ? (
<>
<Button variant="outline" className="flex-1" onClick={handleCancel}>
Cancel
</Button>
<Button className="flex-1 relative" disabled>
<span className="invisible">Syncing...</span>
<span className="absolute inset-0 flex items-center justify-center">
<Spinner size="sm" />
</span>
</Button>
</>
) : (
<Button className="w-full" onClick={handleSubmit}>
Start Folder Sync
</Button>
)}
</Button>
</div>
</>
)}
</div>