mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-17 18:35:19 +02:00
feat: implement real-time document updates and lazy loading for document content in DocumentsTable and DocumentsTableShell
This commit is contained in:
Anish Sarkar
parent
1cb578cffb
commit
c19aa5fa99
9 changed files with 655 additions and 360 deletions
|
|
@ -1,15 +1,21 @@
|
|||
"use client";
|
||||
|
||||
import { formatDistanceToNow } from "date-fns";
|
||||
import {Calendar, ChevronDown, ChevronUp, FileText, FileX, Network, Plus, User } from "lucide-react";
|
||||
import { Calendar, ChevronDown, ChevronUp, FileText, FileX, Loader2, Network, Plus, User } from "lucide-react";
|
||||
import { motion } from "motion/react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import React, { useRef, useState, useEffect } from "react";
|
||||
import React, { useRef, useState, useEffect, useCallback } from "react";
|
||||
import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup";
|
||||
import { DocumentViewer } from "@/components/document-viewer";
|
||||
import { JsonMetadataViewer } from "@/components/json-metadata-viewer";
|
||||
import { MarkdownViewer } from "@/components/markdown-viewer";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import {
|
||||
Table,
|
||||
|
|
@ -20,6 +26,7 @@ import {
|
|||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { DocumentTypeChip } from "./DocumentTypeIcon";
|
||||
import type { ColumnVisibility, Document } from "./types";
|
||||
|
||||
|
|
@ -153,6 +160,42 @@ export function DocumentsTableShell({
|
|||
// State for metadata viewer (opened via Ctrl/Cmd+Click)
|
||||
const [metadataDoc, setMetadataDoc] = useState<Document | null>(null);
|
||||
|
||||
// State for lazy document content viewer
|
||||
// Real-time documents don't sync content - we fetch on-demand when viewing
|
||||
const [viewingDoc, setViewingDoc] = useState<Document | null>(null);
|
||||
const [viewingContent, setViewingContent] = useState<string>("");
|
||||
const [viewingLoading, setViewingLoading] = useState(false);
|
||||
|
||||
// Fetch document content on-demand when viewer is opened
|
||||
const handleViewDocument = useCallback(async (doc: Document) => {
|
||||
setViewingDoc(doc);
|
||||
|
||||
// If content is already available (from API/search), use it directly
|
||||
if (doc.content) {
|
||||
setViewingContent(doc.content);
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, fetch from API (lazy loading for real-time synced documents)
|
||||
setViewingLoading(true);
|
||||
try {
|
||||
const fullDoc = await documentsApiService.getDocument({ id: doc.id });
|
||||
setViewingContent(fullDoc.content);
|
||||
} catch (err) {
|
||||
console.error("[DocumentsTableShell] Failed to fetch document content:", err);
|
||||
setViewingContent("Failed to load document content.");
|
||||
} finally {
|
||||
setViewingLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Close document viewer
|
||||
const handleCloseViewer = useCallback(() => {
|
||||
setViewingDoc(null);
|
||||
setViewingContent("");
|
||||
setViewingLoading(false);
|
||||
}, []);
|
||||
|
||||
const sorted = React.useMemo(
|
||||
() => sortDocuments(documents, sortKey, sortDesc),
|
||||
[documents, sortKey, sortDesc]
|
||||
|
|
@ -185,7 +228,7 @@ export function DocumentsTableShell({
|
|||
|
||||
return (
|
||||
<motion.div
|
||||
className="rounded-lg border border-border/30 bg-background overflow-hidden"
|
||||
className="rounded-lg border border-border/40 bg-background overflow-hidden"
|
||||
initial={{ opacity: 0, y: 20 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ type: "spring", stiffness: 300, damping: 30, delay: 0.2 }}
|
||||
|
|
@ -196,22 +239,22 @@ export function DocumentsTableShell({
|
|||
<div className="hidden md:flex md:flex-col">
|
||||
<Table className="table-fixed w-full">
|
||||
<TableHeader>
|
||||
<TableRow className="hover:bg-transparent border-b border-border/30">
|
||||
<TableHead className="w-8 px-0 text-center border-r border-border/30">
|
||||
<TableRow className="hover:bg-transparent border-b border-border/40">
|
||||
<TableHead className="w-8 px-0 text-center border-r border-border/40">
|
||||
<div className="flex items-center justify-center h-full">
|
||||
<Skeleton className="h-4 w-4 rounded" />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="w-[35%] max-w-0 border-r border-border/30">
|
||||
<TableHead className="w-[35%] max-w-0 border-r border-border/40">
|
||||
<Skeleton className="h-3 w-20" />
|
||||
</TableHead>
|
||||
{columnVisibility.document_type && (
|
||||
<TableHead className="w-[20%] min-w-[120px] max-w-[200px] border-r border-border/30">
|
||||
<TableHead className="w-[20%] min-w-[120px] max-w-[200px] border-r border-border/40">
|
||||
<Skeleton className="h-3 w-14" />
|
||||
</TableHead>
|
||||
)}
|
||||
{columnVisibility.created_by && (
|
||||
<TableHead className="w-36 border-r border-border/30">
|
||||
<TableHead className="w-36 border-r border-border/40">
|
||||
<Skeleton className="h-3 w-10" />
|
||||
</TableHead>
|
||||
)}
|
||||
|
|
@ -229,26 +272,26 @@ export function DocumentsTableShell({
|
|||
{[65, 80, 45, 72, 55, 88, 40, 60, 50, 75].map((widthPercent, index) => (
|
||||
<TableRow
|
||||
key={`skeleton-${index}`}
|
||||
className="border-b border-border/30 hover:bg-transparent"
|
||||
className="border-b border-border/40 hover:bg-transparent"
|
||||
>
|
||||
<TableCell className="w-8 px-0 py-2.5 text-center border-r border-border/30">
|
||||
<TableCell className="w-8 px-0 py-2.5 text-center border-r border-border/40">
|
||||
<div className="flex items-center justify-center h-full">
|
||||
<Skeleton className="h-4 w-4 rounded" />
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell className="w-[35%] py-2.5 max-w-0 border-r border-border/30">
|
||||
<TableCell className="w-[35%] py-2.5 max-w-0 border-r border-border/40">
|
||||
<Skeleton
|
||||
className="h-4"
|
||||
style={{ width: `${widthPercent}%` }}
|
||||
/>
|
||||
</TableCell>
|
||||
{columnVisibility.document_type && (
|
||||
<TableCell className="w-[20%] min-w-[120px] max-w-[200px] py-2.5 border-r border-border/30 overflow-hidden">
|
||||
<TableCell className="w-[20%] min-w-[120px] max-w-[200px] py-2.5 border-r border-border/40 overflow-hidden">
|
||||
<Skeleton className="h-5 w-24 rounded" />
|
||||
</TableCell>
|
||||
)}
|
||||
{columnVisibility.created_by && (
|
||||
<TableCell className="w-36 py-2.5 truncate border-r border-border/30">
|
||||
<TableCell className="w-36 py-2.5 truncate border-r border-border/40">
|
||||
<Skeleton className="h-4 w-20" />
|
||||
</TableCell>
|
||||
)}
|
||||
|
|
@ -329,8 +372,8 @@ export function DocumentsTableShell({
|
|||
{/* Fixed Header */}
|
||||
<Table className="table-fixed w-full">
|
||||
<TableHeader>
|
||||
<TableRow className="hover:bg-transparent border-b border-border/30">
|
||||
<TableHead className="w-8 px-0 text-center border-r border-border/30">
|
||||
<TableRow className="hover:bg-transparent border-b border-border/40">
|
||||
<TableHead className="w-8 px-0 text-center border-r border-border/40">
|
||||
<div className="flex items-center justify-center h-full">
|
||||
<Checkbox
|
||||
checked={allSelectedOnPage || (someSelectedOnPage && "indeterminate")}
|
||||
|
|
@ -340,7 +383,7 @@ export function DocumentsTableShell({
|
|||
/>
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="w-[35%] border-r border-border/30">
|
||||
<TableHead className="w-[35%] border-r border-border/40">
|
||||
<SortableHeader
|
||||
sortKey="title"
|
||||
currentSortKey={sortKey}
|
||||
|
|
@ -352,7 +395,7 @@ export function DocumentsTableShell({
|
|||
</SortableHeader>
|
||||
</TableHead>
|
||||
{columnVisibility.document_type && (
|
||||
<TableHead className="w-[20%] min-w-[120px] max-w-[200px] border-r border-border/30">
|
||||
<TableHead className="w-[20%] min-w-[120px] max-w-[200px] border-r border-border/40">
|
||||
<SortableHeader
|
||||
sortKey="document_type"
|
||||
currentSortKey={sortKey}
|
||||
|
|
@ -365,7 +408,7 @@ export function DocumentsTableShell({
|
|||
</TableHead>
|
||||
)}
|
||||
{columnVisibility.created_by && (
|
||||
<TableHead className="w-36 border-r border-border/30">
|
||||
<TableHead className="w-36 border-r border-border/40">
|
||||
<span className="flex items-center gap-1.5 text-sm font-medium text-muted-foreground/70">
|
||||
<User size={14} className="opacity-60 text-muted-foreground" />
|
||||
User
|
||||
|
|
@ -406,13 +449,13 @@ export function DocumentsTableShell({
|
|||
delay: index * 0.02,
|
||||
},
|
||||
}}
|
||||
className={`border-b border-border/30 transition-colors ${
|
||||
className={`border-b border-border/40 transition-colors ${
|
||||
isSelected
|
||||
? "bg-primary/5 hover:bg-primary/8"
|
||||
: "hover:bg-muted/30"
|
||||
}`}
|
||||
>
|
||||
<TableCell className="w-8 px-0 py-2.5 text-center border-r border-border/30">
|
||||
<TableCell className="w-8 px-0 py-2.5 text-center border-r border-border/40">
|
||||
<div className="flex items-center justify-center h-full">
|
||||
<Checkbox
|
||||
checked={isSelected}
|
||||
|
|
@ -422,42 +465,42 @@ export function DocumentsTableShell({
|
|||
/>
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell className="w-[35%] py-2.5 max-w-0 border-r border-border/30">
|
||||
<DocumentViewer
|
||||
title={doc.title}
|
||||
content={doc.content}
|
||||
trigger={
|
||||
<button
|
||||
type="button"
|
||||
className="block w-full text-left text-sm text-foreground hover:text-foreground transition-colors cursor-pointer bg-transparent border-0 p-0 truncate"
|
||||
onClick={(e) => {
|
||||
// Ctrl (Win/Linux) or Cmd (Mac) + Click opens metadata
|
||||
if (e.ctrlKey || e.metaKey) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setMetadataDoc(doc);
|
||||
}
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
// Ctrl/Cmd + Enter opens metadata
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === "Enter") {
|
||||
e.preventDefault();
|
||||
setMetadataDoc(doc);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<TruncatedText text={title} className="truncate block" />
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
<TableCell className="w-[35%] py-2.5 max-w-0 border-r border-border/40">
|
||||
<button
|
||||
type="button"
|
||||
className="block w-full text-left text-sm text-foreground hover:text-foreground transition-colors cursor-pointer bg-transparent border-0 p-0 truncate"
|
||||
onClick={(e) => {
|
||||
// Ctrl (Win/Linux) or Cmd (Mac) + Click opens metadata
|
||||
if (e.ctrlKey || e.metaKey) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setMetadataDoc(doc);
|
||||
} else {
|
||||
// Normal click opens document viewer (lazy loads content)
|
||||
handleViewDocument(doc);
|
||||
}
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
// Ctrl/Cmd + Enter opens metadata
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === "Enter") {
|
||||
e.preventDefault();
|
||||
setMetadataDoc(doc);
|
||||
} else if (e.key === "Enter") {
|
||||
// Enter opens document viewer
|
||||
handleViewDocument(doc);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<TruncatedText text={title} className="truncate block" />
|
||||
</button>
|
||||
</TableCell>
|
||||
{columnVisibility.document_type && (
|
||||
<TableCell className="w-[20%] min-w-[120px] max-w-[200px] py-2.5 border-r border-border/30 overflow-hidden">
|
||||
<TableCell className="w-[20%] min-w-[120px] max-w-[200px] py-2.5 border-r border-border/40 overflow-hidden">
|
||||
<DocumentTypeChip type={doc.document_type} />
|
||||
</TableCell>
|
||||
)}
|
||||
{columnVisibility.created_by && (
|
||||
<TableCell className="w-36 py-2.5 text-sm text-foreground truncate border-r border-border/30">
|
||||
<TableCell className="w-36 py-2.5 text-sm text-foreground truncate border-r border-border/40">
|
||||
{doc.created_by_name || "—"}
|
||||
</TableCell>
|
||||
)}
|
||||
|
|
@ -482,7 +525,7 @@ export function DocumentsTableShell({
|
|||
</div>
|
||||
|
||||
{/* Mobile Card View - Notion Style */}
|
||||
<div className="md:hidden divide-y divide-border/30 h-[50vh] overflow-auto">
|
||||
<div className="md:hidden divide-y divide-border/40 h-[50vh] overflow-auto">
|
||||
{sorted.map((doc, index) => {
|
||||
const isSelected = selectedIds.has(doc.id);
|
||||
return (
|
||||
|
|
@ -502,33 +545,33 @@ export function DocumentsTableShell({
|
|||
className="border-foreground data-[state=checked]:bg-primary data-[state=checked]:border-primary"
|
||||
/>
|
||||
<div className="flex-1 min-w-0 space-y-1.5">
|
||||
<DocumentViewer
|
||||
title={doc.title}
|
||||
content={doc.content}
|
||||
trigger={
|
||||
<button
|
||||
type="button"
|
||||
className="text-left text-sm text-foreground hover:text-foreground transition-colors cursor-pointer truncate block w-full bg-transparent border-0 p-0"
|
||||
onClick={(e) => {
|
||||
// Ctrl (Win/Linux) or Cmd (Mac) + Click opens metadata
|
||||
if (e.ctrlKey || e.metaKey) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setMetadataDoc(doc);
|
||||
}
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
// Ctrl/Cmd + Enter opens metadata
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === "Enter") {
|
||||
e.preventDefault();
|
||||
setMetadataDoc(doc);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{doc.title}
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
className="text-left text-sm text-foreground hover:text-foreground transition-colors cursor-pointer truncate block w-full bg-transparent border-0 p-0"
|
||||
onClick={(e) => {
|
||||
// Ctrl (Win/Linux) or Cmd (Mac) + Click opens metadata
|
||||
if (e.ctrlKey || e.metaKey) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setMetadataDoc(doc);
|
||||
} else {
|
||||
// Normal click opens document viewer (lazy loads content)
|
||||
handleViewDocument(doc);
|
||||
}
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
// Ctrl/Cmd + Enter opens metadata
|
||||
if ((e.ctrlKey || e.metaKey) && e.key === "Enter") {
|
||||
e.preventDefault();
|
||||
setMetadataDoc(doc);
|
||||
} else if (e.key === "Enter") {
|
||||
// Enter opens document viewer
|
||||
handleViewDocument(doc);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{doc.title}
|
||||
</button>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<DocumentTypeChip type={doc.document_type} />
|
||||
{columnVisibility.created_by && doc.created_by_name && (
|
||||
|
|
@ -567,6 +610,24 @@ export function DocumentsTableShell({
|
|||
if (!open) setMetadataDoc(null);
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Document Content Viewer - lazy loads content on-demand */}
|
||||
<Dialog open={!!viewingDoc} onOpenChange={(open) => !open && handleCloseViewer()}>
|
||||
<DialogContent className="max-w-4xl max-h-[80vh] overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{viewingDoc?.title}</DialogTitle>
|
||||
</DialogHeader>
|
||||
<div className="mt-4">
|
||||
{viewingLoading ? (
|
||||
<div className="flex items-center justify-center py-12">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
) : (
|
||||
<MarkdownViewer content={viewingContent} />
|
||||
)}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</motion.div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ export type Document = {
|
|||
id: number;
|
||||
title: string;
|
||||
document_type: DocumentType;
|
||||
document_metadata: any;
|
||||
content: string;
|
||||
// Optional: Only needed when viewing document details (lazy loaded)
|
||||
document_metadata?: any;
|
||||
content?: string;
|
||||
created_at: string;
|
||||
search_space_id: number;
|
||||
created_by_id?: string | null;
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ import { useTranslations } from "next-intl";
|
|||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
|
||||
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { useDocuments } from "@/hooks/use-documents";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { DocumentsFilters } from "./components/DocumentsFilters";
|
||||
|
|
@ -43,21 +43,20 @@ export default function DocumentsTable() {
|
|||
const [sortKey, setSortKey] = useState<SortKey>("created_at");
|
||||
const [sortDesc, setSortDesc] = useState(true);
|
||||
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
|
||||
const { data: rawTypeCounts } = useAtomValue(documentTypeCountsAtom);
|
||||
const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
|
||||
|
||||
// Build query parameters for fetching documents
|
||||
const queryParams = useMemo(
|
||||
() => ({
|
||||
search_space_id: searchSpaceId,
|
||||
page: pageIndex,
|
||||
page_size: PAGE_SIZE,
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
}),
|
||||
[searchSpaceId, pageIndex, activeTypes]
|
||||
);
|
||||
// REAL-TIME: Use Electric SQL hook for live document updates (when not searching)
|
||||
const {
|
||||
documents: realtimeDocuments,
|
||||
typeCounts: realtimeTypeCounts,
|
||||
loading: realtimeLoading,
|
||||
error: realtimeError,
|
||||
} = useDocuments(searchSpaceId, activeTypes);
|
||||
|
||||
// Build search query parameters
|
||||
// Check if we're in search mode
|
||||
const isSearchMode = !!debouncedSearch.trim();
|
||||
|
||||
// Build search query parameters (only used when searching)
|
||||
const searchQueryParams = useMemo(
|
||||
() => ({
|
||||
search_space_id: searchSpaceId,
|
||||
|
|
@ -69,20 +68,7 @@ export default function DocumentsTable() {
|
|||
[searchSpaceId, pageIndex, activeTypes, debouncedSearch]
|
||||
);
|
||||
|
||||
// Use query for fetching documents
|
||||
const {
|
||||
data: documentsResponse,
|
||||
isLoading: isDocumentsLoading,
|
||||
refetch: refetchDocuments,
|
||||
error: documentsError,
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.globalQueryParams(queryParams),
|
||||
queryFn: () => documentsApiService.getDocuments({ queryParams }),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: !!searchSpaceId && !debouncedSearch.trim(),
|
||||
});
|
||||
|
||||
// Use query for searching documents
|
||||
// API search query (only enabled when searching - Electric doesn't do full-text search)
|
||||
const {
|
||||
data: searchResponse,
|
||||
isLoading: isSearchLoading,
|
||||
|
|
@ -91,73 +77,59 @@ export default function DocumentsTable() {
|
|||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams),
|
||||
queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: !!searchSpaceId && !!debouncedSearch.trim(),
|
||||
staleTime: 30 * 1000, // 30 seconds for search (shorter since it's on-demand)
|
||||
enabled: !!searchSpaceId && isSearchMode,
|
||||
});
|
||||
|
||||
// Determine if we should show SurfSense docs (when no type filter or SURFSENSE_DOCS is selected)
|
||||
const showSurfsenseDocs =
|
||||
activeTypes.length === 0 || activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum);
|
||||
// Client-side sorting for real-time documents
|
||||
const sortedRealtimeDocuments = useMemo(() => {
|
||||
const docs = [...realtimeDocuments];
|
||||
docs.sort((a, b) => {
|
||||
const av = a[sortKey] ?? "";
|
||||
const bv = b[sortKey] ?? "";
|
||||
let cmp: number;
|
||||
if (sortKey === "created_at") {
|
||||
cmp = new Date(av as string).getTime() - new Date(bv as string).getTime();
|
||||
} else {
|
||||
cmp = String(av).localeCompare(String(bv));
|
||||
}
|
||||
return sortDesc ? -cmp : cmp;
|
||||
});
|
||||
return docs;
|
||||
}, [realtimeDocuments, sortKey, sortDesc]);
|
||||
|
||||
// Use query for fetching SurfSense docs
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { data: surfsenseDocsResponse } = useQuery({
|
||||
queryKey: ["surfsense-docs", debouncedSearch, pageIndex, PAGE_SIZE],
|
||||
queryFn: () =>
|
||||
documentsApiService.getSurfsenseDocs({
|
||||
queryParams: {
|
||||
page: pageIndex,
|
||||
page_size: PAGE_SIZE,
|
||||
title: debouncedSearch.trim() || undefined,
|
||||
},
|
||||
}),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: showSurfsenseDocs,
|
||||
});
|
||||
// Client-side pagination for real-time documents
|
||||
const paginatedRealtimeDocuments = useMemo(() => {
|
||||
const start = pageIndex * PAGE_SIZE;
|
||||
const end = start + PAGE_SIZE;
|
||||
return sortedRealtimeDocuments.slice(start, end);
|
||||
}, [sortedRealtimeDocuments, pageIndex]);
|
||||
|
||||
// Transform SurfSense docs to match the Document type
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const surfsenseDocsAsDocuments = useMemo(() => {
|
||||
if (!surfsenseDocsResponse?.items) return [];
|
||||
return surfsenseDocsResponse.items.map((doc) => ({
|
||||
id: doc.id,
|
||||
title: doc.title,
|
||||
document_type: "SURFSENSE_DOCS",
|
||||
document_metadata: { source: doc.source },
|
||||
content: doc.content,
|
||||
created_at: new Date().toISOString(),
|
||||
search_space_id: -1, // Special value for global docs
|
||||
}));
|
||||
}, [surfsenseDocsResponse]);
|
||||
// Determine what to display based on search mode
|
||||
const displayDocs = isSearchMode
|
||||
? (searchResponse?.items || []).map((item) => ({
|
||||
id: item.id,
|
||||
search_space_id: item.search_space_id,
|
||||
document_type: item.document_type,
|
||||
title: item.title,
|
||||
created_by_id: item.created_by_id ?? null,
|
||||
created_by_name: item.created_by_name ?? null,
|
||||
created_at: item.created_at,
|
||||
}))
|
||||
: paginatedRealtimeDocuments;
|
||||
|
||||
// Merge type counts with SURFSENSE_DOCS count
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const typeCounts = useMemo(() => {
|
||||
const counts = { ...(rawTypeCounts || {}) };
|
||||
if (surfsenseDocsResponse?.total) {
|
||||
counts.SURFSENSE_DOCS = surfsenseDocsResponse.total;
|
||||
}
|
||||
return counts;
|
||||
}, [rawTypeCounts, surfsenseDocsResponse?.total]);
|
||||
const displayTotal = isSearchMode
|
||||
? searchResponse?.total || 0
|
||||
: sortedRealtimeDocuments.length;
|
||||
|
||||
// Extract documents and total based on search state
|
||||
const documents = debouncedSearch.trim()
|
||||
? searchResponse?.items || []
|
||||
: documentsResponse?.items || [];
|
||||
const total = debouncedSearch.trim() ? searchResponse?.total || 0 : documentsResponse?.total || 0;
|
||||
const loading = isSearchMode ? isSearchLoading : realtimeLoading;
|
||||
const error = isSearchMode ? searchError : realtimeError;
|
||||
|
||||
const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
|
||||
const error = debouncedSearch.trim() ? searchError : documentsError;
|
||||
|
||||
// Display results directly
|
||||
const displayDocs = documents;
|
||||
const displayTotal = total;
|
||||
const pageEnd = Math.min((pageIndex + 1) * PAGE_SIZE, displayTotal);
|
||||
|
||||
const onToggleType = (type: DocumentTypeEnum, checked: boolean) => {
|
||||
setActiveTypes((prev) => {
|
||||
if (checked) {
|
||||
// Only add if not already in the array
|
||||
return prev.includes(type) ? prev : [...prev, type];
|
||||
} else {
|
||||
return prev.filter((t) => t !== type);
|
||||
|
|
@ -176,16 +148,15 @@ export default function DocumentsTable() {
|
|||
if (isRefreshing) return;
|
||||
setIsRefreshing(true);
|
||||
try {
|
||||
if (debouncedSearch.trim()) {
|
||||
if (isSearchMode) {
|
||||
await refetchSearch();
|
||||
} else {
|
||||
await refetchDocuments();
|
||||
}
|
||||
// Real-time view doesn't need manual refresh - Electric handles it
|
||||
toast.success(t("refresh_success") || "Documents refreshed");
|
||||
} finally {
|
||||
setIsRefreshing(false);
|
||||
}
|
||||
}, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]);
|
||||
}, [isSearchMode, refetchSearch, t, isRefreshing]);
|
||||
|
||||
const onBulkDelete = async () => {
|
||||
if (selectedIds.size === 0) {
|
||||
|
|
@ -208,7 +179,13 @@ export default function DocumentsTable() {
|
|||
if (okCount === selectedIds.size)
|
||||
toast.success(t("delete_success_count", { count: okCount }));
|
||||
else toast.error(t("delete_partial_failed"));
|
||||
// Note: No need to call refreshCurrentView() - the mutation already updates the cache
|
||||
|
||||
// If in search mode, refetch search results to reflect deletion
|
||||
if (isSearchMode) {
|
||||
await refetchSearch();
|
||||
}
|
||||
// Real-time mode: Electric will sync the deletion automatically
|
||||
|
||||
setSelectedIds(new Set());
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
|
@ -227,6 +204,12 @@ export default function DocumentsTable() {
|
|||
});
|
||||
}, []);
|
||||
|
||||
// Reset page when search changes (type filter already resets via onToggleType)
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: Intentionally reset page on search change
|
||||
useEffect(() => {
|
||||
setPageIndex(0);
|
||||
}, [debouncedSearch]);
|
||||
|
||||
useEffect(() => {
|
||||
const mq = window.matchMedia("(max-width: 768px)");
|
||||
const apply = (isSmall: boolean) => {
|
||||
|
|
@ -245,9 +228,9 @@ export default function DocumentsTable() {
|
|||
transition={{ duration: 0.3 }}
|
||||
className="w-full max-w-7xl mx-auto px-6 pt-17 pb-6 space-y-6 min-h-[calc(100vh-64px)]"
|
||||
>
|
||||
{/* Filters */}
|
||||
{/* Filters - use real-time type counts */}
|
||||
<DocumentsFilters
|
||||
typeCounts={rawTypeCounts ?? {}}
|
||||
typeCounts={realtimeTypeCounts}
|
||||
selectedIds={selectedIds}
|
||||
onSearch={setSearch}
|
||||
searchValue={search}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ export const uploadDocumentMutationAtom = atomWithMutation((get) => {
|
|||
},
|
||||
|
||||
onSuccess: () => {
|
||||
toast.success("Files uploaded for processing");
|
||||
// Note: Toast notification is handled by the caller (DocumentUploadTab) to use i18n
|
||||
// Invalidate logs summary to show new processing tasks immediately on documents page
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined),
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import { Spinner } from "@/components/ui/spinner";
|
|||
import { Tabs, TabsContent } from "@/components/ui/tabs";
|
||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
|
||||
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
|
||||
import { useDocuments } from "@/hooks/use-documents";
|
||||
import { useInbox } from "@/hooks/use-inbox";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header";
|
||||
|
|
@ -63,7 +63,9 @@ export const ConnectorIndicator: FC<{ hideTrigger?: boolean }> = ({ hideTrigger
|
|||
const llmConfigLoading = preferencesLoading || globalConfigsLoading;
|
||||
|
||||
// Fetch document type counts using Electric SQL + PGlite for real-time updates
|
||||
const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId);
|
||||
const { typeCounts: documentTypeCounts, loading: documentTypesLoading } = useDocuments(
|
||||
searchSpaceId ? Number(searchSpaceId) : null
|
||||
);
|
||||
|
||||
// Fetch notifications to detect indexing failures
|
||||
const { inboxItems = [] } = useInbox(
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ export const document = z.object({
|
|||
created_at: z.string(),
|
||||
updated_at: z.string().nullable(),
|
||||
search_space_id: z.number(),
|
||||
created_by_id: z.string().nullable().optional(),
|
||||
created_by_name: z.string().nullable().optional(),
|
||||
});
|
||||
|
||||
export const extensionDocumentContent = z.object({
|
||||
|
|
|
|||
|
|
@ -1,185 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import type { SyncHandle } from "@/lib/electric/client";
|
||||
import { useElectricClient } from "@/lib/electric/context";
|
||||
|
||||
interface Document {
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
document_type: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for managing documents with Electric SQL real-time sync
|
||||
*
|
||||
* Uses the Electric client from context (provided by ElectricProvider)
|
||||
* instead of initializing its own - prevents race conditions and memory leaks
|
||||
*/
|
||||
export function useDocumentsElectric(searchSpaceId: number | string | null) {
|
||||
// Get Electric client from context - ElectricProvider handles initialization
|
||||
const electricClient = useElectricClient();
|
||||
|
||||
const [documents, setDocuments] = useState<Document[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<Error | null>(null);
|
||||
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
|
||||
const syncKeyRef = useRef<string | null>(null);
|
||||
|
||||
// Calculate document type counts from synced documents
|
||||
const documentTypeCounts = useMemo(() => {
|
||||
if (!documents.length) return {};
|
||||
|
||||
const counts: Record<string, number> = {};
|
||||
for (const doc of documents) {
|
||||
counts[doc.document_type] = (counts[doc.document_type] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}, [documents]);
|
||||
|
||||
// Start syncing when Electric client is available
|
||||
useEffect(() => {
|
||||
// Wait for both searchSpaceId and Electric client to be available
|
||||
if (!searchSpaceId || !electricClient) {
|
||||
setLoading(!electricClient); // Still loading if waiting for Electric
|
||||
if (!searchSpaceId) {
|
||||
setDocuments([]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a unique key for this sync to prevent duplicate subscriptions
|
||||
const syncKey = `documents_${searchSpaceId}`;
|
||||
if (syncKeyRef.current === syncKey) {
|
||||
// Already syncing for this search space
|
||||
return;
|
||||
}
|
||||
|
||||
let mounted = true;
|
||||
syncKeyRef.current = syncKey;
|
||||
|
||||
async function startSync() {
|
||||
try {
|
||||
console.log("[useDocumentsElectric] Starting sync for search space:", searchSpaceId);
|
||||
|
||||
const handle = await electricClient.syncShape({
|
||||
table: "documents",
|
||||
where: `search_space_id = ${searchSpaceId}`,
|
||||
columns: ["id", "document_type", "search_space_id", "created_at"],
|
||||
primaryKey: ["id"],
|
||||
});
|
||||
|
||||
console.log("[useDocumentsElectric] Sync started:", {
|
||||
isUpToDate: handle.isUpToDate,
|
||||
});
|
||||
|
||||
// Wait for initial sync with timeout
|
||||
if (!handle.isUpToDate && handle.initialSyncPromise) {
|
||||
try {
|
||||
await Promise.race([
|
||||
handle.initialSyncPromise,
|
||||
new Promise((resolve) => setTimeout(resolve, 2000)),
|
||||
]);
|
||||
} catch (syncErr) {
|
||||
console.error("[useDocumentsElectric] Initial sync failed:", syncErr);
|
||||
}
|
||||
}
|
||||
|
||||
if (!mounted) {
|
||||
handle.unsubscribe();
|
||||
return;
|
||||
}
|
||||
|
||||
syncHandleRef.current = handle;
|
||||
setLoading(false);
|
||||
setError(null);
|
||||
|
||||
// Fetch initial documents
|
||||
await fetchDocuments();
|
||||
|
||||
// Set up live query for real-time updates
|
||||
await setupLiveQuery();
|
||||
} catch (err) {
|
||||
if (!mounted) return;
|
||||
console.error("[useDocumentsElectric] Failed to start sync:", err);
|
||||
setError(err instanceof Error ? err : new Error("Failed to sync documents"));
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchDocuments() {
|
||||
try {
|
||||
const result = await electricClient.db.query<Document>(
|
||||
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
||||
[searchSpaceId]
|
||||
);
|
||||
if (mounted) {
|
||||
setDocuments(result.rows || []);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[useDocumentsElectric] Failed to fetch:", err);
|
||||
}
|
||||
}
|
||||
|
||||
async function setupLiveQuery() {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const db = electricClient.db as any;
|
||||
|
||||
if (db.live?.query && typeof db.live.query === "function") {
|
||||
const liveQuery = await db.live.query(
|
||||
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
||||
[searchSpaceId]
|
||||
);
|
||||
|
||||
if (!mounted) {
|
||||
liveQuery.unsubscribe?.();
|
||||
return;
|
||||
}
|
||||
|
||||
// Set initial results
|
||||
if (liveQuery.initialResults?.rows) {
|
||||
setDocuments(liveQuery.initialResults.rows);
|
||||
} else if (liveQuery.rows) {
|
||||
setDocuments(liveQuery.rows);
|
||||
}
|
||||
|
||||
// Subscribe to changes
|
||||
if (typeof liveQuery.subscribe === "function") {
|
||||
liveQuery.subscribe((result: { rows: Document[] }) => {
|
||||
if (mounted && result.rows) {
|
||||
setDocuments(result.rows);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof liveQuery.unsubscribe === "function") {
|
||||
liveQueryRef.current = liveQuery;
|
||||
}
|
||||
}
|
||||
} catch (liveErr) {
|
||||
console.error("[useDocumentsElectric] Failed to set up live query:", liveErr);
|
||||
}
|
||||
}
|
||||
|
||||
startSync();
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
syncKeyRef.current = null;
|
||||
|
||||
if (syncHandleRef.current) {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
if (liveQueryRef.current) {
|
||||
liveQueryRef.current.unsubscribe();
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [searchSpaceId, electricClient]);
|
||||
|
||||
return { documentTypeCounts, loading, error };
|
||||
}
|
||||
427
surfsense_web/hooks/use-documents.ts
Normal file
427
surfsense_web/hooks/use-documents.ts
Normal file
|
|
@ -0,0 +1,427 @@
|
|||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import type { SyncHandle } from "@/lib/electric/client";
|
||||
import { useElectricClient } from "@/lib/electric/context";
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
// Stable empty array to prevent infinite re-renders when no typeFilter is provided
|
||||
const EMPTY_TYPE_FILTER: DocumentTypeEnum[] = [];
|
||||
|
||||
// Document from Electric sync (lightweight table columns - NO content/metadata)
|
||||
interface DocumentElectric {
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
document_type: string;
|
||||
title: string;
|
||||
created_by_id: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
// Document for display (with resolved user name)
|
||||
export interface DocumentDisplay {
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
document_type: string;
|
||||
title: string;
|
||||
created_by_id: string | null;
|
||||
created_by_name: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate by ID and sort by created_at descending (newest first)
|
||||
*/
|
||||
function deduplicateAndSort<T extends { id: number; created_at: string }>(items: T[]): T[] {
|
||||
const seen = new Map<number, T>();
|
||||
for (const item of items) {
|
||||
// Keep the most recent version if duplicate
|
||||
const existing = seen.get(item.id);
|
||||
if (!existing || new Date(item.created_at) > new Date(existing.created_at)) {
|
||||
seen.set(item.id, item);
|
||||
}
|
||||
}
|
||||
return Array.from(seen.values()).sort(
|
||||
(a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a document has valid/complete data
|
||||
*/
|
||||
function isValidDocument(doc: DocumentElectric): boolean {
|
||||
return doc.id != null && doc.title != null && doc.title !== "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Real-time documents hook with Electric SQL
|
||||
*
|
||||
* Architecture (100% Reliable):
|
||||
* 1. API is the PRIMARY source of truth - always loads first
|
||||
* 2. Electric provides REAL-TIME updates for additions and deletions
|
||||
* 3. Use syncHandle.isUpToDate to determine if deletions can be trusted
|
||||
* 4. Handles bulk deletions correctly by checking sync state
|
||||
*
|
||||
* @param searchSpaceId - The search space ID to filter documents
|
||||
* @param typeFilter - Optional document types to filter by
|
||||
*/
|
||||
export function useDocuments(
|
||||
searchSpaceId: number | null,
|
||||
typeFilter: DocumentTypeEnum[] = EMPTY_TYPE_FILTER
|
||||
) {
|
||||
const electricClient = useElectricClient();
|
||||
|
||||
const [documents, setDocuments] = useState<DocumentDisplay[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<Error | null>(null);
|
||||
|
||||
// Track if initial API load is complete (source of truth)
|
||||
const apiLoadedRef = useRef(false);
|
||||
|
||||
// User cache: userId → displayName
|
||||
const userCacheRef = useRef<Map<string, string>>(new Map());
|
||||
|
||||
// Electric sync refs
|
||||
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||
const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null);
|
||||
|
||||
// Real-time type counts
|
||||
const typeCounts = useMemo(() => {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const doc of documents) {
|
||||
counts[doc.document_type] = (counts[doc.document_type] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}, [documents]);
|
||||
|
||||
// Populate user cache from API response
|
||||
const populateUserCache = useCallback(
|
||||
(items: Array<{ created_by_id?: string | null; created_by_name?: string | null }>) => {
|
||||
for (const item of items) {
|
||||
if (item.created_by_id && item.created_by_name) {
|
||||
userCacheRef.current.set(item.created_by_id, item.created_by_name);
|
||||
}
|
||||
}
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
// Convert API item to display doc
|
||||
const apiToDisplayDoc = useCallback(
|
||||
(item: {
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
document_type: string;
|
||||
title: string;
|
||||
created_by_id?: string | null;
|
||||
created_by_name?: string | null;
|
||||
created_at: string;
|
||||
}): DocumentDisplay => ({
|
||||
id: item.id,
|
||||
search_space_id: item.search_space_id,
|
||||
document_type: item.document_type,
|
||||
title: item.title,
|
||||
created_by_id: item.created_by_id ?? null,
|
||||
created_by_name: item.created_by_name ?? null,
|
||||
created_at: item.created_at,
|
||||
}),
|
||||
[]
|
||||
);
|
||||
|
||||
// Convert Electric doc to display doc
|
||||
const electricToDisplayDoc = useCallback(
|
||||
(doc: DocumentElectric): DocumentDisplay => ({
|
||||
...doc,
|
||||
created_by_name: doc.created_by_id
|
||||
? userCacheRef.current.get(doc.created_by_id) ?? null
|
||||
: null,
|
||||
}),
|
||||
[]
|
||||
);
|
||||
|
||||
// EFFECT 1: Load from API (PRIMARY source of truth)
|
||||
useEffect(() => {
|
||||
if (!searchSpaceId) {
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Capture validated value for async closure
|
||||
const spaceId = searchSpaceId;
|
||||
const currentTypeFilter = typeFilter;
|
||||
|
||||
let mounted = true;
|
||||
apiLoadedRef.current = false;
|
||||
|
||||
async function loadFromApi() {
|
||||
try {
|
||||
setLoading(true);
|
||||
console.log("[useDocuments] Loading from API (source of truth):", spaceId);
|
||||
|
||||
const response = await documentsApiService.getDocuments({
|
||||
queryParams: {
|
||||
search_space_id: spaceId,
|
||||
page: 0,
|
||||
page_size: PAGE_SIZE,
|
||||
...(currentTypeFilter.length > 0 && { document_types: currentTypeFilter }),
|
||||
},
|
||||
});
|
||||
|
||||
if (!mounted) return;
|
||||
|
||||
populateUserCache(response.items);
|
||||
const docs = response.items.map(apiToDisplayDoc);
|
||||
setDocuments(docs);
|
||||
apiLoadedRef.current = true;
|
||||
setError(null);
|
||||
console.log("[useDocuments] API loaded", docs.length, "documents");
|
||||
} catch (err) {
|
||||
if (!mounted) return;
|
||||
console.error("[useDocuments] API load failed:", err);
|
||||
setError(err instanceof Error ? err : new Error("Failed to load documents"));
|
||||
} finally {
|
||||
if (mounted) setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
loadFromApi();
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
};
|
||||
}, [searchSpaceId, typeFilter, populateUserCache, apiToDisplayDoc]);
|
||||
|
||||
// EFFECT 2: Start Electric sync + live query for real-time updates
|
||||
useEffect(() => {
|
||||
if (!searchSpaceId || !electricClient) return;
|
||||
|
||||
// Capture validated values for async closure
|
||||
const spaceId = searchSpaceId;
|
||||
const client = electricClient;
|
||||
const currentTypeFilter = typeFilter;
|
||||
|
||||
let mounted = true;
|
||||
|
||||
async function setupElectricRealtime() {
|
||||
// Cleanup previous subscriptions
|
||||
if (syncHandleRef.current) {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
if (liveQueryRef.current) {
|
||||
liveQueryRef.current.unsubscribe?.();
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log("[useDocuments] Starting Electric sync for real-time updates");
|
||||
|
||||
// Start Electric sync
|
||||
const handle = await client.syncShape({
|
||||
table: "documents",
|
||||
where: `search_space_id = ${spaceId}`,
|
||||
columns: ["id", "document_type", "search_space_id", "title", "created_by_id", "created_at"],
|
||||
primaryKey: ["id"],
|
||||
});
|
||||
|
||||
if (!mounted) {
|
||||
handle.unsubscribe();
|
||||
return;
|
||||
}
|
||||
|
||||
syncHandleRef.current = handle;
|
||||
console.log("[useDocuments] Sync started, isUpToDate:", handle.isUpToDate);
|
||||
|
||||
// Wait for initial sync (with timeout)
|
||||
if (!handle.isUpToDate && handle.initialSyncPromise) {
|
||||
await Promise.race([
|
||||
handle.initialSyncPromise,
|
||||
new Promise((resolve) => setTimeout(resolve, 5000)),
|
||||
]);
|
||||
console.log("[useDocuments] Initial sync complete, isUpToDate:", handle.isUpToDate);
|
||||
}
|
||||
|
||||
if (!mounted) return;
|
||||
|
||||
// Set up live query
|
||||
const db = client.db as {
|
||||
live?: {
|
||||
query: <T>(sql: string, params?: (number | string)[]) => Promise<{
|
||||
subscribe: (cb: (result: { rows: T[] }) => void) => void;
|
||||
unsubscribe?: () => void;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
|
||||
if (!db.live?.query) {
|
||||
console.warn("[useDocuments] Live queries not available");
|
||||
return;
|
||||
}
|
||||
|
||||
let query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at
|
||||
FROM documents
|
||||
WHERE search_space_id = $1`;
|
||||
|
||||
const params: (number | string)[] = [spaceId];
|
||||
|
||||
if (currentTypeFilter.length > 0) {
|
||||
const placeholders = currentTypeFilter.map((_, i) => `$${i + 2}`).join(", ");
|
||||
query += ` AND document_type IN (${placeholders})`;
|
||||
params.push(...currentTypeFilter);
|
||||
}
|
||||
|
||||
query += ` ORDER BY created_at DESC`;
|
||||
|
||||
const liveQuery = await db.live.query<DocumentElectric>(query, params);
|
||||
|
||||
if (!mounted) {
|
||||
liveQuery.unsubscribe?.();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[useDocuments] Live query subscribed");
|
||||
|
||||
liveQuery.subscribe((result: { rows: DocumentElectric[] }) => {
|
||||
if (!mounted || !result.rows) return;
|
||||
|
||||
// DEBUG: Log first few raw documents to see what's coming from Electric
|
||||
console.log("[useDocuments] Raw data sample:", result.rows.slice(0, 3));
|
||||
|
||||
|
||||
const validItems = result.rows.filter(isValidDocument);
|
||||
const isFullySynced = syncHandleRef.current?.isUpToDate ?? false;
|
||||
|
||||
console.log(
|
||||
`[useDocuments] Live update: ${result.rows.length} raw, ${validItems.length} valid, synced: ${isFullySynced}`
|
||||
);
|
||||
|
||||
// Fetch user names for new users (non-blocking)
|
||||
const unknownUserIds = validItems
|
||||
.filter((doc): doc is DocumentElectric & { created_by_id: string } =>
|
||||
doc.created_by_id !== null && !userCacheRef.current.has(doc.created_by_id)
|
||||
)
|
||||
.map((doc) => doc.created_by_id);
|
||||
|
||||
if (unknownUserIds.length > 0) {
|
||||
documentsApiService
|
||||
.getDocuments({
|
||||
queryParams: { search_space_id: spaceId, page: 0, page_size: 20 },
|
||||
})
|
||||
.then((response) => {
|
||||
populateUserCache(response.items);
|
||||
if (mounted) {
|
||||
setDocuments((prev) =>
|
||||
prev.map((doc) => ({
|
||||
...doc,
|
||||
created_by_name: doc.created_by_id
|
||||
? userCacheRef.current.get(doc.created_by_id) ?? null
|
||||
: null,
|
||||
}))
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
// Smart update logic based on sync state
|
||||
setDocuments((prev) => {
|
||||
// Don't process if API hasn't loaded yet
|
||||
if (!apiLoadedRef.current) {
|
||||
console.log("[useDocuments] Waiting for API load, skipping live update");
|
||||
return prev;
|
||||
}
|
||||
|
||||
// Case 1: Live query is empty
|
||||
if (validItems.length === 0) {
|
||||
if (isFullySynced && prev.length > 0) {
|
||||
// Electric is fully synced and says 0 items - trust it (all deleted)
|
||||
console.log("[useDocuments] All documents deleted (Electric synced)");
|
||||
return [];
|
||||
}
|
||||
// Partial sync or error - keep existing
|
||||
console.log("[useDocuments] Empty live result, keeping existing");
|
||||
return prev;
|
||||
}
|
||||
|
||||
// Case 2: Electric is fully synced - TRUST IT COMPLETELY (handles bulk deletes)
|
||||
if (isFullySynced) {
|
||||
const liveDocs = deduplicateAndSort(validItems.map(electricToDisplayDoc));
|
||||
console.log(`[useDocuments] Synced update: ${liveDocs.length} docs (was ${prev.length})`);
|
||||
return liveDocs;
|
||||
}
|
||||
|
||||
// Case 3: Partial sync - only ADD new items, don't remove any
|
||||
const existingIds = new Set(prev.map((d) => d.id));
|
||||
const liveIds = new Set(validItems.map((d) => d.id));
|
||||
|
||||
// Find new items (in live but not in prev)
|
||||
const newItems = validItems
|
||||
.filter((item) => !existingIds.has(item.id))
|
||||
.map(electricToDisplayDoc);
|
||||
|
||||
// Find updated items (in both, update with latest data)
|
||||
const updatedPrev = prev.map((doc) => {
|
||||
if (liveIds.has(doc.id)) {
|
||||
const liveItem = validItems.find((v) => v.id === doc.id);
|
||||
if (liveItem) {
|
||||
return electricToDisplayDoc(liveItem);
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
});
|
||||
|
||||
if (newItems.length > 0) {
|
||||
console.log(`[useDocuments] Adding ${newItems.length} new items (partial sync)`);
|
||||
return deduplicateAndSort([...newItems, ...updatedPrev]);
|
||||
}
|
||||
|
||||
return updatedPrev;
|
||||
});
|
||||
});
|
||||
|
||||
liveQueryRef.current = liveQuery;
|
||||
} catch (err) {
|
||||
console.error("[useDocuments] Electric setup failed:", err);
|
||||
// Don't set error - API data is already loaded
|
||||
}
|
||||
}
|
||||
|
||||
setupElectricRealtime();
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
if (syncHandleRef.current) {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
if (liveQueryRef.current) {
|
||||
liveQueryRef.current.unsubscribe?.();
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [searchSpaceId, electricClient, typeFilter, electricToDisplayDoc, populateUserCache]);
|
||||
|
||||
// Track previous searchSpaceId to detect actual changes
|
||||
const prevSearchSpaceIdRef = useRef<number | null>(null);
|
||||
|
||||
// Reset on search space change (not on initial mount)
|
||||
useEffect(() => {
|
||||
if (prevSearchSpaceIdRef.current !== null && prevSearchSpaceIdRef.current !== searchSpaceId) {
|
||||
setDocuments([]);
|
||||
apiLoadedRef.current = false;
|
||||
userCacheRef.current.clear();
|
||||
}
|
||||
prevSearchSpaceIdRef.current = searchSpaceId;
|
||||
}, [searchSpaceId]);
|
||||
|
||||
return {
|
||||
documents,
|
||||
typeCounts,
|
||||
total: documents.length,
|
||||
loading,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
|
@ -70,7 +70,9 @@ const pendingSyncs = new Map<string, Promise<SyncHandle>>();
|
|||
// v5: fixed duplicate key errors (root cause: unstable cutoff dates in use-inbox.ts)
|
||||
// - added onMustRefetch handler for server-side refetch scenarios
|
||||
// - fixed getSyncCutoffDate to use stable midnight UTC timestamps
|
||||
const SYNC_VERSION = 5;
|
||||
// v6: real-time documents table - added title and created_by_id columns for live document display
|
||||
// v7: removed use-documents-electric.ts - consolidated to single documents sync to prevent conflicts
|
||||
const SYNC_VERSION = 7;
|
||||
|
||||
// Database name prefix for identifying SurfSense databases
|
||||
const DB_PREFIX = "surfsense-";
|
||||
|
|
@ -235,12 +237,14 @@ export async function initElectric(userId: string): Promise<ElectricClient> {
|
|||
`);
|
||||
|
||||
// Create the documents table schema in PGlite
|
||||
// Only sync minimal fields needed for type counts: id, document_type, search_space_id
|
||||
// Sync columns needed for real-time table display (lightweight - no content/metadata)
|
||||
await db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS documents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
search_space_id INTEGER NOT NULL,
|
||||
document_type TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
created_by_id TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue