diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx index 7e5f95af4..dd32a3b78 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx @@ -1,15 +1,21 @@ "use client"; import { formatDistanceToNow } from "date-fns"; -import {Calendar, ChevronDown, ChevronUp, FileText, FileX, Network, Plus, User } from "lucide-react"; +import { Calendar, ChevronDown, ChevronUp, FileText, FileX, Loader2, Network, Plus, User } from "lucide-react"; import { motion } from "motion/react"; import { useTranslations } from "next-intl"; -import React, { useRef, useState, useEffect } from "react"; +import React, { useRef, useState, useEffect, useCallback } from "react"; import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup"; -import { DocumentViewer } from "@/components/document-viewer"; import { JsonMetadataViewer } from "@/components/json-metadata-viewer"; +import { MarkdownViewer } from "@/components/markdown-viewer"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; import { Skeleton } from "@/components/ui/skeleton"; import { Table, @@ -20,6 +26,7 @@ import { TableRow, } from "@/components/ui/table"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; import { DocumentTypeChip } from "./DocumentTypeIcon"; import type { ColumnVisibility, Document } from "./types"; @@ -153,6 +160,42 @@ export function DocumentsTableShell({ // State for metadata viewer (opened via Ctrl/Cmd+Click) const [metadataDoc, setMetadataDoc] = useState(null); + // State for lazy document content viewer + // Real-time documents don't sync content - we fetch on-demand when viewing + const [viewingDoc, setViewingDoc] = useState(null); + const [viewingContent, setViewingContent] = useState(""); + const [viewingLoading, setViewingLoading] = useState(false); + + // Fetch document content on-demand when viewer is opened + const handleViewDocument = useCallback(async (doc: Document) => { + setViewingDoc(doc); + + // If content is already available (from API/search), use it directly + if (doc.content) { + setViewingContent(doc.content); + return; + } + + // Otherwise, fetch from API (lazy loading for real-time synced documents) + setViewingLoading(true); + try { + const fullDoc = await documentsApiService.getDocument({ id: doc.id }); + setViewingContent(fullDoc.content); + } catch (err) { + console.error("[DocumentsTableShell] Failed to fetch document content:", err); + setViewingContent("Failed to load document content."); + } finally { + setViewingLoading(false); + } + }, []); + + // Close document viewer + const handleCloseViewer = useCallback(() => { + setViewingDoc(null); + setViewingContent(""); + setViewingLoading(false); + }, []); + const sorted = React.useMemo( () => sortDocuments(documents, sortKey, sortDesc), [documents, sortKey, sortDesc] @@ -185,7 +228,7 @@ export function DocumentsTableShell({ return ( - - + +
- + {columnVisibility.document_type && ( - + )} {columnVisibility.created_by && ( - + )} @@ -229,26 +272,26 @@ export function DocumentsTableShell({ {[65, 80, 45, 72, 55, 88, 40, 60, 50, 75].map((widthPercent, index) => ( - +
- + {columnVisibility.document_type && ( - + )} {columnVisibility.created_by && ( - + )} @@ -329,8 +372,8 @@ export function DocumentsTableShell({ {/* Fixed Header */}
- - + +
- + {columnVisibility.document_type && ( - + )} {columnVisibility.created_by && ( - + User @@ -406,13 +449,13 @@ export function DocumentsTableShell({ delay: index * 0.02, }, }} - className={`border-b border-border/30 transition-colors ${ + className={`border-b border-border/40 transition-colors ${ isSelected ? "bg-primary/5 hover:bg-primary/8" : "hover:bg-muted/30" }`} > - +
- - { - // Ctrl (Win/Linux) or Cmd (Mac) + Click opens metadata - if (e.ctrlKey || e.metaKey) { - e.preventDefault(); - e.stopPropagation(); - setMetadataDoc(doc); - } - }} - onKeyDown={(e) => { - // Ctrl/Cmd + Enter opens metadata - if ((e.ctrlKey || e.metaKey) && e.key === "Enter") { - e.preventDefault(); - setMetadataDoc(doc); - } - }} - > - - - } - /> + + {columnVisibility.document_type && ( - + )} {columnVisibility.created_by && ( - + {doc.created_by_name || "—"} )} @@ -482,7 +525,7 @@ export function DocumentsTableShell({ {/* Mobile Card View - Notion Style */} -
+
{sorted.map((doc, index) => { const isSelected = selectedIds.has(doc.id); return ( @@ -502,33 +545,33 @@ export function DocumentsTableShell({ className="border-foreground data-[state=checked]:bg-primary data-[state=checked]:border-primary" />
- { - // Ctrl (Win/Linux) or Cmd (Mac) + Click opens metadata - if (e.ctrlKey || e.metaKey) { - e.preventDefault(); - e.stopPropagation(); - setMetadataDoc(doc); - } - }} - onKeyDown={(e) => { - // Ctrl/Cmd + Enter opens metadata - if ((e.ctrlKey || e.metaKey) && e.key === "Enter") { - e.preventDefault(); - setMetadataDoc(doc); - } - }} - > - {doc.title} - - } - /> +
{columnVisibility.created_by && doc.created_by_name && ( @@ -567,6 +610,24 @@ export function DocumentsTableShell({ if (!open) setMetadataDoc(null); }} /> + + {/* Document Content Viewer - lazy loads content on-demand */} + !open && handleCloseViewer()}> + + + {viewingDoc?.title} + +
+ {viewingLoading ? ( +
+ +
+ ) : ( + + )} +
+ + ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/types.ts b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/types.ts index b52054dcd..5485be0ef 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/types.ts +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/types.ts @@ -4,8 +4,9 @@ export type Document = { id: number; title: string; document_type: DocumentType; - document_metadata: any; - content: string; + // Optional: Only needed when viewing document details (lazy loaded) + document_metadata?: any; + content?: string; created_at: string; search_space_id: number; created_by_id?: string | null; diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 0e08f7500..31c95e5e6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -8,8 +8,8 @@ import { useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms"; -import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import type { DocumentTypeEnum } from "@/contracts/types/document.types"; +import { useDocuments } from "@/hooks/use-documents"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import { DocumentsFilters } from "./components/DocumentsFilters"; @@ -43,21 +43,20 @@ export default function DocumentsTable() { const [sortKey, setSortKey] = useState("created_at"); const [sortDesc, setSortDesc] = useState(true); const [selectedIds, setSelectedIds] = useState>(new Set()); - const { data: rawTypeCounts } = useAtomValue(documentTypeCountsAtom); const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom); - // Build query parameters for fetching documents - const queryParams = useMemo( - () => ({ - search_space_id: searchSpaceId, - page: pageIndex, - page_size: PAGE_SIZE, - ...(activeTypes.length > 0 && { document_types: activeTypes }), - }), - [searchSpaceId, pageIndex, activeTypes] - ); + // REAL-TIME: Use Electric SQL hook for live document updates (when not searching) + const { + documents: realtimeDocuments, + typeCounts: realtimeTypeCounts, + loading: realtimeLoading, + error: realtimeError, + } = useDocuments(searchSpaceId, activeTypes); - // Build search query parameters + // Check if we're in search mode + const isSearchMode = !!debouncedSearch.trim(); + + // Build search query parameters (only used when searching) const searchQueryParams = useMemo( () => ({ search_space_id: searchSpaceId, @@ -69,20 +68,7 @@ export default function DocumentsTable() { [searchSpaceId, pageIndex, activeTypes, debouncedSearch] ); - // Use query for fetching documents - const { - data: documentsResponse, - isLoading: isDocumentsLoading, - refetch: refetchDocuments, - error: documentsError, - } = useQuery({ - queryKey: cacheKeys.documents.globalQueryParams(queryParams), - queryFn: () => documentsApiService.getDocuments({ queryParams }), - staleTime: 3 * 60 * 1000, // 3 minutes - enabled: !!searchSpaceId && !debouncedSearch.trim(), - }); - - // Use query for searching documents + // API search query (only enabled when searching - Electric doesn't do full-text search) const { data: searchResponse, isLoading: isSearchLoading, @@ -91,73 +77,59 @@ export default function DocumentsTable() { } = useQuery({ queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams), queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }), - staleTime: 3 * 60 * 1000, // 3 minutes - enabled: !!searchSpaceId && !!debouncedSearch.trim(), + staleTime: 30 * 1000, // 30 seconds for search (shorter since it's on-demand) + enabled: !!searchSpaceId && isSearchMode, }); - // Determine if we should show SurfSense docs (when no type filter or SURFSENSE_DOCS is selected) - const showSurfsenseDocs = - activeTypes.length === 0 || activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum); + // Client-side sorting for real-time documents + const sortedRealtimeDocuments = useMemo(() => { + const docs = [...realtimeDocuments]; + docs.sort((a, b) => { + const av = a[sortKey] ?? ""; + const bv = b[sortKey] ?? ""; + let cmp: number; + if (sortKey === "created_at") { + cmp = new Date(av as string).getTime() - new Date(bv as string).getTime(); + } else { + cmp = String(av).localeCompare(String(bv)); + } + return sortDesc ? -cmp : cmp; + }); + return docs; + }, [realtimeDocuments, sortKey, sortDesc]); - // Use query for fetching SurfSense docs - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { data: surfsenseDocsResponse } = useQuery({ - queryKey: ["surfsense-docs", debouncedSearch, pageIndex, PAGE_SIZE], - queryFn: () => - documentsApiService.getSurfsenseDocs({ - queryParams: { - page: pageIndex, - page_size: PAGE_SIZE, - title: debouncedSearch.trim() || undefined, - }, - }), - staleTime: 3 * 60 * 1000, // 3 minutes - enabled: showSurfsenseDocs, - }); + // Client-side pagination for real-time documents + const paginatedRealtimeDocuments = useMemo(() => { + const start = pageIndex * PAGE_SIZE; + const end = start + PAGE_SIZE; + return sortedRealtimeDocuments.slice(start, end); + }, [sortedRealtimeDocuments, pageIndex]); - // Transform SurfSense docs to match the Document type - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const surfsenseDocsAsDocuments = useMemo(() => { - if (!surfsenseDocsResponse?.items) return []; - return surfsenseDocsResponse.items.map((doc) => ({ - id: doc.id, - title: doc.title, - document_type: "SURFSENSE_DOCS", - document_metadata: { source: doc.source }, - content: doc.content, - created_at: new Date().toISOString(), - search_space_id: -1, // Special value for global docs - })); - }, [surfsenseDocsResponse]); + // Determine what to display based on search mode + const displayDocs = isSearchMode + ? (searchResponse?.items || []).map((item) => ({ + id: item.id, + search_space_id: item.search_space_id, + document_type: item.document_type, + title: item.title, + created_by_id: item.created_by_id ?? null, + created_by_name: item.created_by_name ?? null, + created_at: item.created_at, + })) + : paginatedRealtimeDocuments; - // Merge type counts with SURFSENSE_DOCS count - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const typeCounts = useMemo(() => { - const counts = { ...(rawTypeCounts || {}) }; - if (surfsenseDocsResponse?.total) { - counts.SURFSENSE_DOCS = surfsenseDocsResponse.total; - } - return counts; - }, [rawTypeCounts, surfsenseDocsResponse?.total]); + const displayTotal = isSearchMode + ? searchResponse?.total || 0 + : sortedRealtimeDocuments.length; - // Extract documents and total based on search state - const documents = debouncedSearch.trim() - ? searchResponse?.items || [] - : documentsResponse?.items || []; - const total = debouncedSearch.trim() ? searchResponse?.total || 0 : documentsResponse?.total || 0; + const loading = isSearchMode ? isSearchLoading : realtimeLoading; + const error = isSearchMode ? searchError : realtimeError; - const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading; - const error = debouncedSearch.trim() ? searchError : documentsError; - - // Display results directly - const displayDocs = documents; - const displayTotal = total; const pageEnd = Math.min((pageIndex + 1) * PAGE_SIZE, displayTotal); const onToggleType = (type: DocumentTypeEnum, checked: boolean) => { setActiveTypes((prev) => { if (checked) { - // Only add if not already in the array return prev.includes(type) ? prev : [...prev, type]; } else { return prev.filter((t) => t !== type); @@ -176,16 +148,15 @@ export default function DocumentsTable() { if (isRefreshing) return; setIsRefreshing(true); try { - if (debouncedSearch.trim()) { + if (isSearchMode) { await refetchSearch(); - } else { - await refetchDocuments(); } + // Real-time view doesn't need manual refresh - Electric handles it toast.success(t("refresh_success") || "Documents refreshed"); } finally { setIsRefreshing(false); } - }, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]); + }, [isSearchMode, refetchSearch, t, isRefreshing]); const onBulkDelete = async () => { if (selectedIds.size === 0) { @@ -208,7 +179,13 @@ export default function DocumentsTable() { if (okCount === selectedIds.size) toast.success(t("delete_success_count", { count: okCount })); else toast.error(t("delete_partial_failed")); - // Note: No need to call refreshCurrentView() - the mutation already updates the cache + + // If in search mode, refetch search results to reflect deletion + if (isSearchMode) { + await refetchSearch(); + } + // Real-time mode: Electric will sync the deletion automatically + setSelectedIds(new Set()); } catch (e) { console.error(e); @@ -227,6 +204,12 @@ export default function DocumentsTable() { }); }, []); + // Reset page when search changes (type filter already resets via onToggleType) + // biome-ignore lint/correctness/useExhaustiveDependencies: Intentionally reset page on search change + useEffect(() => { + setPageIndex(0); + }, [debouncedSearch]); + useEffect(() => { const mq = window.matchMedia("(max-width: 768px)"); const apply = (isSmall: boolean) => { @@ -245,9 +228,9 @@ export default function DocumentsTable() { transition={{ duration: 0.3 }} className="w-full max-w-7xl mx-auto px-6 pt-17 pb-6 space-y-6 min-h-[calc(100vh-64px)]" > - {/* Filters */} + {/* Filters - use real-time type counts */} { }, onSuccess: () => { - toast.success("Files uploaded for processing"); + // Note: Toast notification is handled by the caller (DocumentUploadTab) to use i18n // Invalidate logs summary to show new processing tasks immediately on documents page queryClient.invalidateQueries({ queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined), diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index abb32dde1..ec8399198 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -19,7 +19,7 @@ import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsContent } from "@/components/ui/tabs"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { useConnectorsElectric } from "@/hooks/use-connectors-electric"; -import { useDocumentsElectric } from "@/hooks/use-documents-electric"; +import { useDocuments } from "@/hooks/use-documents"; import { useInbox } from "@/hooks/use-inbox"; import { cn } from "@/lib/utils"; import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header"; @@ -63,7 +63,9 @@ export const ConnectorIndicator: FC<{ hideTrigger?: boolean }> = ({ hideTrigger const llmConfigLoading = preferencesLoading || globalConfigsLoading; // Fetch document type counts using Electric SQL + PGlite for real-time updates - const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId); + const { typeCounts: documentTypeCounts, loading: documentTypesLoading } = useDocuments( + searchSpaceId ? Number(searchSpaceId) : null + ); // Fetch notifications to detect indexing failures const { inboxItems = [] } = useInbox( diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index c181119d3..b7a2d2cf8 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -42,6 +42,8 @@ export const document = z.object({ created_at: z.string(), updated_at: z.string().nullable(), search_space_id: z.number(), + created_by_id: z.string().nullable().optional(), + created_by_name: z.string().nullable().optional(), }); export const extensionDocumentContent = z.object({ diff --git a/surfsense_web/hooks/use-documents-electric.ts b/surfsense_web/hooks/use-documents-electric.ts deleted file mode 100644 index 43809499e..000000000 --- a/surfsense_web/hooks/use-documents-electric.ts +++ /dev/null @@ -1,185 +0,0 @@ -"use client"; - -import { useEffect, useMemo, useRef, useState } from "react"; -import type { SyncHandle } from "@/lib/electric/client"; -import { useElectricClient } from "@/lib/electric/context"; - -interface Document { - id: number; - search_space_id: number; - document_type: string; - created_at: string; -} - -/** - * Hook for managing documents with Electric SQL real-time sync - * - * Uses the Electric client from context (provided by ElectricProvider) - * instead of initializing its own - prevents race conditions and memory leaks - */ -export function useDocumentsElectric(searchSpaceId: number | string | null) { - // Get Electric client from context - ElectricProvider handles initialization - const electricClient = useElectricClient(); - - const [documents, setDocuments] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - const syncHandleRef = useRef(null); - const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null); - const syncKeyRef = useRef(null); - - // Calculate document type counts from synced documents - const documentTypeCounts = useMemo(() => { - if (!documents.length) return {}; - - const counts: Record = {}; - for (const doc of documents) { - counts[doc.document_type] = (counts[doc.document_type] || 0) + 1; - } - return counts; - }, [documents]); - - // Start syncing when Electric client is available - useEffect(() => { - // Wait for both searchSpaceId and Electric client to be available - if (!searchSpaceId || !electricClient) { - setLoading(!electricClient); // Still loading if waiting for Electric - if (!searchSpaceId) { - setDocuments([]); - } - return; - } - - // Create a unique key for this sync to prevent duplicate subscriptions - const syncKey = `documents_${searchSpaceId}`; - if (syncKeyRef.current === syncKey) { - // Already syncing for this search space - return; - } - - let mounted = true; - syncKeyRef.current = syncKey; - - async function startSync() { - try { - console.log("[useDocumentsElectric] Starting sync for search space:", searchSpaceId); - - const handle = await electricClient.syncShape({ - table: "documents", - where: `search_space_id = ${searchSpaceId}`, - columns: ["id", "document_type", "search_space_id", "created_at"], - primaryKey: ["id"], - }); - - console.log("[useDocumentsElectric] Sync started:", { - isUpToDate: handle.isUpToDate, - }); - - // Wait for initial sync with timeout - if (!handle.isUpToDate && handle.initialSyncPromise) { - try { - await Promise.race([ - handle.initialSyncPromise, - new Promise((resolve) => setTimeout(resolve, 2000)), - ]); - } catch (syncErr) { - console.error("[useDocumentsElectric] Initial sync failed:", syncErr); - } - } - - if (!mounted) { - handle.unsubscribe(); - return; - } - - syncHandleRef.current = handle; - setLoading(false); - setError(null); - - // Fetch initial documents - await fetchDocuments(); - - // Set up live query for real-time updates - await setupLiveQuery(); - } catch (err) { - if (!mounted) return; - console.error("[useDocumentsElectric] Failed to start sync:", err); - setError(err instanceof Error ? err : new Error("Failed to sync documents")); - setLoading(false); - } - } - - async function fetchDocuments() { - try { - const result = await electricClient.db.query( - `SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`, - [searchSpaceId] - ); - if (mounted) { - setDocuments(result.rows || []); - } - } catch (err) { - console.error("[useDocumentsElectric] Failed to fetch:", err); - } - } - - async function setupLiveQuery() { - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const db = electricClient.db as any; - - if (db.live?.query && typeof db.live.query === "function") { - const liveQuery = await db.live.query( - `SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`, - [searchSpaceId] - ); - - if (!mounted) { - liveQuery.unsubscribe?.(); - return; - } - - // Set initial results - if (liveQuery.initialResults?.rows) { - setDocuments(liveQuery.initialResults.rows); - } else if (liveQuery.rows) { - setDocuments(liveQuery.rows); - } - - // Subscribe to changes - if (typeof liveQuery.subscribe === "function") { - liveQuery.subscribe((result: { rows: Document[] }) => { - if (mounted && result.rows) { - setDocuments(result.rows); - } - }); - } - - if (typeof liveQuery.unsubscribe === "function") { - liveQueryRef.current = liveQuery; - } - } - } catch (liveErr) { - console.error("[useDocumentsElectric] Failed to set up live query:", liveErr); - } - } - - startSync(); - - return () => { - mounted = false; - syncKeyRef.current = null; - - if (syncHandleRef.current) { - syncHandleRef.current.unsubscribe(); - syncHandleRef.current = null; - } - if (liveQueryRef.current) { - liveQueryRef.current.unsubscribe(); - liveQueryRef.current = null; - } - }; - }, [searchSpaceId, electricClient]); - - return { documentTypeCounts, loading, error }; -} diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts new file mode 100644 index 000000000..4d1f8f67c --- /dev/null +++ b/surfsense_web/hooks/use-documents.ts @@ -0,0 +1,427 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { DocumentTypeEnum } from "@/contracts/types/document.types"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import type { SyncHandle } from "@/lib/electric/client"; +import { useElectricClient } from "@/lib/electric/context"; + +const PAGE_SIZE = 100; + +// Stable empty array to prevent infinite re-renders when no typeFilter is provided +const EMPTY_TYPE_FILTER: DocumentTypeEnum[] = []; + +// Document from Electric sync (lightweight table columns - NO content/metadata) +interface DocumentElectric { + id: number; + search_space_id: number; + document_type: string; + title: string; + created_by_id: string | null; + created_at: string; +} + +// Document for display (with resolved user name) +export interface DocumentDisplay { + id: number; + search_space_id: number; + document_type: string; + title: string; + created_by_id: string | null; + created_by_name: string | null; + created_at: string; +} + +/** + * Deduplicate by ID and sort by created_at descending (newest first) + */ +function deduplicateAndSort(items: T[]): T[] { + const seen = new Map(); + for (const item of items) { + // Keep the most recent version if duplicate + const existing = seen.get(item.id); + if (!existing || new Date(item.created_at) > new Date(existing.created_at)) { + seen.set(item.id, item); + } + } + return Array.from(seen.values()).sort( + (a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime() + ); +} + +/** + * Check if a document has valid/complete data + */ +function isValidDocument(doc: DocumentElectric): boolean { + return doc.id != null && doc.title != null && doc.title !== ""; +} + +/** + * Real-time documents hook with Electric SQL + * + * Architecture (100% Reliable): + * 1. API is the PRIMARY source of truth - always loads first + * 2. Electric provides REAL-TIME updates for additions and deletions + * 3. Use syncHandle.isUpToDate to determine if deletions can be trusted + * 4. Handles bulk deletions correctly by checking sync state + * + * @param searchSpaceId - The search space ID to filter documents + * @param typeFilter - Optional document types to filter by + */ +export function useDocuments( + searchSpaceId: number | null, + typeFilter: DocumentTypeEnum[] = EMPTY_TYPE_FILTER +) { + const electricClient = useElectricClient(); + + const [documents, setDocuments] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + // Track if initial API load is complete (source of truth) + const apiLoadedRef = useRef(false); + + // User cache: userId → displayName + const userCacheRef = useRef>(new Map()); + + // Electric sync refs + const syncHandleRef = useRef(null); + const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null); + + // Real-time type counts + const typeCounts = useMemo(() => { + const counts: Record = {}; + for (const doc of documents) { + counts[doc.document_type] = (counts[doc.document_type] || 0) + 1; + } + return counts; + }, [documents]); + + // Populate user cache from API response + const populateUserCache = useCallback( + (items: Array<{ created_by_id?: string | null; created_by_name?: string | null }>) => { + for (const item of items) { + if (item.created_by_id && item.created_by_name) { + userCacheRef.current.set(item.created_by_id, item.created_by_name); + } + } + }, + [] + ); + + // Convert API item to display doc + const apiToDisplayDoc = useCallback( + (item: { + id: number; + search_space_id: number; + document_type: string; + title: string; + created_by_id?: string | null; + created_by_name?: string | null; + created_at: string; + }): DocumentDisplay => ({ + id: item.id, + search_space_id: item.search_space_id, + document_type: item.document_type, + title: item.title, + created_by_id: item.created_by_id ?? null, + created_by_name: item.created_by_name ?? null, + created_at: item.created_at, + }), + [] + ); + + // Convert Electric doc to display doc + const electricToDisplayDoc = useCallback( + (doc: DocumentElectric): DocumentDisplay => ({ + ...doc, + created_by_name: doc.created_by_id + ? userCacheRef.current.get(doc.created_by_id) ?? null + : null, + }), + [] + ); + + // EFFECT 1: Load from API (PRIMARY source of truth) + useEffect(() => { + if (!searchSpaceId) { + setLoading(false); + return; + } + + // Capture validated value for async closure + const spaceId = searchSpaceId; + const currentTypeFilter = typeFilter; + + let mounted = true; + apiLoadedRef.current = false; + + async function loadFromApi() { + try { + setLoading(true); + console.log("[useDocuments] Loading from API (source of truth):", spaceId); + + const response = await documentsApiService.getDocuments({ + queryParams: { + search_space_id: spaceId, + page: 0, + page_size: PAGE_SIZE, + ...(currentTypeFilter.length > 0 && { document_types: currentTypeFilter }), + }, + }); + + if (!mounted) return; + + populateUserCache(response.items); + const docs = response.items.map(apiToDisplayDoc); + setDocuments(docs); + apiLoadedRef.current = true; + setError(null); + console.log("[useDocuments] API loaded", docs.length, "documents"); + } catch (err) { + if (!mounted) return; + console.error("[useDocuments] API load failed:", err); + setError(err instanceof Error ? err : new Error("Failed to load documents")); + } finally { + if (mounted) setLoading(false); + } + } + + loadFromApi(); + + return () => { + mounted = false; + }; + }, [searchSpaceId, typeFilter, populateUserCache, apiToDisplayDoc]); + + // EFFECT 2: Start Electric sync + live query for real-time updates + useEffect(() => { + if (!searchSpaceId || !electricClient) return; + + // Capture validated values for async closure + const spaceId = searchSpaceId; + const client = electricClient; + const currentTypeFilter = typeFilter; + + let mounted = true; + + async function setupElectricRealtime() { + // Cleanup previous subscriptions + if (syncHandleRef.current) { + syncHandleRef.current.unsubscribe(); + syncHandleRef.current = null; + } + if (liveQueryRef.current) { + liveQueryRef.current.unsubscribe?.(); + liveQueryRef.current = null; + } + + try { + console.log("[useDocuments] Starting Electric sync for real-time updates"); + + // Start Electric sync + const handle = await client.syncShape({ + table: "documents", + where: `search_space_id = ${spaceId}`, + columns: ["id", "document_type", "search_space_id", "title", "created_by_id", "created_at"], + primaryKey: ["id"], + }); + + if (!mounted) { + handle.unsubscribe(); + return; + } + + syncHandleRef.current = handle; + console.log("[useDocuments] Sync started, isUpToDate:", handle.isUpToDate); + + // Wait for initial sync (with timeout) + if (!handle.isUpToDate && handle.initialSyncPromise) { + await Promise.race([ + handle.initialSyncPromise, + new Promise((resolve) => setTimeout(resolve, 5000)), + ]); + console.log("[useDocuments] Initial sync complete, isUpToDate:", handle.isUpToDate); + } + + if (!mounted) return; + + // Set up live query + const db = client.db as { + live?: { + query: (sql: string, params?: (number | string)[]) => Promise<{ + subscribe: (cb: (result: { rows: T[] }) => void) => void; + unsubscribe?: () => void; + }>; + }; + }; + + if (!db.live?.query) { + console.warn("[useDocuments] Live queries not available"); + return; + } + + let query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at + FROM documents + WHERE search_space_id = $1`; + + const params: (number | string)[] = [spaceId]; + + if (currentTypeFilter.length > 0) { + const placeholders = currentTypeFilter.map((_, i) => `$${i + 2}`).join(", "); + query += ` AND document_type IN (${placeholders})`; + params.push(...currentTypeFilter); + } + + query += ` ORDER BY created_at DESC`; + + const liveQuery = await db.live.query(query, params); + + if (!mounted) { + liveQuery.unsubscribe?.(); + return; + } + + console.log("[useDocuments] Live query subscribed"); + + liveQuery.subscribe((result: { rows: DocumentElectric[] }) => { + if (!mounted || !result.rows) return; + + // DEBUG: Log first few raw documents to see what's coming from Electric + console.log("[useDocuments] Raw data sample:", result.rows.slice(0, 3)); + + + const validItems = result.rows.filter(isValidDocument); + const isFullySynced = syncHandleRef.current?.isUpToDate ?? false; + + console.log( + `[useDocuments] Live update: ${result.rows.length} raw, ${validItems.length} valid, synced: ${isFullySynced}` + ); + + // Fetch user names for new users (non-blocking) + const unknownUserIds = validItems + .filter((doc): doc is DocumentElectric & { created_by_id: string } => + doc.created_by_id !== null && !userCacheRef.current.has(doc.created_by_id) + ) + .map((doc) => doc.created_by_id); + + if (unknownUserIds.length > 0) { + documentsApiService + .getDocuments({ + queryParams: { search_space_id: spaceId, page: 0, page_size: 20 }, + }) + .then((response) => { + populateUserCache(response.items); + if (mounted) { + setDocuments((prev) => + prev.map((doc) => ({ + ...doc, + created_by_name: doc.created_by_id + ? userCacheRef.current.get(doc.created_by_id) ?? null + : null, + })) + ); + } + }) + .catch(() => {}); + } + + // Smart update logic based on sync state + setDocuments((prev) => { + // Don't process if API hasn't loaded yet + if (!apiLoadedRef.current) { + console.log("[useDocuments] Waiting for API load, skipping live update"); + return prev; + } + + // Case 1: Live query is empty + if (validItems.length === 0) { + if (isFullySynced && prev.length > 0) { + // Electric is fully synced and says 0 items - trust it (all deleted) + console.log("[useDocuments] All documents deleted (Electric synced)"); + return []; + } + // Partial sync or error - keep existing + console.log("[useDocuments] Empty live result, keeping existing"); + return prev; + } + + // Case 2: Electric is fully synced - TRUST IT COMPLETELY (handles bulk deletes) + if (isFullySynced) { + const liveDocs = deduplicateAndSort(validItems.map(electricToDisplayDoc)); + console.log(`[useDocuments] Synced update: ${liveDocs.length} docs (was ${prev.length})`); + return liveDocs; + } + + // Case 3: Partial sync - only ADD new items, don't remove any + const existingIds = new Set(prev.map((d) => d.id)); + const liveIds = new Set(validItems.map((d) => d.id)); + + // Find new items (in live but not in prev) + const newItems = validItems + .filter((item) => !existingIds.has(item.id)) + .map(electricToDisplayDoc); + + // Find updated items (in both, update with latest data) + const updatedPrev = prev.map((doc) => { + if (liveIds.has(doc.id)) { + const liveItem = validItems.find((v) => v.id === doc.id); + if (liveItem) { + return electricToDisplayDoc(liveItem); + } + } + return doc; + }); + + if (newItems.length > 0) { + console.log(`[useDocuments] Adding ${newItems.length} new items (partial sync)`); + return deduplicateAndSort([...newItems, ...updatedPrev]); + } + + return updatedPrev; + }); + }); + + liveQueryRef.current = liveQuery; + } catch (err) { + console.error("[useDocuments] Electric setup failed:", err); + // Don't set error - API data is already loaded + } + } + + setupElectricRealtime(); + + return () => { + mounted = false; + if (syncHandleRef.current) { + syncHandleRef.current.unsubscribe(); + syncHandleRef.current = null; + } + if (liveQueryRef.current) { + liveQueryRef.current.unsubscribe?.(); + liveQueryRef.current = null; + } + }; + }, [searchSpaceId, electricClient, typeFilter, electricToDisplayDoc, populateUserCache]); + + // Track previous searchSpaceId to detect actual changes + const prevSearchSpaceIdRef = useRef(null); + + // Reset on search space change (not on initial mount) + useEffect(() => { + if (prevSearchSpaceIdRef.current !== null && prevSearchSpaceIdRef.current !== searchSpaceId) { + setDocuments([]); + apiLoadedRef.current = false; + userCacheRef.current.clear(); + } + prevSearchSpaceIdRef.current = searchSpaceId; + }, [searchSpaceId]); + + return { + documents, + typeCounts, + total: documents.length, + loading, + error, + }; +} diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index 7ef8f7bbf..788a9444d 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -70,7 +70,9 @@ const pendingSyncs = new Map>(); // v5: fixed duplicate key errors (root cause: unstable cutoff dates in use-inbox.ts) // - added onMustRefetch handler for server-side refetch scenarios // - fixed getSyncCutoffDate to use stable midnight UTC timestamps -const SYNC_VERSION = 5; +// v6: real-time documents table - added title and created_by_id columns for live document display +// v7: removed use-documents-electric.ts - consolidated to single documents sync to prevent conflicts +const SYNC_VERSION = 7; // Database name prefix for identifying SurfSense databases const DB_PREFIX = "surfsense-"; @@ -235,12 +237,14 @@ export async function initElectric(userId: string): Promise { `); // Create the documents table schema in PGlite - // Only sync minimal fields needed for type counts: id, document_type, search_space_id + // Sync columns needed for real-time table display (lightweight - no content/metadata) await db.exec(` CREATE TABLE IF NOT EXISTS documents ( id INTEGER PRIMARY KEY, search_space_id INTEGER NOT NULL, document_type TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + created_by_id TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() );