mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-10 16:22:38 +02:00
refactor: add sorting functionality to document retrieval and enhance DocumentsSidebar for improved search and pagination handling
This commit is contained in:
Anish Sarkar
parent
0feb17cb75
commit
dc6c18b3f6
5 changed files with 311 additions and 281 deletions
|
|
@ -320,6 +320,8 @@ async def read_documents(
|
|||
page_size: int = 50,
|
||||
search_space_id: int | None = None,
|
||||
document_types: str | None = None,
|
||||
sort_by: str = "created_at",
|
||||
sort_order: str = "desc",
|
||||
session: AsyncSession = Depends(get_async_session),
|
||||
user: User = Depends(current_active_user),
|
||||
):
|
||||
|
|
@ -392,6 +394,19 @@ async def read_documents(
|
|||
total_result = await session.execute(count_query)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
# Apply sorting
|
||||
from sqlalchemy import asc as sa_asc, desc as sa_desc
|
||||
|
||||
sort_column_map = {
|
||||
"created_at": Document.created_at,
|
||||
"title": Document.title,
|
||||
"document_type": Document.document_type,
|
||||
}
|
||||
sort_col = sort_column_map.get(sort_by, Document.created_at)
|
||||
query = query.order_by(
|
||||
sa_desc(sort_col) if sort_order == "desc" else sa_asc(sort_col)
|
||||
)
|
||||
|
||||
# Calculate offset
|
||||
offset = 0
|
||||
if skip is not None:
|
||||
|
|
|
|||
|
|
@ -301,6 +301,7 @@ export function DocumentsTableShell({
|
|||
hasMore = false,
|
||||
loadingMore = false,
|
||||
onLoadMore,
|
||||
isSearchMode = false,
|
||||
}: {
|
||||
documents: Document[];
|
||||
loading: boolean;
|
||||
|
|
@ -316,6 +317,7 @@ export function DocumentsTableShell({
|
|||
hasMore?: boolean;
|
||||
loadingMore?: boolean;
|
||||
onLoadMore?: () => void;
|
||||
isSearchMode?: boolean;
|
||||
}) {
|
||||
const t = useTranslations("documents");
|
||||
const { openDialog } = useDocumentUploadDialog();
|
||||
|
|
@ -473,9 +475,9 @@ export function DocumentsTableShell({
|
|||
<div className="flex-1 overflow-auto">
|
||||
<Table className="table-fixed w-full">
|
||||
<TableBody>
|
||||
{[65, 80, 45, 72, 55, 88, 40, 60, 50, 75].map((widthPercent, index) => (
|
||||
{[65, 80, 45, 72, 55, 88, 40, 60, 50, 75].map((widthPercent) => (
|
||||
<TableRow
|
||||
key={`skeleton-${index}`}
|
||||
key={`skeleton-${widthPercent}`}
|
||||
className="border-b border-border/50 hover:bg-transparent"
|
||||
>
|
||||
<TableCell className="w-10 pl-3 pr-0 py-1.5 text-center">
|
||||
|
|
@ -500,8 +502,8 @@ export function DocumentsTableShell({
|
|||
</div>
|
||||
{/* Mobile Skeleton */}
|
||||
<div className="md:hidden divide-y divide-border/50 flex-1 overflow-auto">
|
||||
{[70, 85, 55, 78, 62, 90].map((widthPercent, index) => (
|
||||
<div key={`skeleton-mobile-${index}`} className="px-3 py-2">
|
||||
{[70, 85, 55, 78, 62, 90].map((widthPercent) => (
|
||||
<div key={`skeleton-mobile-${widthPercent}`} className="px-3 py-2">
|
||||
<div className="flex items-center gap-3">
|
||||
<Skeleton className="h-4 w-4 rounded shrink-0" />
|
||||
<div className="flex-1 min-w-0">
|
||||
|
|
@ -602,9 +604,9 @@ export function DocumentsTableShell({
|
|||
searchSpaceId={searchSpaceId}
|
||||
>
|
||||
<motion.tr
|
||||
initial={index < 20 ? { opacity: 0 } : false}
|
||||
initial={!isSearchMode && index < 20 ? { opacity: 0 } : false}
|
||||
animate={{ opacity: 1 }}
|
||||
transition={index < 20 ? { duration: 0.15, delay: index * 0.02 } : { duration: 0 }}
|
||||
transition={!isSearchMode && index < 20 ? { duration: 0.15, delay: index * 0.02 } : { duration: 0 }}
|
||||
className={`border-b border-border/50 transition-colors ${
|
||||
isSelected
|
||||
? "bg-primary/5 hover:bg-primary/8"
|
||||
|
|
@ -680,9 +682,9 @@ export function DocumentsTableShell({
|
|||
searchSpaceId={searchSpaceId}
|
||||
>
|
||||
<motion.div
|
||||
initial={index < 20 ? { opacity: 0 } : false}
|
||||
initial={!isSearchMode && index < 20 ? { opacity: 0 } : false}
|
||||
animate={{ opacity: 1 }}
|
||||
transition={index < 20 ? { duration: 0.15, delay: index * 0.03 } : { duration: 0 }}
|
||||
transition={!isSearchMode && index < 20 ? { duration: 0.15, delay: index * 0.03 } : { duration: 0 }}
|
||||
className={`px-3 py-2 transition-colors ${
|
||||
isSelected ? "bg-primary/5" : "hover:bg-muted/20"
|
||||
}`}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { useAtomValue } from "jotai";
|
|||
import { ChevronLeft, SquareLibrary } from "lucide-react";
|
||||
import { useParams } from "next/navigation";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -22,8 +22,8 @@ import {
|
|||
import type { ColumnVisibility } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/types";
|
||||
import { SidebarSlideOutPanel } from "./SidebarSlideOutPanel";
|
||||
|
||||
const INITIAL_LOAD_SIZE = 20;
|
||||
const SCROLL_LOAD_SIZE = 5;
|
||||
const SEARCH_INITIAL_SIZE = 20;
|
||||
const SEARCH_SCROLL_SIZE = 5;
|
||||
|
||||
function useDebounced<T>(value: T, delay = 250) {
|
||||
const [debounced, setDebounced] = useState(value);
|
||||
|
|
@ -60,17 +60,21 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
|
||||
const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
|
||||
|
||||
// Paginated realtime documents from the hook (server-side sorted)
|
||||
const {
|
||||
documents: realtimeDocuments,
|
||||
typeCounts: realtimeTypeCounts,
|
||||
loading: realtimeLoading,
|
||||
loadingMore: realtimeLoadingMore,
|
||||
hasMore: realtimeHasMore,
|
||||
loadMore: realtimeLoadMore,
|
||||
error: realtimeError,
|
||||
} = useDocuments(searchSpaceId, activeTypes);
|
||||
} = useDocuments(searchSpaceId, activeTypes, sortKey, sortDesc ? "desc" : "asc");
|
||||
|
||||
const isSearchMode = !!debouncedSearch.trim();
|
||||
|
||||
// --- Infinite scroll state ---
|
||||
const [visibleCount, setVisibleCount] = useState(INITIAL_LOAD_SIZE);
|
||||
// --- Search mode state ---
|
||||
const searchApiLoadedRef = useRef(0);
|
||||
const [searchItems, setSearchItems] = useState<Array<{
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
|
|
@ -82,56 +86,33 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
created_at: string;
|
||||
status: { state: "ready" | "pending" | "processing" | "failed"; reason?: string };
|
||||
}>>([]);
|
||||
const [searchTotal, setSearchTotal] = useState(0);
|
||||
const [searchPageIndex, setSearchPageIndex] = useState(0);
|
||||
const [searchLoadingMore, setSearchLoadingMore] = useState(false);
|
||||
const [searchInitialLoading, setSearchInitialLoading] = useState(false);
|
||||
const [searchHasMore, setSearchHasMore] = useState(false);
|
||||
const searchQueryRef = useRef(debouncedSearch);
|
||||
|
||||
const sortedRealtimeDocuments = useMemo(() => {
|
||||
const docs = [...realtimeDocuments];
|
||||
docs.sort((a, b) => {
|
||||
const av = a[sortKey] ?? "";
|
||||
const bv = b[sortKey] ?? "";
|
||||
let cmp: number;
|
||||
if (sortKey === "created_at") {
|
||||
cmp = new Date(av as string).getTime() - new Date(bv as string).getTime();
|
||||
} else {
|
||||
cmp = String(av).localeCompare(String(bv));
|
||||
}
|
||||
return sortDesc ? -cmp : cmp;
|
||||
});
|
||||
return docs;
|
||||
}, [realtimeDocuments, sortKey, sortDesc]);
|
||||
|
||||
// Reset visible count when sort/filter changes
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: intentional reset
|
||||
useEffect(() => {
|
||||
setVisibleCount(INITIAL_LOAD_SIZE);
|
||||
}, [sortKey, sortDesc, activeTypes]);
|
||||
|
||||
// Initial search fetch when search query changes
|
||||
useEffect(() => {
|
||||
if (!isSearchMode || !searchSpaceId || !open) {
|
||||
setSearchItems([]);
|
||||
setSearchTotal(0);
|
||||
setSearchPageIndex(0);
|
||||
setSearchHasMore(false);
|
||||
searchApiLoadedRef.current = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
searchQueryRef.current = debouncedSearch;
|
||||
setSearchInitialLoading(true);
|
||||
|
||||
const queryParams = {
|
||||
search_space_id: searchSpaceId,
|
||||
page: 0,
|
||||
page_size: INITIAL_LOAD_SIZE,
|
||||
title: debouncedSearch.trim(),
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
};
|
||||
|
||||
documentsApiService
|
||||
.searchDocuments({ queryParams })
|
||||
.searchDocuments({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId,
|
||||
page: 0,
|
||||
page_size: SEARCH_INITIAL_SIZE,
|
||||
title: debouncedSearch.trim(),
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
},
|
||||
})
|
||||
.then((response) => {
|
||||
if (searchQueryRef.current !== debouncedSearch) return;
|
||||
const mapped = response.items.map((item) => ({
|
||||
|
|
@ -150,8 +131,8 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
).status ?? { state: "ready" as const },
|
||||
}));
|
||||
setSearchItems(mapped);
|
||||
setSearchTotal(response.total);
|
||||
setSearchPageIndex(0);
|
||||
setSearchHasMore(response.has_more);
|
||||
searchApiLoadedRef.current = response.items.length;
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error("[DocumentsSidebar] Search failed:", err);
|
||||
|
|
@ -161,22 +142,21 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
});
|
||||
}, [debouncedSearch, searchSpaceId, open, isSearchMode, activeTypes]);
|
||||
|
||||
// Load more search results
|
||||
// Load more search results (uses skip for correct offset with mixed page sizes)
|
||||
const loadMoreSearch = useCallback(async () => {
|
||||
if (searchLoadingMore || !isSearchMode) return;
|
||||
const nextPage = searchPageIndex + 1;
|
||||
if (searchItems.length >= searchTotal) return;
|
||||
if (searchLoadingMore || !isSearchMode || !searchHasMore) return;
|
||||
|
||||
setSearchLoadingMore(true);
|
||||
try {
|
||||
const queryParams = {
|
||||
search_space_id: searchSpaceId,
|
||||
page: nextPage,
|
||||
page_size: SCROLL_LOAD_SIZE,
|
||||
title: debouncedSearch.trim(),
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
};
|
||||
const response = await documentsApiService.searchDocuments({ queryParams });
|
||||
const response = await documentsApiService.searchDocuments({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId,
|
||||
skip: searchApiLoadedRef.current,
|
||||
page_size: SEARCH_SCROLL_SIZE,
|
||||
title: debouncedSearch.trim(),
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
},
|
||||
});
|
||||
if (searchQueryRef.current !== debouncedSearch) return;
|
||||
|
||||
const mapped = response.items.map((item) => ({
|
||||
|
|
@ -195,36 +175,22 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
).status ?? { state: "ready" as const },
|
||||
}));
|
||||
setSearchItems((prev) => [...prev, ...mapped]);
|
||||
setSearchTotal(response.total);
|
||||
setSearchPageIndex(nextPage);
|
||||
setSearchHasMore(response.has_more);
|
||||
searchApiLoadedRef.current += response.items.length;
|
||||
} catch (err) {
|
||||
console.error("[DocumentsSidebar] Load more search failed:", err);
|
||||
} finally {
|
||||
setSearchLoadingMore(false);
|
||||
}
|
||||
}, [searchLoadingMore, isSearchMode, searchPageIndex, searchItems.length, searchTotal, searchSpaceId, debouncedSearch, activeTypes]);
|
||||
}, [searchLoadingMore, isSearchMode, searchHasMore, searchSpaceId, debouncedSearch, activeTypes]);
|
||||
|
||||
// Load more for realtime (client-side, just increase visible count)
|
||||
const loadMoreRealtime = useCallback(() => {
|
||||
setVisibleCount((prev) => Math.min(prev + SCROLL_LOAD_SIZE, sortedRealtimeDocuments.length));
|
||||
}, [sortedRealtimeDocuments.length]);
|
||||
|
||||
const visibleRealtimeDocs = useMemo(
|
||||
() => sortedRealtimeDocuments.slice(0, visibleCount),
|
||||
[sortedRealtimeDocuments, visibleCount]
|
||||
);
|
||||
|
||||
const displayDocs = isSearchMode ? searchItems : visibleRealtimeDocs;
|
||||
// Unified interface — pick between realtime and search mode
|
||||
const displayDocs = isSearchMode ? searchItems : realtimeDocuments;
|
||||
const loading = isSearchMode ? searchInitialLoading : realtimeLoading;
|
||||
const error = isSearchMode ? false : realtimeError;
|
||||
|
||||
const hasMore = isSearchMode
|
||||
? searchItems.length < searchTotal
|
||||
: visibleCount < sortedRealtimeDocuments.length;
|
||||
|
||||
const loadingMore = isSearchMode ? searchLoadingMore : false;
|
||||
|
||||
const onLoadMore = isSearchMode ? loadMoreSearch : loadMoreRealtime;
|
||||
const hasMore = isSearchMode ? searchHasMore : realtimeHasMore;
|
||||
const loadingMore = isSearchMode ? searchLoadingMore : realtimeLoadingMore;
|
||||
const onLoadMore = isSearchMode ? loadMoreSearch : realtimeLoadMore;
|
||||
|
||||
const onToggleType = (type: DocumentTypeEnum, checked: boolean) => {
|
||||
setActiveTypes((prev) => {
|
||||
|
|
@ -244,7 +210,7 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
|
||||
const allDocs = isSearchMode
|
||||
? searchItems.map((item) => ({ id: item.id, status: item.status }))
|
||||
: sortedRealtimeDocuments.map((doc) => ({ id: doc.id, status: doc.status }));
|
||||
: realtimeDocuments.map((doc) => ({ id: doc.id, status: doc.status }));
|
||||
|
||||
const selectedDocs = allDocs.filter((doc) => selectedIds.has(doc.id));
|
||||
const deletableIds = selectedDocs
|
||||
|
|
@ -286,7 +252,6 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
}
|
||||
if (isSearchMode) {
|
||||
setSearchItems((prev) => prev.filter((item) => !deletableIds.includes(item.id)));
|
||||
setSearchTotal((prev) => prev - okCount);
|
||||
}
|
||||
setSelectedIds(new Set());
|
||||
} catch (e) {
|
||||
|
|
@ -302,7 +267,6 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
toast.success(t("delete_success") || "Document deleted");
|
||||
if (isSearchMode) {
|
||||
setSearchItems((prev) => prev.filter((item) => item.id !== id));
|
||||
setSearchTotal((prev) => prev - 1);
|
||||
}
|
||||
return true;
|
||||
} catch (e) {
|
||||
|
|
@ -391,6 +355,7 @@ export function DocumentsSidebar({ open, onOpenChange }: DocumentsSidebarProps)
|
|||
hasMore={hasMore}
|
||||
loadingMore={loadingMore}
|
||||
onLoadMore={onLoadMore}
|
||||
isSearchMode={isSearchMode}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -92,11 +92,16 @@ export const surfsenseDocsDocumentWithChunks = surfsenseDocsDocument.extend({
|
|||
/**
|
||||
* Get documents
|
||||
*/
|
||||
export const documentSortByEnum = z.enum(["created_at", "title", "document_type"]);
|
||||
export const sortOrderEnum = z.enum(["asc", "desc"]);
|
||||
|
||||
export const getDocumentsRequest = z.object({
|
||||
queryParams: paginationQueryParams
|
||||
.extend({
|
||||
search_space_id: z.number().or(z.string()).optional(),
|
||||
document_types: z.array(documentTypeEnum).optional(),
|
||||
sort_by: documentSortByEnum.optional(),
|
||||
sort_order: sortOrderEnum.optional(),
|
||||
})
|
||||
.nullish(),
|
||||
});
|
||||
|
|
@ -311,6 +316,8 @@ export type UpdateDocumentResponse = z.infer<typeof updateDocumentResponse>;
|
|||
export type DeleteDocumentRequest = z.infer<typeof deleteDocumentRequest>;
|
||||
export type DeleteDocumentResponse = z.infer<typeof deleteDocumentResponse>;
|
||||
export type DocumentTypeEnum = z.infer<typeof documentTypeEnum>;
|
||||
export type DocumentSortBy = z.infer<typeof documentSortByEnum>;
|
||||
export type SortOrder = z.infer<typeof sortOrderEnum>;
|
||||
export type SurfsenseDocsChunk = z.infer<typeof surfsenseDocsChunk>;
|
||||
export type SurfsenseDocsDocument = z.infer<typeof surfsenseDocsDocument>;
|
||||
export type SurfsenseDocsDocumentWithChunks = z.infer<typeof surfsenseDocsDocumentWithChunks>;
|
||||
|
|
|
|||
|
|
@ -1,22 +1,20 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import type {
|
||||
DocumentSortBy,
|
||||
DocumentTypeEnum,
|
||||
SortOrder,
|
||||
} from "@/contracts/types/document.types";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import type { SyncHandle } from "@/lib/electric/client";
|
||||
import { useElectricClient } from "@/lib/electric/context";
|
||||
|
||||
// Stable empty array to prevent infinite re-renders when no typeFilter is provided
|
||||
const EMPTY_TYPE_FILTER: DocumentTypeEnum[] = [];
|
||||
|
||||
// Document status type (matches backend DocumentStatus JSONB)
|
||||
export interface DocumentStatusType {
|
||||
state: "ready" | "pending" | "processing" | "failed";
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// Document from Electric sync (lightweight table columns - NO content/metadata)
|
||||
interface DocumentElectric {
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
|
|
@ -27,7 +25,6 @@ interface DocumentElectric {
|
|||
status: DocumentStatusType | null;
|
||||
}
|
||||
|
||||
// Document for display (with resolved user name and email)
|
||||
export interface DocumentDisplay {
|
||||
id: number;
|
||||
search_space_id: number;
|
||||
|
|
@ -40,87 +37,57 @@ export interface DocumentDisplay {
|
|||
status: DocumentStatusType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate by ID and sort by created_at descending (newest first)
|
||||
*/
|
||||
function deduplicateAndSort<T extends { id: number; created_at: string }>(items: T[]): T[] {
|
||||
const seen = new Map<number, T>();
|
||||
for (const item of items) {
|
||||
// Keep the most recent version if duplicate
|
||||
const existing = seen.get(item.id);
|
||||
if (!existing || new Date(item.created_at) > new Date(existing.created_at)) {
|
||||
seen.set(item.id, item);
|
||||
}
|
||||
}
|
||||
return Array.from(seen.values()).sort(
|
||||
(a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()
|
||||
);
|
||||
}
|
||||
const EMPTY_TYPE_FILTER: DocumentTypeEnum[] = [];
|
||||
const INITIAL_PAGE_SIZE = 20;
|
||||
const SCROLL_PAGE_SIZE = 5;
|
||||
|
||||
/**
|
||||
* Check if a document has valid/complete data
|
||||
*/
|
||||
function isValidDocument(doc: DocumentElectric): boolean {
|
||||
return doc.id != null && doc.title != null && doc.title !== "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Real-time documents hook with Electric SQL
|
||||
* Paginated documents hook with Electric SQL real-time updates.
|
||||
*
|
||||
* Architecture (100% Reliable):
|
||||
* 1. API is the PRIMARY source of truth - always loads first
|
||||
* 2. Electric provides REAL-TIME updates for additions and deletions
|
||||
* 3. Use syncHandle.isUpToDate to determine if deletions can be trusted
|
||||
* 4. Handles bulk deletions correctly by checking sync state
|
||||
* Architecture:
|
||||
* 1. API is the PRIMARY data source — fetches pages on demand
|
||||
* 2. Type counts come from a dedicated lightweight API endpoint
|
||||
* 3. Electric provides REAL-TIME updates (new docs, deletions, status changes)
|
||||
* 4. Server-side sorting via sort_by + sort_order params
|
||||
*
|
||||
* Filtering strategy:
|
||||
* - Internal state always stores ALL documents (unfiltered)
|
||||
* - typeFilter is applied client-side when returning documents
|
||||
* - typeCounts always reflect the full dataset so the filter sidebar stays complete
|
||||
* - Changing filters is instant (no API re-fetch or Electric re-sync)
|
||||
*
|
||||
* @param searchSpaceId - The search space ID to filter documents
|
||||
* @param typeFilter - Optional document types to filter by (applied client-side)
|
||||
* @param searchSpaceId - The search space to load documents for
|
||||
* @param typeFilter - Document types to filter by (server-side)
|
||||
* @param sortBy - Column to sort by (server-side)
|
||||
* @param sortOrder - Sort direction (server-side)
|
||||
*/
|
||||
export function useDocuments(
|
||||
searchSpaceId: number | null,
|
||||
typeFilter: DocumentTypeEnum[] = EMPTY_TYPE_FILTER
|
||||
typeFilter: DocumentTypeEnum[] = EMPTY_TYPE_FILTER,
|
||||
sortBy: DocumentSortBy = "created_at",
|
||||
sortOrder: SortOrder = "desc"
|
||||
) {
|
||||
const electricClient = useElectricClient();
|
||||
|
||||
// Internal state: ALL documents (unfiltered)
|
||||
const [allDocuments, setAllDocuments] = useState<DocumentDisplay[]>([]);
|
||||
const [documents, setDocuments] = useState<DocumentDisplay[]>([]);
|
||||
const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
|
||||
const [total, setTotal] = useState(0);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadingMore, setLoadingMore] = useState(false);
|
||||
const [hasMore, setHasMore] = useState(false);
|
||||
const [error, setError] = useState<Error | null>(null);
|
||||
|
||||
// Track if initial API load is complete (source of truth)
|
||||
const apiLoadedRef = useRef(false);
|
||||
|
||||
// User cache: userId → displayName / email
|
||||
const apiLoadedCountRef = useRef(0);
|
||||
const initialLoadDoneRef = useRef(false);
|
||||
// Snapshot of all doc IDs from Electric's first callback after initial load.
|
||||
// Anything appearing in subsequent callbacks NOT in this set is genuinely new.
|
||||
const electricBaselineIdsRef = useRef<Set<number> | null>(null);
|
||||
const knownApiIdsRef = useRef<Set<number>>(new Set());
|
||||
const userCacheRef = useRef<Map<string, string>>(new Map());
|
||||
const emailCacheRef = useRef<Map<string, string>>(new Map());
|
||||
|
||||
// Electric sync refs
|
||||
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||
const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null);
|
||||
|
||||
// Type counts from ALL documents (unfiltered) — keeps filter sidebar complete
|
||||
const typeCounts = useMemo(() => {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const doc of allDocuments) {
|
||||
counts[doc.document_type] = (counts[doc.document_type] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}, [allDocuments]);
|
||||
const typeFilterKey = typeFilter.join(",");
|
||||
|
||||
// Client-side filtered documents for display
|
||||
const documents = useMemo(() => {
|
||||
if (typeFilter.length === 0) return allDocuments;
|
||||
const filterSet = new Set<string>(typeFilter);
|
||||
return allDocuments.filter((doc) => filterSet.has(doc.document_type));
|
||||
}, [allDocuments, typeFilter]);
|
||||
|
||||
// Populate user cache from API response
|
||||
const populateUserCache = useCallback(
|
||||
(
|
||||
items: Array<{
|
||||
|
|
@ -143,7 +110,6 @@ export function useDocuments(
|
|||
[]
|
||||
);
|
||||
|
||||
// Convert API item to display doc
|
||||
const apiToDisplayDoc = useCallback(
|
||||
(item: {
|
||||
id: number;
|
||||
|
|
@ -169,7 +135,6 @@ export function useDocuments(
|
|||
[]
|
||||
);
|
||||
|
||||
// Convert Electric doc to display doc
|
||||
const electricToDisplayDoc = useCallback(
|
||||
(doc: DocumentElectric): DocumentDisplay => ({
|
||||
...doc,
|
||||
|
|
@ -184,66 +149,85 @@ export function useDocuments(
|
|||
[]
|
||||
);
|
||||
|
||||
// STEP 1: Load ALL documents from API (PRIMARY source of truth).
|
||||
// Uses React Query for automatic deduplication, caching, and staleTime so
|
||||
// multiple components mounting useDocuments(sameId) share a single request.
|
||||
const {
|
||||
data: apiResponse,
|
||||
isLoading: apiLoading,
|
||||
error: apiError,
|
||||
} = useQuery({
|
||||
queryKey: ["documents", "all", searchSpaceId],
|
||||
queryFn: () =>
|
||||
documentsApiService.getDocuments({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId!,
|
||||
page: 0,
|
||||
page_size: -1,
|
||||
},
|
||||
}),
|
||||
enabled: !!searchSpaceId,
|
||||
staleTime: 30_000,
|
||||
});
|
||||
|
||||
// Seed local state from API response (runs once per fresh fetch)
|
||||
// EFFECT 1: Fetch first page + type counts when params change
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: typeFilterKey serializes typeFilter
|
||||
useEffect(() => {
|
||||
if (!apiResponse) return;
|
||||
populateUserCache(apiResponse.items);
|
||||
const docs = apiResponse.items.map(apiToDisplayDoc);
|
||||
setAllDocuments(docs);
|
||||
apiLoadedRef.current = true;
|
||||
setError(null);
|
||||
}, [apiResponse, populateUserCache, apiToDisplayDoc]);
|
||||
if (!searchSpaceId) return;
|
||||
|
||||
// Propagate loading / error from React Query
|
||||
useEffect(() => {
|
||||
setLoading(apiLoading);
|
||||
}, [apiLoading]);
|
||||
let cancelled = false;
|
||||
|
||||
useEffect(() => {
|
||||
if (apiError) {
|
||||
setError(apiError instanceof Error ? apiError : new Error("Failed to load documents"));
|
||||
}
|
||||
}, [apiError]);
|
||||
setLoading(true);
|
||||
setDocuments([]);
|
||||
setTotal(0);
|
||||
setHasMore(false);
|
||||
apiLoadedCountRef.current = 0;
|
||||
initialLoadDoneRef.current = false;
|
||||
electricBaselineIdsRef.current = null;
|
||||
knownApiIdsRef.current = new Set();
|
||||
|
||||
// EFFECT 2: Start Electric sync + live query for real-time updates
|
||||
// No type filter — syncs and queries ALL documents; filtering is client-side
|
||||
const fetchInitialData = async () => {
|
||||
try {
|
||||
const [docsResponse, countsResponse] = await Promise.all([
|
||||
documentsApiService.getDocuments({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId,
|
||||
page: 0,
|
||||
page_size: INITIAL_PAGE_SIZE,
|
||||
...(typeFilter.length > 0 && { document_types: typeFilter }),
|
||||
sort_by: sortBy,
|
||||
sort_order: sortOrder,
|
||||
},
|
||||
}),
|
||||
documentsApiService.getDocumentTypeCounts({
|
||||
queryParams: { search_space_id: searchSpaceId },
|
||||
}),
|
||||
]);
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
populateUserCache(docsResponse.items);
|
||||
const docs = docsResponse.items.map(apiToDisplayDoc);
|
||||
setDocuments(docs);
|
||||
setTotal(docsResponse.total);
|
||||
setHasMore(docsResponse.has_more);
|
||||
setTypeCounts(countsResponse);
|
||||
setError(null);
|
||||
apiLoadedCountRef.current = docsResponse.items.length;
|
||||
initialLoadDoneRef.current = true;
|
||||
for (const doc of docs) {
|
||||
knownApiIdsRef.current.add(doc.id);
|
||||
}
|
||||
} catch (err) {
|
||||
if (cancelled) return;
|
||||
console.error("[useDocuments] Initial load failed:", err);
|
||||
setError(
|
||||
err instanceof Error ? err : new Error("Failed to load documents")
|
||||
);
|
||||
} finally {
|
||||
if (!cancelled) setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchInitialData();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [searchSpaceId, typeFilterKey, sortBy, sortOrder, populateUserCache, apiToDisplayDoc]);
|
||||
|
||||
// EFFECT 2: Electric sync + live query for real-time updates
|
||||
useEffect(() => {
|
||||
if (!searchSpaceId || !electricClient) return;
|
||||
|
||||
// Capture validated values for async closure
|
||||
const spaceId = searchSpaceId;
|
||||
const client = electricClient;
|
||||
|
||||
let mounted = true;
|
||||
|
||||
async function setupElectricRealtime() {
|
||||
// Cleanup previous subscriptions
|
||||
if (syncHandleRef.current) {
|
||||
try {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
} catch {
|
||||
// PGlite may already be closed during cleanup
|
||||
/* PGlite may already be closed */
|
||||
}
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
|
|
@ -251,15 +235,12 @@ export function useDocuments(
|
|||
try {
|
||||
liveQueryRef.current.unsubscribe?.();
|
||||
} catch {
|
||||
// PGlite may already be closed during cleanup
|
||||
/* PGlite may already be closed */
|
||||
}
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log("[useDocuments] Starting Electric sync for real-time updates");
|
||||
|
||||
// Start Electric sync (all documents for this search space)
|
||||
const handle = await client.syncShape({
|
||||
table: "documents",
|
||||
where: `search_space_id = ${spaceId}`,
|
||||
|
|
@ -281,20 +262,16 @@ export function useDocuments(
|
|||
}
|
||||
|
||||
syncHandleRef.current = handle;
|
||||
console.log("[useDocuments] Sync started, isUpToDate:", handle.isUpToDate);
|
||||
|
||||
// Wait for initial sync (with timeout)
|
||||
if (!handle.isUpToDate && handle.initialSyncPromise) {
|
||||
await Promise.race([
|
||||
handle.initialSyncPromise,
|
||||
new Promise((resolve) => setTimeout(resolve, 5000)),
|
||||
]);
|
||||
console.log("[useDocuments] Initial sync complete, isUpToDate:", handle.isUpToDate);
|
||||
}
|
||||
|
||||
if (!mounted) return;
|
||||
|
||||
// Set up live query (unfiltered — type filtering is done client-side)
|
||||
const db = client.db as {
|
||||
live?: {
|
||||
query: <T>(
|
||||
|
|
@ -307,62 +284,62 @@ export function useDocuments(
|
|||
};
|
||||
};
|
||||
|
||||
if (!db.live?.query) {
|
||||
console.warn("[useDocuments] Live queries not available");
|
||||
return;
|
||||
}
|
||||
if (!db.live?.query) return;
|
||||
|
||||
const query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at, status
|
||||
FROM documents
|
||||
FROM documents
|
||||
WHERE search_space_id = $1
|
||||
ORDER BY created_at DESC`;
|
||||
|
||||
const liveQuery = await db.live.query<DocumentElectric>(query, [spaceId]);
|
||||
const liveQuery = await db.live.query<DocumentElectric>(query, [
|
||||
spaceId,
|
||||
]);
|
||||
|
||||
if (!mounted) {
|
||||
liveQuery.unsubscribe?.();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[useDocuments] Live query subscribed");
|
||||
|
||||
liveQuery.subscribe((result: { rows: DocumentElectric[] }) => {
|
||||
if (!mounted || !result.rows) return;
|
||||
|
||||
// DEBUG: Log first few raw documents to see what's coming from Electric
|
||||
console.log("[useDocuments] Raw data sample:", result.rows.slice(0, 3));
|
||||
if (!mounted || !result.rows || !initialLoadDoneRef.current) return;
|
||||
|
||||
const validItems = result.rows.filter(isValidDocument);
|
||||
const isFullySynced = syncHandleRef.current?.isUpToDate ?? false;
|
||||
|
||||
console.log(
|
||||
`[useDocuments] Live update: ${result.rows.length} raw, ${validItems.length} valid, synced: ${isFullySynced}`
|
||||
);
|
||||
|
||||
// Fetch user names for new users (non-blocking)
|
||||
const unknownUserIds = validItems
|
||||
.filter(
|
||||
(doc): doc is DocumentElectric & { created_by_id: string } =>
|
||||
doc.created_by_id !== null && !userCacheRef.current.has(doc.created_by_id)
|
||||
(
|
||||
doc
|
||||
): doc is DocumentElectric & { created_by_id: string } =>
|
||||
doc.created_by_id !== null &&
|
||||
!userCacheRef.current.has(doc.created_by_id)
|
||||
)
|
||||
.map((doc) => doc.created_by_id);
|
||||
|
||||
if (unknownUserIds.length > 0) {
|
||||
documentsApiService
|
||||
.getDocuments({
|
||||
queryParams: { search_space_id: spaceId, page: 0, page_size: 20 },
|
||||
queryParams: {
|
||||
search_space_id: spaceId,
|
||||
page: 0,
|
||||
page_size: 20,
|
||||
},
|
||||
})
|
||||
.then((response) => {
|
||||
populateUserCache(response.items);
|
||||
if (mounted) {
|
||||
setAllDocuments((prev) =>
|
||||
setDocuments((prev) =>
|
||||
prev.map((doc) => ({
|
||||
...doc,
|
||||
created_by_name: doc.created_by_id
|
||||
? (userCacheRef.current.get(doc.created_by_id) ?? null)
|
||||
? (userCacheRef.current.get(
|
||||
doc.created_by_id
|
||||
) ?? null)
|
||||
: null,
|
||||
created_by_email: doc.created_by_id
|
||||
? (emailCacheRef.current.get(doc.created_by_id) ?? null)
|
||||
? (emailCacheRef.current.get(
|
||||
doc.created_by_id
|
||||
) ?? null)
|
||||
: null,
|
||||
}))
|
||||
);
|
||||
|
|
@ -371,48 +348,39 @@ export function useDocuments(
|
|||
.catch(() => {});
|
||||
}
|
||||
|
||||
// Smart update logic based on sync state
|
||||
setAllDocuments((prev) => {
|
||||
// Don't process if API hasn't loaded yet
|
||||
if (!apiLoadedRef.current) {
|
||||
console.log("[useDocuments] Waiting for API load, skipping live update");
|
||||
return prev;
|
||||
}
|
||||
|
||||
// Case 1: Live query is empty
|
||||
if (validItems.length === 0) {
|
||||
if (isFullySynced && prev.length > 0) {
|
||||
// Electric is fully synced and says 0 items - trust it (all deleted)
|
||||
console.log("[useDocuments] All documents deleted (Electric synced)");
|
||||
return [];
|
||||
}
|
||||
// Partial sync or error - keep existing
|
||||
console.log("[useDocuments] Empty live result, keeping existing");
|
||||
return prev;
|
||||
}
|
||||
|
||||
// Case 2: Electric is fully synced - TRUST IT COMPLETELY (handles bulk deletes)
|
||||
if (isFullySynced) {
|
||||
const liveDocs = deduplicateAndSort(validItems.map(electricToDisplayDoc));
|
||||
console.log(
|
||||
`[useDocuments] Synced update: ${liveDocs.length} docs (was ${prev.length})`
|
||||
);
|
||||
return liveDocs;
|
||||
}
|
||||
|
||||
// Case 3: Partial sync - only ADD new items, don't remove any
|
||||
const existingIds = new Set(prev.map((d) => d.id));
|
||||
setDocuments((prev) => {
|
||||
const liveIds = new Set(validItems.map((d) => d.id));
|
||||
const prevIds = new Set(prev.map((d) => d.id));
|
||||
|
||||
// Find new items (in live but not in prev)
|
||||
// First callback: snapshot all Electric IDs as the baseline.
|
||||
// Everything in this set existed before the sidebar opened and
|
||||
// should only appear via API pagination, not Electric.
|
||||
if (electricBaselineIdsRef.current === null) {
|
||||
electricBaselineIdsRef.current = new Set(liveIds);
|
||||
}
|
||||
|
||||
// Genuinely new = not in rendered list, not in baseline snapshot.
|
||||
// These are docs created AFTER the sidebar opened.
|
||||
const baseline = electricBaselineIdsRef.current;
|
||||
const newItems = validItems
|
||||
.filter((item) => !existingIds.has(item.id))
|
||||
.filter((item) => {
|
||||
if (prevIds.has(item.id)) return false;
|
||||
if (baseline.has(item.id)) return false;
|
||||
return true;
|
||||
})
|
||||
.map(electricToDisplayDoc);
|
||||
|
||||
// Find updated items (in both, update with latest data)
|
||||
const updatedPrev = prev.map((doc) => {
|
||||
// Track new items in baseline so they aren't re-added
|
||||
for (const item of newItems) {
|
||||
baseline.add(item.id);
|
||||
}
|
||||
|
||||
// Update existing docs (status changes, title edits)
|
||||
let updated = prev.map((doc) => {
|
||||
if (liveIds.has(doc.id)) {
|
||||
const liveItem = validItems.find((v) => v.id === doc.id);
|
||||
const liveItem = validItems.find(
|
||||
(v) => v.id === doc.id
|
||||
);
|
||||
if (liveItem) {
|
||||
return electricToDisplayDoc(liveItem);
|
||||
}
|
||||
|
|
@ -420,19 +388,33 @@ export function useDocuments(
|
|||
return doc;
|
||||
});
|
||||
|
||||
if (newItems.length > 0) {
|
||||
console.log(`[useDocuments] Adding ${newItems.length} new items (partial sync)`);
|
||||
return deduplicateAndSort([...newItems, ...updatedPrev]);
|
||||
// Remove deleted docs (only when fully synced)
|
||||
if (isFullySynced) {
|
||||
updated = updated.filter((doc) => liveIds.has(doc.id));
|
||||
}
|
||||
|
||||
return updatedPrev;
|
||||
if (newItems.length > 0) {
|
||||
return [...newItems, ...updated];
|
||||
}
|
||||
|
||||
return updated;
|
||||
});
|
||||
|
||||
// Update type counts when Electric detects changes
|
||||
if (isFullySynced && validItems.length > 0) {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const item of validItems) {
|
||||
counts[item.document_type] =
|
||||
(counts[item.document_type] || 0) + 1;
|
||||
}
|
||||
setTypeCounts(counts);
|
||||
setTotal(validItems.length);
|
||||
}
|
||||
});
|
||||
|
||||
liveQueryRef.current = liveQuery;
|
||||
} catch (err) {
|
||||
console.error("[useDocuments] Electric setup failed:", err);
|
||||
// Don't set error - API data is already loaded
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -444,7 +426,7 @@ export function useDocuments(
|
|||
try {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
} catch {
|
||||
// PGlite may already be closed during cleanup
|
||||
/* PGlite may already be closed */
|
||||
}
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
|
|
@ -452,32 +434,91 @@ export function useDocuments(
|
|||
try {
|
||||
liveQueryRef.current.unsubscribe?.();
|
||||
} catch {
|
||||
// PGlite may already be closed during cleanup
|
||||
/* PGlite may already be closed */
|
||||
}
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [searchSpaceId, electricClient, electricToDisplayDoc, populateUserCache]);
|
||||
|
||||
// Track previous searchSpaceId to detect actual changes
|
||||
// Reset on search space change
|
||||
const prevSearchSpaceIdRef = useRef<number | null>(null);
|
||||
|
||||
// Reset on search space change (not on initial mount)
|
||||
useEffect(() => {
|
||||
if (prevSearchSpaceIdRef.current !== null && prevSearchSpaceIdRef.current !== searchSpaceId) {
|
||||
setAllDocuments([]);
|
||||
apiLoadedRef.current = false;
|
||||
if (
|
||||
prevSearchSpaceIdRef.current !== null &&
|
||||
prevSearchSpaceIdRef.current !== searchSpaceId
|
||||
) {
|
||||
setDocuments([]);
|
||||
setTypeCounts({});
|
||||
setTotal(0);
|
||||
setHasMore(false);
|
||||
apiLoadedCountRef.current = 0;
|
||||
initialLoadDoneRef.current = false;
|
||||
electricBaselineIdsRef.current = null;
|
||||
knownApiIdsRef.current = new Set();
|
||||
userCacheRef.current.clear();
|
||||
emailCacheRef.current.clear();
|
||||
}
|
||||
prevSearchSpaceIdRef.current = searchSpaceId;
|
||||
}, [searchSpaceId]);
|
||||
|
||||
// Load more pages via API
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: typeFilterKey serializes typeFilter
|
||||
const loadMore = useCallback(async () => {
|
||||
if (loadingMore || !hasMore || !searchSpaceId) return;
|
||||
|
||||
setLoadingMore(true);
|
||||
try {
|
||||
const response = await documentsApiService.getDocuments({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId,
|
||||
skip: apiLoadedCountRef.current,
|
||||
page_size: SCROLL_PAGE_SIZE,
|
||||
...(typeFilter.length > 0 && { document_types: typeFilter }),
|
||||
sort_by: sortBy,
|
||||
sort_order: sortOrder,
|
||||
},
|
||||
});
|
||||
|
||||
populateUserCache(response.items);
|
||||
const newDocs = response.items.map(apiToDisplayDoc);
|
||||
for (const doc of newDocs) {
|
||||
knownApiIdsRef.current.add(doc.id);
|
||||
}
|
||||
|
||||
setDocuments((prev) => {
|
||||
const existingIds = new Set(prev.map((d) => d.id));
|
||||
const deduped = newDocs.filter((d) => !existingIds.has(d.id));
|
||||
return [...prev, ...deduped];
|
||||
});
|
||||
setTotal(response.total);
|
||||
setHasMore(response.has_more);
|
||||
apiLoadedCountRef.current += response.items.length;
|
||||
} catch (err) {
|
||||
console.error("[useDocuments] Load more failed:", err);
|
||||
} finally {
|
||||
setLoadingMore(false);
|
||||
}
|
||||
}, [
|
||||
loadingMore,
|
||||
hasMore,
|
||||
searchSpaceId,
|
||||
typeFilterKey,
|
||||
sortBy,
|
||||
sortOrder,
|
||||
populateUserCache,
|
||||
apiToDisplayDoc,
|
||||
]);
|
||||
|
||||
return {
|
||||
documents,
|
||||
typeCounts,
|
||||
total: documents.length,
|
||||
total,
|
||||
loading,
|
||||
loadingMore,
|
||||
hasMore,
|
||||
loadMore,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue