apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-27 09:46:25 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: rewrite use-documents hook from Electric to Zero

Browse source 
Keep all API calls (initial load, pagination, type counts, user
enrichment). Replace Electric sync+PGlite with Zero useQuery for
real-time deltas. Remove baseline dedup. 511 → 361 lines.
This commit is contained in:
CREDO23 2026-03-23 19:16:38 +02:00
parent cd9d8ca991
commit 833fbb2a76
1 changed files with 94 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

248
surfsense_web/hooks/use-documents.ts
View file

@ -3,25 +3,14 @@
import { useCallback, useEffect, useRef, useState } from "react"; import { useCallback, useEffect, useRef, useState } from "react";
import type { DocumentSortBy, DocumentTypeEnum, SortOrder } from "@/contracts/types/document.types"; import type { DocumentSortBy, DocumentTypeEnum, SortOrder } from "@/contracts/types/document.types";
import { documentsApiService } from "@/lib/apis/documents-api.service"; import { documentsApiService } from "@/lib/apis/documents-api.service";
import { filterNewElectricItems, getNewestTimestamp } from "@/lib/electric/baseline"; import { queries } from "@/zero/queries";
import type { SyncHandle } from "@/lib/electric/client"; import { useQuery } from "@rocicorp/zero/react";
import { useElectricClient } from "@/lib/electric/context";
export interface DocumentStatusType { export interface DocumentStatusType {
state: "ready" | "pending" | "processing" | "failed"; state: "ready" | "pending" | "processing" | "failed";
reason?: string; reason?: string;
} }
interface DocumentElectric {
id: number;
search_space_id: number;
document_type: string;
title: string;
created_by_id: string | null;
created_at: string;
status: DocumentStatusType | null;
}
export interface DocumentDisplay { export interface DocumentDisplay {
id: number; id: number;
search_space_id: number; search_space_id: number;
@ -64,23 +53,14 @@ const EMPTY_TYPE_FILTER: DocumentTypeEnum[] = [];
const INITIAL_PAGE_SIZE = 50; const INITIAL_PAGE_SIZE = 50;
const SCROLL_PAGE_SIZE = 5; const SCROLL_PAGE_SIZE = 5;
function isValidDocument(doc: DocumentElectric): boolean {
return doc.id != null && doc.title != null && doc.title !== "";
}
/** /**
* Paginated documents hook with Electric SQL real-time updates. * Paginated documents hook with Zero real-time updates.
* *
* Architecture: * Architecture:
* 1. API is the PRIMARY data source — fetches pages on demand * 1. API is the PRIMARY data source — fetches pages on demand
* 2. Type counts come from a dedicated lightweight API endpoint * 2. Type counts come from a dedicated lightweight API endpoint
* 3. Electric provides REAL-TIME updates (new docs, deletions, status changes) * 3. Zero provides REAL-TIME updates (new docs, deletions, status changes)
* 4. Server-side sorting via sort_by + sort_order params * 4. Server-side sorting via sort_by + sort_order params
*
* @param searchSpaceId - The search space to load documents for
* @param typeFilter - Document types to filter by (server-side)
* @param sortBy - Column to sort by (server-side)
* @param sortOrder - Sort direction (server-side)
*/ */
export function useDocuments( export function useDocuments(
searchSpaceId: number | null, searchSpaceId: number | null,
@ -88,8 +68,6 @@ export function useDocuments(
sortBy: DocumentSortBy = "created_at", sortBy: DocumentSortBy = "created_at",
sortOrder: SortOrder = "desc" sortOrder: SortOrder = "desc"
) { ) {
const electricClient = useElectricClient();
const [documents, setDocuments] = useState<DocumentDisplay[]>([]); const [documents, setDocuments] = useState<DocumentDisplay[]>([]);
const [typeCounts, setTypeCounts] = useState<Record<string, number>>({}); const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
const [total, setTotal] = useState(0); const [total, setTotal] = useState(0);
@ -103,14 +81,8 @@ export function useDocuments(
const prevParamsRef = useRef<{ sortBy: string; sortOrder: string; typeFilterKey: string } | null>( const prevParamsRef = useRef<{ sortBy: string; sortOrder: string; typeFilterKey: string } | null>(
null null
); );
// Snapshot of all doc IDs from Electric's first callback after initial load.
// Anything appearing in subsequent callbacks NOT in this set is genuinely new.
const electricBaselineIdsRef = useRef<Set<number> | null>(null);
const newestApiTimestampRef = useRef<string | null>(null);
const userCacheRef = useRef<Map<string, string>>(new Map()); const userCacheRef = useRef<Map<string, string>>(new Map());
const emailCacheRef = useRef<Map<string, string>>(new Map()); const emailCacheRef = useRef<Map<string, string>>(new Map());
const syncHandleRef = useRef<SyncHandle | null>(null);
const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null);
const typeFilterKey = typeFilter.join(","); const typeFilterKey = typeFilter.join(",");
@ -141,20 +113,6 @@ export function useDocuments(
[] []
); );
const electricToDisplayDoc = useCallback(
(doc: DocumentElectric): DocumentDisplay => ({
...doc,
created_by_name: doc.created_by_id
? (userCacheRef.current.get(doc.created_by_id) ?? null)
: null,
created_by_email: doc.created_by_id
? (emailCacheRef.current.get(doc.created_by_id) ?? null)
: null,
status: doc.status ?? { state: "ready" },
}),
[]
);
// EFFECT 1: Fetch first page + type counts when params change // EFFECT 1: Fetch first page + type counts when params change
// biome-ignore lint/correctness/useExhaustiveDependencies: typeFilterKey serializes typeFilter // biome-ignore lint/correctness/useExhaustiveDependencies: typeFilterKey serializes typeFilter
useEffect(() => { useEffect(() => {
@ -178,8 +136,6 @@ export function useDocuments(
} }
apiLoadedCountRef.current = 0; apiLoadedCountRef.current = 0;
initialLoadDoneRef.current = false; initialLoadDoneRef.current = false;
electricBaselineIdsRef.current = null;
newestApiTimestampRef.current = null;
const fetchInitialData = async () => { const fetchInitialData = async () => {
try { try {
@ -209,7 +165,6 @@ export function useDocuments(
setTypeCounts(countsResponse); setTypeCounts(countsResponse);
setError(null); setError(null);
apiLoadedCountRef.current = docsResponse.items.length; apiLoadedCountRef.current = docsResponse.items.length;
newestApiTimestampRef.current = getNewestTimestamp(docsResponse.items);
initialLoadDoneRef.current = true; initialLoadDoneRef.current = true;
} catch (err) { } catch (err) {
if (cancelled) return; if (cancelled) return;
@ -226,115 +181,33 @@ export function useDocuments(
}; };
}, [searchSpaceId, typeFilterKey, sortBy, sortOrder, populateUserCache, apiToDisplayDoc]); }, [searchSpaceId, typeFilterKey, sortBy, sortOrder, populateUserCache, apiToDisplayDoc]);
// EFFECT 2: Electric sync + live query for real-time updates // EFFECT 2: Zero real-time sync for document updates
const [zeroDocuments] = useQuery(
queries.documents.bySpace({ searchSpaceId: searchSpaceId ?? -1 })
);
useEffect(() => { useEffect(() => {
if (!searchSpaceId || !electricClient) return; if (!searchSpaceId || !zeroDocuments || !initialLoadDoneRef.current) return;
const spaceId = searchSpaceId; const validItems = zeroDocuments.filter(
const client = electricClient; (doc) => doc.id != null && doc.title != null && doc.title !== ""
let mounted = true; );
async function setupElectricRealtime() { const unknownUserIds = validItems.filter(
if (syncHandleRef.current) { (doc) => doc.createdById !== null && !userCacheRef.current.has(doc.createdById!)
try { );
syncHandleRef.current.unsubscribe();
} catch {
/* PGlite may already be closed */
}
syncHandleRef.current = null;
}
if (liveQueryRef.current) {
try {
liveQueryRef.current.unsubscribe?.();
} catch {
/* PGlite may already be closed */
}
liveQueryRef.current = null;
}
try {
const handle = await client.syncShape({
table: "documents",
where: `search_space_id = ${spaceId}`,
columns: [
"id",
"document_type",
"search_space_id",
"title",
"created_by_id",
"created_at",
"status",
],
primaryKey: ["id"],
});
if (!mounted) {
handle.unsubscribe();
return;
}
syncHandleRef.current = handle;
if (!handle.isUpToDate && handle.initialSyncPromise) {
await Promise.race([
handle.initialSyncPromise,
new Promise((resolve) => setTimeout(resolve, 5000)),
]);
}
if (!mounted) return;
const db = client.db as {
live?: {
query: <T>(
sql: string,
params?: (number | string)[]
) => Promise<{
subscribe: (cb: (result: { rows: T[] }) => void) => void;
unsubscribe?: () => void;
}>;
};
};
if (!db.live?.query) return;
const query = `SELECT id, document_type, search_space_id, title, created_by_id, created_at, status
FROM documents
WHERE search_space_id = $1
ORDER BY created_at DESC`;
const liveQuery = await db.live.query<DocumentElectric>(query, [spaceId]);
if (!mounted) {
liveQuery.unsubscribe?.();
return;
}
liveQuery.subscribe((result: { rows: DocumentElectric[] }) => {
if (!mounted || !result.rows || !initialLoadDoneRef.current) return;
const validItems = result.rows.filter(isValidDocument);
const isFullySynced = syncHandleRef.current?.isUpToDate ?? false;
const unknownUserIds = validItems
.filter(
(doc): doc is DocumentElectric & { created_by_id: string } =>
doc.created_by_id !== null && !userCacheRef.current.has(doc.created_by_id)
)
.map((doc) => doc.created_by_id);
if (unknownUserIds.length > 0) { if (unknownUserIds.length > 0) {
documentsApiService documentsApiService
.getDocuments({ .getDocuments({
queryParams: { queryParams: {
search_space_id: spaceId, search_space_id: searchSpaceId,
page: 0, page: 0,
page_size: 20, page_size: 20,
}, },
}) })
.then((response) => { .then((response) => {
populateUserCache(response.items); populateUserCache(response.items);
if (mounted) {
setDocuments((prev) => setDocuments((prev) =>
prev.map((doc) => ({ prev.map((doc) => ({
...doc, ...doc,
@ -346,38 +219,49 @@ export function useDocuments(
: null, : null,
})) }))
); );
}
}) })
.catch(() => {}); .catch(() => {});
} }
setDocuments((prev) => {
const liveIds = new Set(validItems.map((d) => d.id)); const liveIds = new Set(validItems.map((d) => d.id));
setDocuments((prev) => {
const prevIds = new Set(prev.map((d) => d.id)); const prevIds = new Set(prev.map((d) => d.id));
const newItems = filterNewElectricItems( const newItems: DocumentDisplay[] = validItems
validItems, .filter((d) => !prevIds.has(d.id))
liveIds, .map((doc) => ({
prevIds, id: doc.id,
electricBaselineIdsRef, search_space_id: doc.searchSpaceId,
newestApiTimestampRef.current document_type: doc.documentType,
).map(electricToDisplayDoc); title: doc.title,
created_by_id: doc.createdById ?? null,
created_by_name: doc.createdById
? (userCacheRef.current.get(doc.createdById) ?? null)
: null,
created_by_email: doc.createdById
? (emailCacheRef.current.get(doc.createdById) ?? null)
: null,
created_at: String(doc.createdAt),
status: (doc.status as unknown as DocumentStatusType) ?? { state: "ready" },
}));
// Update existing docs (status changes, title edits) let updated = prev.map((existing) => {
let updated = prev.map((doc) => { if (liveIds.has(existing.id)) {
if (liveIds.has(doc.id)) { const liveItem = validItems.find((v) => v.id === existing.id);
const liveItem = validItems.find((v) => v.id === doc.id);
if (liveItem) { if (liveItem) {
return electricToDisplayDoc(liveItem); return {
...existing,
title: liveItem.title,
document_type: liveItem.documentType,
status: (liveItem.status as unknown as DocumentStatusType) ?? { state: "ready" as const },
};
} }
} }
return doc; return existing;
}); });
// Remove deleted docs (only when fully synced)
if (isFullySynced) {
updated = updated.filter((doc) => liveIds.has(doc.id)); updated = updated.filter((doc) => liveIds.has(doc.id));
}
if (newItems.length > 0) { if (newItems.length > 0) {
return [...newItems, ...updated]; return [...newItems, ...updated];
@ -386,47 +270,15 @@ export function useDocuments(
return updated; return updated;
}); });
// Update type counts when Electric detects changes
if (isFullySynced && validItems.length > 0) {
const counts: Record<string, number> = {}; const counts: Record<string, number> = {};
for (const item of validItems) { for (const item of validItems) {
counts[item.document_type] = (counts[item.document_type] || 0) + 1; counts[item.documentType] = (counts[item.documentType] || 0) + 1;
} }
setTypeCounts(counts); setTypeCounts(counts);
setTotal(validItems.length); setTotal(validItems.length);
} }, [searchSpaceId, zeroDocuments, populateUserCache]);
});
liveQueryRef.current = liveQuery; // EFFECT 3: Reset on search space change
} catch (err) {
console.error("[useDocuments] Electric setup failed:", err);
}
}
setupElectricRealtime();
return () => {
mounted = false;
if (syncHandleRef.current) {
try {
syncHandleRef.current.unsubscribe();
} catch {
/* PGlite may already be closed */
}
syncHandleRef.current = null;
}
if (liveQueryRef.current) {
try {
liveQueryRef.current.unsubscribe?.();
} catch {
/* PGlite may already be closed */
}
liveQueryRef.current = null;
}
};
}, [searchSpaceId, electricClient, electricToDisplayDoc, populateUserCache]);
// Reset on search space change
const prevSearchSpaceIdRef = useRef<number | null>(null); const prevSearchSpaceIdRef = useRef<number | null>(null);
useEffect(() => { useEffect(() => {
@ -437,8 +289,6 @@ export function useDocuments(
setHasMore(false); setHasMore(false);
apiLoadedCountRef.current = 0; apiLoadedCountRef.current = 0;
initialLoadDoneRef.current = false; initialLoadDoneRef.current = false;
electricBaselineIdsRef.current = null;
newestApiTimestampRef.current = null;
userCacheRef.current.clear(); userCacheRef.current.clear();
emailCacheRef.current.clear(); emailCacheRef.current.clear();
} }