diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts index 50c2f73ce..e0b576de6 100644 --- a/surfsense_web/hooks/use-documents.ts +++ b/surfsense_web/hooks/use-documents.ts @@ -4,6 +4,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import type { DocumentSortBy, DocumentTypeEnum, SortOrder } from "@/contracts/types/document.types"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import type { SyncHandle } from "@/lib/electric/client"; +import { filterNewElectricItems, getNewestTimestamp } from "@/lib/electric/baseline"; import { useElectricClient } from "@/lib/electric/context"; export interface DocumentStatusType { @@ -105,6 +106,7 @@ export function useDocuments( // Snapshot of all doc IDs from Electric's first callback after initial load. // Anything appearing in subsequent callbacks NOT in this set is genuinely new. const electricBaselineIdsRef = useRef | null>(null); + const newestApiTimestampRef = useRef(null); const userCacheRef = useRef>(new Map()); const emailCacheRef = useRef>(new Map()); const syncHandleRef = useRef(null); @@ -177,6 +179,7 @@ export function useDocuments( apiLoadedCountRef.current = 0; initialLoadDoneRef.current = false; electricBaselineIdsRef.current = null; + newestApiTimestampRef.current = null; const fetchInitialData = async () => { try { @@ -206,6 +209,7 @@ export function useDocuments( setTypeCounts(countsResponse); setError(null); apiLoadedCountRef.current = docsResponse.items.length; + newestApiTimestampRef.current = getNewestTimestamp(docsResponse.items); initialLoadDoneRef.current = true; } catch (err) { if (cancelled) return; @@ -351,30 +355,10 @@ export function useDocuments( const liveIds = new Set(validItems.map((d) => d.id)); const prevIds = new Set(prev.map((d) => d.id)); - // Only baseline items already rendered from API. - // Items in Electric but NOT in prev are genuinely new - // (created between the API fetch and Electric's first callback). - if (electricBaselineIdsRef.current === null) { - electricBaselineIdsRef.current = new Set( - [...liveIds].filter((id) => prevIds.has(id)) - ); - } - - // Genuinely new = not in rendered list, not in baseline snapshot. - // These are docs created AFTER the sidebar opened. - const baseline = electricBaselineIdsRef.current; - const newItems = validItems - .filter((item) => { - if (prevIds.has(item.id)) return false; - if (baseline.has(item.id)) return false; - return true; - }) - .map(electricToDisplayDoc); - - // Track new items in baseline so they aren't re-added - for (const item of newItems) { - baseline.add(item.id); - } + const newItems = filterNewElectricItems( + validItems, liveIds, prevIds, + electricBaselineIdsRef, newestApiTimestampRef.current, + ).map(electricToDisplayDoc); // Update existing docs (status changes, title edits) let updated = prev.map((doc) => { @@ -451,6 +435,7 @@ export function useDocuments( apiLoadedCountRef.current = 0; initialLoadDoneRef.current = false; electricBaselineIdsRef.current = null; + newestApiTimestampRef.current = null; userCacheRef.current.clear(); emailCacheRef.current.clear(); } diff --git a/surfsense_web/hooks/use-inbox.ts b/surfsense_web/hooks/use-inbox.ts index 9c20a1cda..5a991c30d 100644 --- a/surfsense_web/hooks/use-inbox.ts +++ b/surfsense_web/hooks/use-inbox.ts @@ -3,6 +3,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import type { InboxItem, NotificationCategory } from "@/contracts/types/inbox.types"; import { notificationsApiService } from "@/lib/apis/notifications-api.service"; +import { filterNewElectricItems, getNewestTimestamp } from "@/lib/electric/baseline"; import { useElectricClient } from "@/lib/electric/context"; export type { InboxItem, InboxItemTypeEnum, NotificationCategory } from "@/contracts/types/inbox.types"; @@ -64,6 +65,7 @@ export function useInbox( const initialLoadDoneRef = useRef(false); const electricBaselineIdsRef = useRef | null>(null); + const newestApiTimestampRef = useRef(null); const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null); const unreadLiveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null); @@ -81,6 +83,7 @@ export function useInbox( setHasMore(false); initialLoadDoneRef.current = false; electricBaselineIdsRef.current = null; + newestApiTimestampRef.current = null; olderUnreadOffsetRef.current = null; apiUnreadTotalRef.current = 0; @@ -103,6 +106,7 @@ export function useInbox( setHasMore(notificationsResponse.has_more); setUnreadCount(unreadResponse.total_unread); apiUnreadTotalRef.current = unreadResponse.total_unread; + newestApiTimestampRef.current = getNewestTimestamp(notificationsResponse.items); setError(null); initialLoadDoneRef.current = true; } catch (err) { @@ -202,24 +206,10 @@ export function useInbox( setInboxItems((prev) => { const prevIds = new Set(prev.map((d) => d.id)); - if (electricBaselineIdsRef.current === null) { - // Only baseline items already rendered from API. - // Items in Electric but NOT in prev are genuinely new. - electricBaselineIdsRef.current = new Set( - [...liveIds].filter((id) => prevIds.has(id)) - ); - } - - const baseline = electricBaselineIdsRef.current; - const newItems = validItems.filter((item) => { - if (prevIds.has(item.id)) return false; - if (baseline.has(item.id)) return false; - return true; - }); - - for (const item of newItems) { - baseline.add(item.id); - } + const newItems = filterNewElectricItems( + validItems, liveIds, prevIds, + electricBaselineIdsRef, newestApiTimestampRef.current, + ); let updated = prev.map((item) => { const liveItem = liveItemMap.get(item.id); diff --git a/surfsense_web/lib/electric/baseline.ts b/surfsense_web/lib/electric/baseline.ts new file mode 100644 index 000000000..018163f9d --- /dev/null +++ b/surfsense_web/lib/electric/baseline.ts @@ -0,0 +1,62 @@ +import type { MutableRefObject } from "react"; + +/** + * Extract the newest `created_at` timestamp from a list of items. + * Used to establish the server-clock cutoff for the baseline timing-gap check. + * + * Uses Date parsing instead of string comparison because the API (Python + * isoformat: "+00:00" suffix) and Electric/PGlite ("Z" suffix, variable + * fractional-second precision) produce different string formats. + */ +export function getNewestTimestamp(items: T[]): string | null { + if (items.length === 0) return null; + let newest = items[0].created_at; + let newestMs = new Date(newest).getTime(); + for (let i = 1; i < items.length; i++) { + const ms = new Date(items[i].created_at).getTime(); + if (ms > newestMs) { + newest = items[i].created_at; + newestMs = ms; + } + } + return newest; +} + +/** + * Identify genuinely new items from an Electric live query callback. + * + * On Electric's first callback, ALL live IDs are snapshotted as the baseline. + * Items beyond the API's first page are in this baseline and stay hidden + * (they'll appear via scroll pagination). Items created in the timing gap + * between the API fetch and Electric's first callback are rescued via the + * `newestApiTimestamp` check — their `created_at` is newer than anything + * the API returned, so they pass through. + * + */ +export function filterNewElectricItems( + validItems: T[], + liveIds: Set, + prevIds: Set, + baselineRef: MutableRefObject | null>, + newestApiTimestamp: string | null, +): T[] { + if (baselineRef.current === null) { + baselineRef.current = new Set(liveIds); + } + + const baseline = baselineRef.current; + const cutoffMs = newestApiTimestamp ? new Date(newestApiTimestamp).getTime() : null; + + const newItems = validItems.filter((item) => { + if (prevIds.has(item.id)) return false; + if (!baseline.has(item.id)) return true; + if (cutoffMs !== null && new Date(item.created_at).getTime() > cutoffMs) return true; + return false; + }); + + for (const item of newItems) { + baseline.add(item.id); + } + + return newItems; +}