feat: introduce baseline utility functions for tracking new items in useDocuments and useInbox hooks, improving accuracy in handling Electric updates and timestamps

surfsense_web/hooks/use-documents.ts

@@ -4,6 +4,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import type { DocumentSortBy, DocumentTypeEnum, SortOrder } from "@/contracts/types/document.types";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import type { SyncHandle } from "@/lib/electric/client";
+import { filterNewElectricItems, getNewestTimestamp } from "@/lib/electric/baseline";
 import { useElectricClient } from "@/lib/electric/context";
 
 export interface DocumentStatusType {
@@ -105,6 +106,7 @@ export function useDocuments(
 	// Snapshot of all doc IDs from Electric's first callback after initial load.
 	// Anything appearing in subsequent callbacks NOT in this set is genuinely new.
 	const electricBaselineIdsRef = useRef<Set<number> | null>(null);
+	const newestApiTimestampRef = useRef<string | null>(null);
 	const userCacheRef = useRef<Map<string, string>>(new Map());
 	const emailCacheRef = useRef<Map<string, string>>(new Map());
 	const syncHandleRef = useRef<SyncHandle | null>(null);
@@ -177,6 +179,7 @@ export function useDocuments(
 		apiLoadedCountRef.current = 0;
 		initialLoadDoneRef.current = false;
 		electricBaselineIdsRef.current = null;
+		newestApiTimestampRef.current = null;
 
 		const fetchInitialData = async () => {
 			try {
@@ -206,6 +209,7 @@ export function useDocuments(
 				setTypeCounts(countsResponse);
 				setError(null);
 				apiLoadedCountRef.current = docsResponse.items.length;
+				newestApiTimestampRef.current = getNewestTimestamp(docsResponse.items);
 				initialLoadDoneRef.current = true;
 			} catch (err) {
 				if (cancelled) return;
@@ -351,30 +355,10 @@ export function useDocuments(
 						const liveIds = new Set(validItems.map((d) => d.id));
 						const prevIds = new Set(prev.map((d) => d.id));
 
-					// Only baseline items already rendered from API.
-					// Items in Electric but NOT in prev are genuinely new
-					// (created between the API fetch and Electric's first callback).
-					if (electricBaselineIdsRef.current === null) {
-						electricBaselineIdsRef.current = new Set(
-							[...liveIds].filter((id) => prevIds.has(id))
-						);
-					}
-
-						// Genuinely new = not in rendered list, not in baseline snapshot.
-						// These are docs created AFTER the sidebar opened.
-						const baseline = electricBaselineIdsRef.current;
-						const newItems = validItems
-							.filter((item) => {
-								if (prevIds.has(item.id)) return false;
-								if (baseline.has(item.id)) return false;
-								return true;
-							})
-							.map(electricToDisplayDoc);
-
-						// Track new items in baseline so they aren't re-added
-						for (const item of newItems) {
-							baseline.add(item.id);
-						}
+						const newItems = filterNewElectricItems(
+							validItems, liveIds, prevIds,
+							electricBaselineIdsRef, newestApiTimestampRef.current,
+						).map(electricToDisplayDoc);
 
 						// Update existing docs (status changes, title edits)
 						let updated = prev.map((doc) => {
@@ -451,6 +435,7 @@ export function useDocuments(
 			apiLoadedCountRef.current = 0;
 			initialLoadDoneRef.current = false;
 			electricBaselineIdsRef.current = null;
+			newestApiTimestampRef.current = null;
 			userCacheRef.current.clear();
 			emailCacheRef.current.clear();
 		}

surfsense_web/hooks/use-inbox.ts

@@ -3,6 +3,7 @@
 import { useCallback, useEffect, useRef, useState } from "react";
 import type { InboxItem, NotificationCategory } from "@/contracts/types/inbox.types";
 import { notificationsApiService } from "@/lib/apis/notifications-api.service";
+import { filterNewElectricItems, getNewestTimestamp } from "@/lib/electric/baseline";
 import { useElectricClient } from "@/lib/electric/context";
 
 export type { InboxItem, InboxItemTypeEnum, NotificationCategory } from "@/contracts/types/inbox.types";
@@ -64,6 +65,7 @@ export function useInbox(
 
 	const initialLoadDoneRef = useRef(false);
 	const electricBaselineIdsRef = useRef<Set<number> | null>(null);
+	const newestApiTimestampRef = useRef<string | null>(null);
 	const liveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null);
 	const unreadLiveQueryRef = useRef<{ unsubscribe?: () => void } | null>(null);
 
@@ -81,6 +83,7 @@ export function useInbox(
 		setHasMore(false);
 		initialLoadDoneRef.current = false;
 		electricBaselineIdsRef.current = null;
+		newestApiTimestampRef.current = null;
 		olderUnreadOffsetRef.current = null;
 		apiUnreadTotalRef.current = 0;
 
@@ -103,6 +106,7 @@ export function useInbox(
 				setHasMore(notificationsResponse.has_more);
 				setUnreadCount(unreadResponse.total_unread);
 				apiUnreadTotalRef.current = unreadResponse.total_unread;
+				newestApiTimestampRef.current = getNewestTimestamp(notificationsResponse.items);
 				setError(null);
 				initialLoadDoneRef.current = true;
 			} catch (err) {
@@ -202,24 +206,10 @@ export function useInbox(
 					setInboxItems((prev) => {
 						const prevIds = new Set(prev.map((d) => d.id));
 
-						if (electricBaselineIdsRef.current === null) {
-							// Only baseline items already rendered from API.
-							// Items in Electric but NOT in prev are genuinely new.
-							electricBaselineIdsRef.current = new Set(
-								[...liveIds].filter((id) => prevIds.has(id))
-							);
-						}
-
-						const baseline = electricBaselineIdsRef.current;
-						const newItems = validItems.filter((item) => {
-							if (prevIds.has(item.id)) return false;
-							if (baseline.has(item.id)) return false;
-							return true;
-						});
-
-						for (const item of newItems) {
-							baseline.add(item.id);
-						}
+						const newItems = filterNewElectricItems(
+							validItems, liveIds, prevIds,
+							electricBaselineIdsRef, newestApiTimestampRef.current,
+						);
 
 						let updated = prev.map((item) => {
 							const liveItem = liveItemMap.get(item.id);

surfsense_web/lib/electric/baseline.ts

@@ -0,0 +1,62 @@
+import type { MutableRefObject } from "react";
+
+/**
+ * Extract the newest `created_at` timestamp from a list of items.
+ * Used to establish the server-clock cutoff for the baseline timing-gap check.
+ *
+ * Uses Date parsing instead of string comparison because the API (Python
+ * isoformat: "+00:00" suffix) and Electric/PGlite ("Z" suffix, variable
+ * fractional-second precision) produce different string formats.
+ */
+export function getNewestTimestamp<T extends { created_at: string }>(items: T[]): string | null {
+	if (items.length === 0) return null;
+	let newest = items[0].created_at;
+	let newestMs = new Date(newest).getTime();
+	for (let i = 1; i < items.length; i++) {
+		const ms = new Date(items[i].created_at).getTime();
+		if (ms > newestMs) {
+			newest = items[i].created_at;
+			newestMs = ms;
+		}
+	}
+	return newest;
+}
+
+/**
+ * Identify genuinely new items from an Electric live query callback.
+ *
+ * On Electric's first callback, ALL live IDs are snapshotted as the baseline.
+ * Items beyond the API's first page are in this baseline and stay hidden
+ * (they'll appear via scroll pagination). Items created in the timing gap
+ * between the API fetch and Electric's first callback are rescued via the
+ * `newestApiTimestamp` check — their `created_at` is newer than anything
+ * the API returned, so they pass through.
+ *
+ */
+export function filterNewElectricItems<T extends { id: number; created_at: string }>(
+	validItems: T[],
+	liveIds: Set<number>,
+	prevIds: Set<number>,
+	baselineRef: MutableRefObject<Set<number> | null>,
+	newestApiTimestamp: string | null,
+): T[] {
+	if (baselineRef.current === null) {
+		baselineRef.current = new Set(liveIds);
+	}
+
+	const baseline = baselineRef.current;
+	const cutoffMs = newestApiTimestamp ? new Date(newestApiTimestamp).getTime() : null;
+
+	const newItems = validItems.filter((item) => {
+		if (prevIds.has(item.id)) return false;
+		if (!baseline.has(item.id)) return true;
+		if (cutoffMs !== null && new Date(item.created_at).getTime() > cutoffMs) return true;
+		return false;
+	});
+
+	for (const item of newItems) {
+		baseline.add(item.id);
+	}
+
+	return newItems;
+}