mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-04-26 01:06:23 +02:00
feat: Add notifications table and integrate Electric SQL for real-time updates
- Introduced a new notifications table in the database schema to manage user notifications. - Implemented Electric SQL replication setup for the notifications table, ensuring real-time synchronization. - Updated existing database functions to support real-time updates for connectors and documents using Electric SQL. - Refactored UI components to utilize new hooks for fetching connectors and documents, enhancing performance and user experience.
This commit is contained in:
Anish Sarkar
parent
7a92ecc1ab
commit
e38e6d90e0
9 changed files with 489 additions and 178 deletions
|
|
@ -1,17 +1,17 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { Cable, Loader2 } from "lucide-react";
|
||||
import { useSearchParams } from "next/navigation";
|
||||
import { type FC, useEffect, useMemo } from "react";
|
||||
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
|
||||
import { type FC, useMemo } from "react";
|
||||
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
|
||||
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
|
||||
import { Dialog, DialogContent } from "@/components/ui/dialog";
|
||||
import { Tabs, TabsContent } from "@/components/ui/tabs";
|
||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||
import { useLogsSummary } from "@/hooks/use-logs";
|
||||
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
|
||||
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
|
||||
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
|
@ -29,8 +29,9 @@ import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view
|
|||
export const ConnectorIndicator: FC = () => {
|
||||
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
|
||||
const searchParams = useSearchParams();
|
||||
const { data: documentTypeCounts, isLoading: documentTypesLoading } =
|
||||
useAtomValue(documentTypeCountsAtom);
|
||||
|
||||
// Fetch document type counts using Electric SQL + PGlite for real-time updates
|
||||
const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId);
|
||||
|
||||
// Check if YouTube view is active
|
||||
const isYouTubeView = searchParams.get("view") === "youtube";
|
||||
|
|
@ -93,47 +94,31 @@ export const ConnectorIndicator: FC = () => {
|
|||
setConnectorName,
|
||||
} = useConnectorDialog();
|
||||
|
||||
// Fetch connectors using React Query with conditional refetchInterval
|
||||
// This automatically refetches when mutations invalidate the cache (event-driven)
|
||||
// and also polls when dialog is open to catch external changes
|
||||
// Fetch connectors using Electric SQL + PGlite for real-time updates
|
||||
// This provides instant updates when connectors change, without polling
|
||||
const {
|
||||
data: connectors = [],
|
||||
isLoading: connectorsLoading,
|
||||
refetch: refreshConnectors,
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.connectors.all(searchSpaceId || ""),
|
||||
queryFn: () =>
|
||||
connectorsApiService.getConnectors({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId ? Number(searchSpaceId) : undefined,
|
||||
},
|
||||
}),
|
||||
enabled: !!searchSpaceId,
|
||||
staleTime: 5 * 60 * 1000, // 5 minutes (same as connectorsAtom)
|
||||
// Poll when dialog is open to catch external changes
|
||||
refetchInterval: isOpen ? 5000 : false, // 5 seconds when open, no polling when closed
|
||||
});
|
||||
connectors: connectorsFromElectric = [],
|
||||
loading: connectorsLoading,
|
||||
error: connectorsError,
|
||||
refreshConnectors: refreshConnectorsElectric,
|
||||
} = useConnectorsElectric(searchSpaceId);
|
||||
|
||||
const queryClient = useQueryClient();
|
||||
// Fallback to API if Electric fails or is not available
|
||||
const connectors = connectorsFromElectric.length > 0 || !connectorsError
|
||||
? connectorsFromElectric
|
||||
: allConnectors || [];
|
||||
|
||||
// Also refresh document type counts when dialog is open
|
||||
useEffect(() => {
|
||||
if (!isOpen || !searchSpaceId) return;
|
||||
// Manual refresh function that works with both Electric and API
|
||||
const refreshConnectors = async () => {
|
||||
if (connectorsFromElectric.length > 0 || !connectorsError) {
|
||||
await refreshConnectorsElectric();
|
||||
} else {
|
||||
// Fallback: use allConnectors from useConnectorDialog (which uses connectorsAtom)
|
||||
// The connectorsAtom will handle refetching if needed
|
||||
}
|
||||
};
|
||||
|
||||
const POLL_INTERVAL = 5000; // 5 seconds, same as connectors
|
||||
|
||||
const intervalId = setInterval(() => {
|
||||
// Invalidate document type counts to refresh active document types
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.typeCounts(searchSpaceId),
|
||||
});
|
||||
}, POLL_INTERVAL);
|
||||
|
||||
// Cleanup interval on unmount or when dialog closes
|
||||
return () => {
|
||||
clearInterval(intervalId);
|
||||
};
|
||||
}, [isOpen, searchSpaceId, queryClient]);
|
||||
// Document type counts now update in real-time via Electric SQL - no polling needed!
|
||||
|
||||
// Get connector IDs that are currently being indexed
|
||||
const indexingConnectorIds = useMemo(() => {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { IconBrandYoutube } from "@tabler/icons-react";
|
||||
import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
|
||||
import { FileText, Loader2 } from "lucide-react";
|
||||
import type { FC } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -20,7 +19,6 @@ interface ConnectorCardProps {
|
|||
isConnecting?: boolean;
|
||||
documentCount?: number;
|
||||
accountCount?: number;
|
||||
lastIndexedAt?: string | null;
|
||||
isIndexing?: boolean;
|
||||
activeTask?: LogActiveTask;
|
||||
onConnect?: () => void;
|
||||
|
|
@ -52,45 +50,6 @@ function formatDocumentCount(count: number | undefined): string {
|
|||
return `${m.replace(/\.0$/, "")}M docs`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format last indexed date with contextual messages
|
||||
* Examples: "Just now", "10 minutes ago", "Today at 2:30 PM", "Yesterday at 3:45 PM", "3 days ago", "Jan 15, 2026"
|
||||
*/
|
||||
function formatLastIndexedDate(dateString: string): string {
|
||||
const date = new Date(dateString);
|
||||
const now = new Date();
|
||||
const minutesAgo = differenceInMinutes(now, date);
|
||||
const daysAgo = differenceInDays(now, date);
|
||||
|
||||
// Just now (within last minute)
|
||||
if (minutesAgo < 1) {
|
||||
return "Just now";
|
||||
}
|
||||
|
||||
// X minutes ago (less than 1 hour)
|
||||
if (minutesAgo < 60) {
|
||||
return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
|
||||
}
|
||||
|
||||
// Today at [time]
|
||||
if (isToday(date)) {
|
||||
return `Today at ${format(date, "h:mm a")}`;
|
||||
}
|
||||
|
||||
// Yesterday at [time]
|
||||
if (isYesterday(date)) {
|
||||
return `Yesterday at ${format(date, "h:mm a")}`;
|
||||
}
|
||||
|
||||
// X days ago (less than 7 days)
|
||||
if (daysAgo < 7) {
|
||||
return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
|
||||
}
|
||||
|
||||
// Full date for older entries
|
||||
return format(date, "MMM d, yyyy");
|
||||
}
|
||||
|
||||
export const ConnectorCard: FC<ConnectorCardProps> = ({
|
||||
id,
|
||||
title,
|
||||
|
|
@ -100,7 +59,6 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
|
|||
isConnecting = false,
|
||||
documentCount,
|
||||
accountCount,
|
||||
lastIndexedAt,
|
||||
isIndexing = false,
|
||||
activeTask,
|
||||
onConnect,
|
||||
|
|
@ -118,37 +76,29 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
|
|||
// Extract count from active task message during indexing
|
||||
const indexingCount = extractIndexedCount(activeTask?.message);
|
||||
|
||||
// Determine the status content to display
|
||||
const getStatusContent = () => {
|
||||
if (isIndexing) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 w-full max-w-[200px]">
|
||||
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
|
||||
{indexingCount !== null ? <>{indexingCount.toLocaleString()} indexed</> : "Syncing..."}
|
||||
</span>
|
||||
{/* Indeterminate progress bar with animation */}
|
||||
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
|
||||
<div className="absolute h-full bg-primary rounded-full animate-progress-indeterminate" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (isConnected) {
|
||||
// Show last indexed date for connected connectors
|
||||
if (lastIndexedAt) {
|
||||
// Determine the status content to display
|
||||
const getStatusContent = () => {
|
||||
if (isIndexing) {
|
||||
return (
|
||||
<span className="whitespace-nowrap text-[10px]">
|
||||
Last indexed: {formatLastIndexedDate(lastIndexedAt)}
|
||||
</span>
|
||||
<div className="flex items-center gap-2 w-full max-w-[200px]">
|
||||
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
|
||||
{indexingCount !== null ? <>{indexingCount.toLocaleString()} indexed</> : "Syncing..."}
|
||||
</span>
|
||||
{/* Indeterminate progress bar with animation */}
|
||||
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
|
||||
<div className="absolute h-full bg-primary rounded-full animate-progress-indeterminate" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
// Fallback for connected but never indexed
|
||||
return <span className="whitespace-nowrap text-[10px]">Never indexed</span>;
|
||||
}
|
||||
|
||||
return description;
|
||||
};
|
||||
if (isConnected) {
|
||||
// Don't show last indexed in overview tabs - only show in accounts list view
|
||||
return null;
|
||||
}
|
||||
|
||||
return description;
|
||||
};
|
||||
|
||||
const cardContent = (
|
||||
<div
|
||||
|
|
@ -186,9 +136,10 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
|
|||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-[10px] text-muted-foreground mt-1">{getStatusContent()}</div>
|
||||
{isConnected && documentCount !== undefined && (
|
||||
<p className="text-[10px] text-muted-foreground mt-0.5 flex items-center gap-1.5">
|
||||
{isIndexing ? (
|
||||
<div className="text-[10px] text-muted-foreground mt-1">{getStatusContent()}</div>
|
||||
) : isConnected ? (
|
||||
<p className="text-[10px] text-muted-foreground mt-1 flex items-center gap-1.5">
|
||||
<span>{formatDocumentCount(documentCount)}</span>
|
||||
{accountCount !== undefined && accountCount > 0 && (
|
||||
<>
|
||||
|
|
@ -199,6 +150,8 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
|
|||
</>
|
||||
)}
|
||||
</p>
|
||||
) : (
|
||||
<div className="text-[10px] text-muted-foreground mt-1">{getStatusContent()}</div>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
"use client";
|
||||
|
||||
import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
|
||||
import { ArrowRight, Cable, Loader2 } from "lucide-react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import type { FC } from "react";
|
||||
|
|
@ -67,31 +66,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
|||
return `${m.replace(/\.0$/, "")}M docs`;
|
||||
};
|
||||
|
||||
// Format last indexed date with contextual messages
|
||||
const formatLastIndexedDate = (dateString: string): string => {
|
||||
const date = new Date(dateString);
|
||||
const now = new Date();
|
||||
const minutesAgo = differenceInMinutes(now, date);
|
||||
const daysAgo = differenceInDays(now, date);
|
||||
|
||||
if (minutesAgo < 1) return "Just now";
|
||||
if (minutesAgo < 60) return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
|
||||
if (isToday(date)) return `Today at ${format(date, "h:mm a")}`;
|
||||
if (isYesterday(date)) return `Yesterday at ${format(date, "h:mm a")}`;
|
||||
if (daysAgo < 7) return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
|
||||
return format(date, "MMM d, yyyy");
|
||||
};
|
||||
|
||||
// Get most recent last indexed date from a list of connectors
|
||||
const getMostRecentLastIndexed = (
|
||||
connectorsList: SearchSourceConnector[]
|
||||
): string | undefined => {
|
||||
return connectorsList.reduce<string | undefined>((latest, c) => {
|
||||
if (!c.last_indexed_at) return latest;
|
||||
if (!latest) return c.last_indexed_at;
|
||||
return new Date(c.last_indexed_at) > new Date(latest) ? c.last_indexed_at : latest;
|
||||
}, undefined);
|
||||
};
|
||||
|
||||
// Document types that should be shown as standalone cards (not from connectors)
|
||||
const standaloneDocumentTypes = ["EXTENSION", "FILE", "NOTE", "YOUTUBE_VIDEO", "CRAWLED_URL"];
|
||||
|
|
@ -190,7 +164,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
|||
documentTypeCounts
|
||||
);
|
||||
const accountCount = typeConnectors.length;
|
||||
const mostRecentLastIndexed = getMostRecentLastIndexed(typeConnectors);
|
||||
|
||||
const handleManageClick = () => {
|
||||
if (onViewAccountsList) {
|
||||
|
|
@ -222,19 +195,13 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
|||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-[14px] font-semibold leading-tight truncate">{title}</p>
|
||||
{isAnyIndexing ? (
|
||||
{isAnyIndexing && (
|
||||
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
|
||||
<Loader2 className="size-3 animate-spin" />
|
||||
Indexing...
|
||||
</p>
|
||||
) : (
|
||||
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap">
|
||||
{mostRecentLastIndexed
|
||||
? `Last indexed: ${formatLastIndexedDate(mostRecentLastIndexed)}`
|
||||
: "Never indexed"}
|
||||
</p>
|
||||
)}
|
||||
<p className="text-[10px] text-muted-foreground mt-0.5 flex items-center gap-1.5">
|
||||
<p className="text-[10px] text-muted-foreground mt-1 flex items-center gap-1.5">
|
||||
<span>{formatDocumentCount(documentCount)}</span>
|
||||
<span className="text-muted-foreground/50">•</span>
|
||||
<span>
|
||||
|
|
@ -289,7 +256,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
|||
<p className="text-[14px] font-semibold leading-tight truncate">
|
||||
{connector.name}
|
||||
</p>
|
||||
{isIndexing ? (
|
||||
{isIndexing && (
|
||||
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
|
||||
<Loader2 className="size-3 animate-spin" />
|
||||
Indexing...
|
||||
|
|
@ -299,14 +266,8 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
|||
</span>
|
||||
)}
|
||||
</p>
|
||||
) : (
|
||||
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap">
|
||||
{connector.last_indexed_at
|
||||
? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}`
|
||||
: "Never indexed"}
|
||||
</p>
|
||||
)}
|
||||
<p className="text-[10px] text-muted-foreground mt-0.5">
|
||||
<p className="text-[10px] text-muted-foreground mt-1">
|
||||
{formatDocumentCount(documentCount)}
|
||||
</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -103,18 +103,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
)
|
||||
: [];
|
||||
|
||||
// Get the most recent last_indexed_at across all accounts
|
||||
const mostRecentLastIndexed = typeConnectors.reduce<string | undefined>(
|
||||
(latest, c) => {
|
||||
if (!c.last_indexed_at) return latest;
|
||||
if (!latest) return c.last_indexed_at;
|
||||
return new Date(c.last_indexed_at) > new Date(latest)
|
||||
? c.last_indexed_at
|
||||
: latest;
|
||||
},
|
||||
undefined
|
||||
);
|
||||
|
||||
const documentCount = getDocumentCountForConnector(
|
||||
connector.connectorType,
|
||||
documentTypeCounts
|
||||
|
|
@ -139,7 +127,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
isConnecting={isConnecting}
|
||||
documentCount={documentCount}
|
||||
accountCount={typeConnectors.length}
|
||||
lastIndexedAt={mostRecentLastIndexed}
|
||||
isIndexing={isIndexing}
|
||||
activeTask={activeTask}
|
||||
onConnect={() => onConnectOAuth(connector)}
|
||||
|
|
@ -197,7 +184,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
isConnected={isConnected}
|
||||
isConnecting={isConnecting}
|
||||
documentCount={documentCount}
|
||||
lastIndexedAt={actualConnector?.last_indexed_at}
|
||||
isIndexing={isIndexing}
|
||||
activeTask={activeTask}
|
||||
onConnect={handleConnect}
|
||||
|
|
@ -267,7 +253,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
isConnected={isConnected}
|
||||
isConnecting={isConnecting}
|
||||
documentCount={documentCount}
|
||||
lastIndexedAt={actualConnector?.last_indexed_at}
|
||||
isIndexing={isIndexing}
|
||||
activeTask={activeTask}
|
||||
onConnect={handleConnect}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue