diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 883ac63c5..86c0cf59e 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -3,17 +3,14 @@ import { useAtomValue } from "jotai"; import { Cable, Loader2 } from "lucide-react"; import { useSearchParams } from "next/navigation"; -import { type FC, useMemo } from "react"; +import type { FC } from "react"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { Dialog, DialogContent } from "@/components/ui/dialog"; import { Tabs, TabsContent } from "@/components/ui/tabs"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { useLogsSummary } from "@/hooks/use-logs"; import { useConnectorsElectric } from "@/hooks/use-connectors-electric"; import { useDocumentsElectric } from "@/hooks/use-documents-electric"; -import { connectorsApiService } from "@/lib/apis/connectors-api.service"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; import { cn } from "@/lib/utils"; import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header"; import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view"; @@ -21,6 +18,7 @@ import { ConnectorEditView } from "./connector-popup/connector-configs/views/con import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view"; import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants"; import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog"; +import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-connectors"; import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab"; import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab"; import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view"; @@ -36,12 +34,6 @@ export const ConnectorIndicator: FC = () => { // Check if YouTube view is active const isYouTubeView = searchParams.get("view") === "youtube"; - // Track active indexing tasks - const { summary: logsSummary } = useLogsSummary(searchSpaceId ? Number(searchSpaceId) : 0, 24, { - enablePolling: true, - refetchInterval: 5000, - }); - // Use the custom hook for dialog state management const { isOpen, @@ -118,17 +110,10 @@ export const ConnectorIndicator: FC = () => { } }; - // Document type counts now update in real-time via Electric SQL - no polling needed! - - // Get connector IDs that are currently being indexed - const indexingConnectorIds = useMemo(() => { - if (!logsSummary?.active_tasks) return new Set(); - return new Set( - logsSummary.active_tasks - .filter((task) => task.source?.includes("connector_indexing") && task.connector_id != null) - .map((task) => task.connector_id as number) - ); - }, [logsSummary?.active_tasks]); + // Track indexing state locally - clears automatically when Electric SQL detects last_indexed_at changed + const { indexingConnectorIds, startIndexing } = useIndexingConnectors( + connectors as SearchSourceConnector[] + ); const isLoading = connectorsLoading || documentTypesLoading; @@ -143,8 +128,9 @@ export const ConnectorIndicator: FC = () => { const activeConnectorsCount = connectors.length; // Only actual connectors, not document types // Check which connectors are already connected + // Using Electric SQL + PGlite for real-time connector updates const connectedTypes = new Set( - (allConnectors || []).map((c: SearchSourceConnector) => c.connector_type) + (connectors || []).map((c: SearchSourceConnector) => c.connector_type) ); if (!searchSpaceId) return null; @@ -188,9 +174,8 @@ export const ConnectorIndicator: FC = () => { { @@ -228,13 +213,18 @@ export const ConnectorIndicator: FC = () => { onEndDateChange={setEndDate} onPeriodicEnabledChange={setPeriodicEnabled} onFrequencyChange={setFrequencyMinutes} - onSave={() => handleSaveConnector(() => refreshConnectors())} + onSave={() => { + startIndexing(editingConnector.id); + handleSaveConnector(() => refreshConnectors()); + }} onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())} onBack={handleBackFromEdit} onQuickIndex={ editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" - ? () => - handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type) + ? () => { + startIndexing(editingConnector.id); + handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type); + } : undefined } onConfigChange={setConnectorConfig} @@ -261,7 +251,12 @@ export const ConnectorIndicator: FC = () => { onPeriodicEnabledChange={setPeriodicEnabled} onFrequencyChange={setFrequencyMinutes} onConfigChange={setIndexingConnectorConfig} - onStartIndexing={() => handleStartIndexing(() => refreshConnectors())} + onStartIndexing={() => { + if (indexingConfig.connectorId) { + startIndexing(indexingConfig.connectorId); + } + handleStartIndexing(() => refreshConnectors()); + }} onSkip={handleSkipIndexing} /> ) : ( @@ -290,10 +285,9 @@ export const ConnectorIndicator: FC = () => { searchSpaceId={searchSpaceId} connectedTypes={connectedTypes} connectingId={connectingId} - allConnectors={allConnectors} + allConnectors={connectors} documentTypeCounts={documentTypeCounts} indexingConnectorIds={indexingConnectorIds} - logsSummary={logsSummary} onConnectOAuth={handleConnectOAuth} onConnectNonOAuth={handleConnectNonOAuth} onCreateWebcrawler={handleCreateWebcrawler} @@ -310,7 +304,6 @@ export const ConnectorIndicator: FC = () => { activeDocumentTypes={activeDocumentTypes} connectors={connectors as SearchSourceConnector[]} indexingConnectorIds={indexingConnectorIds} - logsSummary={logsSummary} searchSpaceId={searchSpaceId} onTabChange={handleTabChange} onManage={handleStartEdit} diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx index 27c9608ef..cd8e2cf99 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx @@ -5,7 +5,6 @@ import { FileText, Loader2 } from "lucide-react"; import type { FC } from "react"; import { Button } from "@/components/ui/button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { LogActiveTask } from "@/contracts/types/log.types"; import { cn } from "@/lib/utils"; import { useConnectorStatus } from "../hooks/use-connector-status"; import { ConnectorStatusBadge } from "./connector-status-badge"; @@ -20,22 +19,10 @@ interface ConnectorCardProps { documentCount?: number; accountCount?: number; isIndexing?: boolean; - activeTask?: LogActiveTask; onConnect?: () => void; onManage?: () => void; } -/** - * Extract a number from the active task message for display - * Looks for patterns like "45 indexed", "Processing 123", etc. - */ -function extractIndexedCount(message: string | undefined): number | null { - if (!message) return null; - // Try to find a number in the message - const match = message.match(/(\d+)/); - return match ? parseInt(match[1], 10) : null; -} - /** * Format document count (e.g., "1.2k docs", "500 docs", "1.5M docs") */ @@ -60,7 +47,6 @@ export const ConnectorCard: FC = ({ documentCount, accountCount, isIndexing = false, - activeTask, onConnect, onManage, }) => { @@ -73,32 +59,29 @@ export const ConnectorCard: FC = ({ const statusMessage = getConnectorStatusMessage(connectorType); const showWarnings = shouldShowWarnings(); - // Extract count from active task message during indexing - const indexingCount = extractIndexedCount(activeTask?.message); - - // Determine the status content to display - const getStatusContent = () => { - if (isIndexing) { - return ( -
- - {indexingCount !== null ? <>{indexingCount.toLocaleString()} indexed : "Syncing..."} - - {/* Indeterminate progress bar with animation */} -
-
-
+ // Determine the status content to display + const getStatusContent = () => { + if (isIndexing) { + return ( +
+ + Syncing... + + {/* Indeterminate progress bar with animation */} +
+
- ); - } +
+ ); + } - if (isConnected) { - // Don't show last indexed in overview tabs - only show in accounts list view - return null; - } + if (isConnected) { + // Don't show last indexed in overview tabs - only show in accounts list view + return null; + } - return description; - }; + return description; + }; const cardContent = (
>(new Set()); + + // Track previous last_indexed_at values to detect changes + const previousLastIndexedAtRef = useRef>(new Map()); + + // Detect when last_indexed_at changes (indexing completed) via Electric SQL + useEffect(() => { + const previousValues = previousLastIndexedAtRef.current; + const newIndexingIds = new Set(indexingConnectorIds); + let hasChanges = false; + + for (const connector of connectors) { + const previousValue = previousValues.get(connector.id); + const currentValue = connector.last_indexed_at; + + // If last_indexed_at changed and connector was in indexing state, clear it + if ( + previousValue !== undefined && // We've seen this connector before + previousValue !== currentValue && // Value changed + indexingConnectorIds.has(connector.id) // It was marked as indexing + ) { + newIndexingIds.delete(connector.id); + hasChanges = true; + } + + // Update previous value tracking + previousValues.set(connector.id, currentValue); + } + + if (hasChanges) { + setIndexingConnectorIds(newIndexingIds); + } + }, [connectors, indexingConnectorIds]); + + // Add a connector to the indexing set (called when indexing starts) + const startIndexing = useCallback((connectorId: number) => { + setIndexingConnectorIds((prev) => { + const next = new Set(prev); + next.add(connectorId); + return next; + }); + }, []); + + // Remove a connector from the indexing set (called manually if needed) + const stopIndexing = useCallback((connectorId: number) => { + setIndexingConnectorIds((prev) => { + const next = new Set(prev); + next.delete(connectorId); + return next; + }); + }, []); + + // Check if a connector is currently indexing + const isIndexing = useCallback( + (connectorId: number) => indexingConnectorIds.has(connectorId), + [indexingConnectorIds] + ); + + return { + indexingConnectorIds, + startIndexing, + stopIndexing, + isIndexing, + }; +} + diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx index 556995919..54ad1db0a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx @@ -8,7 +8,6 @@ import { Button } from "@/components/ui/button"; import { TabsContent } from "@/components/ui/tabs"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types"; import { cn } from "@/lib/utils"; import { OAUTH_CONNECTORS } from "../constants/connector-constants"; import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; @@ -20,7 +19,6 @@ interface ActiveConnectorsTabProps { activeDocumentTypes: Array<[string, number]>; connectors: SearchSourceConnector[]; indexingConnectorIds: Set; - logsSummary: LogSummary | undefined; searchSpaceId: string; onTabChange: (value: string) => void; onManage?: (connector: SearchSourceConnector) => void; @@ -33,7 +31,6 @@ export const ActiveConnectorsTab: FC = ({ activeDocumentTypes, connectors, indexingConnectorIds, - logsSummary, searchSpaceId, onTabChange, onManage, @@ -224,9 +221,6 @@ export const ActiveConnectorsTab: FC = ({ {/* Non-OAuth Connectors - Individual Cards */} {filteredNonOAuthConnectors.map((connector) => { const isIndexing = indexingConnectorIds.has(connector.id); - const activeTask = logsSummary?.active_tasks?.find( - (task: LogActiveTask) => task.connector_id === connector.id - ); const documentCount = getDocumentCountForConnector( connector.connector_type, documentTypeCounts @@ -259,12 +253,7 @@ export const ActiveConnectorsTab: FC = ({ {isIndexing && (

- Indexing... - {activeTask?.message && ( - - • {activeTask.message} - - )} + Syncing...

)}

diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index e596c9faf..721b66f01 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -1,10 +1,7 @@ "use client"; -import { Plus } from "lucide-react"; import type { FC } from "react"; -import { Button } from "@/components/ui/button"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types"; import { ConnectorCard } from "../components/connector-card"; import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; @@ -30,7 +27,6 @@ interface AllConnectorsTabProps { allConnectors: SearchSourceConnector[] | undefined; documentTypeCounts?: Record; indexingConnectorIds?: Set; - logsSummary?: LogSummary; onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void; onConnectNonOAuth?: (connectorType: string) => void; onCreateWebcrawler?: () => void; @@ -41,13 +37,11 @@ interface AllConnectorsTabProps { export const AllConnectorsTab: FC = ({ searchQuery, - searchSpaceId, connectedTypes, connectingId, allConnectors, documentTypeCounts, indexingConnectorIds, - logsSummary, onConnectOAuth, onConnectNonOAuth, onCreateWebcrawler, @@ -55,13 +49,6 @@ export const AllConnectorsTab: FC = ({ onManage, onViewAccountsList, }) => { - // Helper to find active task for a connector - const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => { - if (!logsSummary?.active_tasks) return undefined; - return logsSummary.active_tasks.find( - (task: LogActiveTask) => task.connector_id === connectorId - ); - }; // Filter connectors based on search const filteredOAuth = OAUTH_CONNECTORS.filter( @@ -111,11 +98,6 @@ export const AllConnectorsTab: FC = ({ // Check if any account is currently indexing const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id)); - // Get active task from any indexing account - const activeTask = typeConnectors - .map((c) => getActiveTaskForConnector(c.id)) - .find((task) => task !== undefined); - return ( = ({ documentCount={documentCount} accountCount={typeConnectors.length} isIndexing={isIndexing} - activeTask={activeTask} onConnect={() => onConnectOAuth(connector)} onManage={ isConnected && onViewAccountsList @@ -166,9 +147,6 @@ export const AllConnectorsTab: FC = ({ documentTypeCounts ); const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id); - const activeTask = actualConnector - ? getActiveTaskForConnector(actualConnector.id) - : undefined; const handleConnect = onConnectNonOAuth ? () => onConnectNonOAuth(connector.connectorType) @@ -185,7 +163,6 @@ export const AllConnectorsTab: FC = ({ isConnecting={isConnecting} documentCount={documentCount} isIndexing={isIndexing} - activeTask={activeTask} onConnect={handleConnect} onManage={ actualConnector && onManage ? () => onManage(actualConnector) : undefined @@ -226,9 +203,6 @@ export const AllConnectorsTab: FC = ({ ? getDocumentCountForConnector(crawler.connectorType, documentTypeCounts) : undefined; const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id); - const activeTask = actualConnector - ? getActiveTaskForConnector(actualConnector.id) - : undefined; const handleConnect = isYouTube && onCreateYouTubeCrawler @@ -254,7 +228,6 @@ export const AllConnectorsTab: FC = ({ isConnecting={isConnecting} documentCount={documentCount} isIndexing={isIndexing} - activeTask={activeTask} onConnect={handleConnect} onManage={ actualConnector && onManage ? () => onManage(actualConnector) : undefined diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx index bec4bfcb8..2bbbc98e8 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx @@ -6,7 +6,6 @@ import type { FC } from "react"; import { Button } from "@/components/ui/button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types"; import { cn } from "@/lib/utils"; import { useConnectorStatus } from "../hooks/use-connector-status"; import { getConnectorDisplayName } from "../tabs/all-connectors-tab"; @@ -16,7 +15,6 @@ interface ConnectorAccountsListViewProps { connectorTitle: string; connectors: SearchSourceConnector[]; indexingConnectorIds: Set; - logsSummary: LogSummary | undefined; onBack: () => void; onManage: (connector: SearchSourceConnector) => void; onAddAccount: () => void; @@ -60,7 +58,6 @@ export const ConnectorAccountsListView: FC = ({ connectorTitle, connectors, indexingConnectorIds, - logsSummary, onBack, onManage, onAddAccount, @@ -137,9 +134,6 @@ export const ConnectorAccountsListView: FC = ({

{typeConnectors.map((connector) => { const isIndexing = indexingConnectorIds.has(connector.id); - const activeTask = logsSummary?.active_tasks?.find( - (task: LogActiveTask) => task.connector_id === connector.id - ); return (
= ({ {isIndexing ? (

- Indexing... - {activeTask?.message && ( - - • {activeTask.message} - - )} + Syncing...

) : (

diff --git a/surfsense_web/hooks/use-connectors-electric.ts b/surfsense_web/hooks/use-connectors-electric.ts index d746f74ab..50c398d48 100644 --- a/surfsense_web/hooks/use-connectors-electric.ts +++ b/surfsense_web/hooks/use-connectors-electric.ts @@ -12,6 +12,29 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) { const syncHandleRef = useRef(null) const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null) + // Transform connector data from Electric SQL/PGlite to match expected format + // Converts Date objects to ISO strings as expected by Zod schema + function transformConnector(connector: any): SearchSourceConnector { + return { + ...connector, + last_indexed_at: connector.last_indexed_at + ? typeof connector.last_indexed_at === 'string' + ? connector.last_indexed_at + : new Date(connector.last_indexed_at).toISOString() + : null, + next_scheduled_at: connector.next_scheduled_at + ? typeof connector.next_scheduled_at === 'string' + ? connector.next_scheduled_at + : new Date(connector.next_scheduled_at).toISOString() + : null, + created_at: connector.created_at + ? typeof connector.created_at === 'string' + ? connector.created_at + : new Date(connector.created_at).toISOString() + : new Date().toISOString(), // fallback + } + } + // Initialize Electric SQL and start syncing with real-time updates useEffect(() => { if (!searchSpaceId) { @@ -107,20 +130,20 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) { // Set initial results immediately from the resolved query if (liveQuery.initialResults?.rows) { console.log('📋 Initial live query results for connectors:', liveQuery.initialResults.rows.length) - setConnectors(liveQuery.initialResults.rows) + setConnectors(liveQuery.initialResults.rows.map(transformConnector)) } else if (liveQuery.rows) { // Some versions have rows directly on the result console.log('📋 Initial live query results for connectors (direct):', liveQuery.rows.length) - setConnectors(liveQuery.rows) + setConnectors(liveQuery.rows.map(transformConnector)) } // Subscribe to changes - this is the correct API! // The callback fires automatically when Electric SQL syncs new data to PGlite if (typeof liveQuery.subscribe === 'function') { - liveQuery.subscribe((result: { rows: SearchSourceConnector[] }) => { + liveQuery.subscribe((result: { rows: any[] }) => { if (mounted && result.rows) { console.log('🔄 Connectors updated via live query:', result.rows.length) - setConnectors(result.rows) + setConnectors(result.rows.map(transformConnector)) } }) @@ -161,7 +184,7 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) { [searchSpaceId] ) console.log('📋 Fetched connectors from PGlite:', result.rows?.length || 0) - setConnectors(result.rows || []) + setConnectors((result.rows || []).map(transformConnector)) } catch (err) { console.error('Failed to fetch connectors from PGlite:', err) }