feat: Implement real-time indexing state management for connectors

- Introduced a new custom hook, useIndexingConnectors, to track indexing states of connectors without polling.
- Updated ConnectorIndicator and related components to utilize the new indexing state management, enhancing user experience with immediate feedback during indexing.
- Removed deprecated log summary references and replaced them with real-time updates from Electric SQL.
- Adjusted UI components to reflect changes in indexing status, improving clarity and responsiveness.
This commit is contained in:
Anish Sarkar 2026-01-13 20:53:30 +05:30
parent 48b67d9bc1
commit f250fa177a
7 changed files with 156 additions and 124 deletions

View file

@ -3,17 +3,14 @@
import { useAtomValue } from "jotai";
import { Cable, Loader2 } from "lucide-react";
import { useSearchParams } from "next/navigation";
import { type FC, useMemo } from "react";
import type { FC } from "react";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
import { Dialog, DialogContent } from "@/components/ui/dialog";
import { Tabs, TabsContent } from "@/components/ui/tabs";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { useLogsSummary } from "@/hooks/use-logs";
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { cn } from "@/lib/utils";
import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header";
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
@ -21,6 +18,7 @@ import { ConnectorEditView } from "./connector-popup/connector-configs/views/con
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants";
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-connectors";
import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab";
import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab";
import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view";
@ -36,12 +34,6 @@ export const ConnectorIndicator: FC = () => {
// Check if YouTube view is active
const isYouTubeView = searchParams.get("view") === "youtube";
// Track active indexing tasks
const { summary: logsSummary } = useLogsSummary(searchSpaceId ? Number(searchSpaceId) : 0, 24, {
enablePolling: true,
refetchInterval: 5000,
});
// Use the custom hook for dialog state management
const {
isOpen,
@ -118,17 +110,10 @@ export const ConnectorIndicator: FC = () => {
}
};
// Document type counts now update in real-time via Electric SQL - no polling needed!
// Get connector IDs that are currently being indexed
const indexingConnectorIds = useMemo(() => {
if (!logsSummary?.active_tasks) return new Set<number>();
return new Set(
logsSummary.active_tasks
.filter((task) => task.source?.includes("connector_indexing") && task.connector_id != null)
.map((task) => task.connector_id as number)
);
}, [logsSummary?.active_tasks]);
// Track indexing state locally - clears automatically when Electric SQL detects last_indexed_at changed
const { indexingConnectorIds, startIndexing } = useIndexingConnectors(
connectors as SearchSourceConnector[]
);
const isLoading = connectorsLoading || documentTypesLoading;
@ -143,8 +128,9 @@ export const ConnectorIndicator: FC = () => {
const activeConnectorsCount = connectors.length; // Only actual connectors, not document types
// Check which connectors are already connected
// Using Electric SQL + PGlite for real-time connector updates
const connectedTypes = new Set(
(allConnectors || []).map((c: SearchSourceConnector) => c.connector_type)
(connectors || []).map((c: SearchSourceConnector) => c.connector_type)
);
if (!searchSpaceId) return null;
@ -188,9 +174,8 @@ export const ConnectorIndicator: FC = () => {
<ConnectorAccountsListView
connectorType={viewingAccountsType.connectorType}
connectorTitle={viewingAccountsType.connectorTitle}
connectors={(allConnectors || []) as SearchSourceConnector[]}
connectors={(connectors || []) as SearchSourceConnector[]} // Using Electric SQL + PGlite for real-time connector updates (all connector types)
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onBack={handleBackFromAccountsList}
onManage={handleStartEdit}
onAddAccount={() => {
@ -228,13 +213,18 @@ export const ConnectorIndicator: FC = () => {
onEndDateChange={setEndDate}
onPeriodicEnabledChange={setPeriodicEnabled}
onFrequencyChange={setFrequencyMinutes}
onSave={() => handleSaveConnector(() => refreshConnectors())}
onSave={() => {
startIndexing(editingConnector.id);
handleSaveConnector(() => refreshConnectors());
}}
onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())}
onBack={handleBackFromEdit}
onQuickIndex={
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
? () =>
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type)
? () => {
startIndexing(editingConnector.id);
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type);
}
: undefined
}
onConfigChange={setConnectorConfig}
@ -261,7 +251,12 @@ export const ConnectorIndicator: FC = () => {
onPeriodicEnabledChange={setPeriodicEnabled}
onFrequencyChange={setFrequencyMinutes}
onConfigChange={setIndexingConnectorConfig}
onStartIndexing={() => handleStartIndexing(() => refreshConnectors())}
onStartIndexing={() => {
if (indexingConfig.connectorId) {
startIndexing(indexingConfig.connectorId);
}
handleStartIndexing(() => refreshConnectors());
}}
onSkip={handleSkipIndexing}
/>
) : (
@ -290,10 +285,9 @@ export const ConnectorIndicator: FC = () => {
searchSpaceId={searchSpaceId}
connectedTypes={connectedTypes}
connectingId={connectingId}
allConnectors={allConnectors}
allConnectors={connectors}
documentTypeCounts={documentTypeCounts}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onConnectOAuth={handleConnectOAuth}
onConnectNonOAuth={handleConnectNonOAuth}
onCreateWebcrawler={handleCreateWebcrawler}
@ -310,7 +304,6 @@ export const ConnectorIndicator: FC = () => {
activeDocumentTypes={activeDocumentTypes}
connectors={connectors as SearchSourceConnector[]}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
searchSpaceId={searchSpaceId}
onTabChange={handleTabChange}
onManage={handleStartEdit}

View file

@ -5,7 +5,6 @@ import { FileText, Loader2 } from "lucide-react";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { LogActiveTask } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { useConnectorStatus } from "../hooks/use-connector-status";
import { ConnectorStatusBadge } from "./connector-status-badge";
@ -20,22 +19,10 @@ interface ConnectorCardProps {
documentCount?: number;
accountCount?: number;
isIndexing?: boolean;
activeTask?: LogActiveTask;
onConnect?: () => void;
onManage?: () => void;
}
/**
* Extract a number from the active task message for display
* Looks for patterns like "45 indexed", "Processing 123", etc.
*/
function extractIndexedCount(message: string | undefined): number | null {
if (!message) return null;
// Try to find a number in the message
const match = message.match(/(\d+)/);
return match ? parseInt(match[1], 10) : null;
}
/**
* Format document count (e.g., "1.2k docs", "500 docs", "1.5M docs")
*/
@ -60,7 +47,6 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
documentCount,
accountCount,
isIndexing = false,
activeTask,
onConnect,
onManage,
}) => {
@ -73,32 +59,29 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
const statusMessage = getConnectorStatusMessage(connectorType);
const showWarnings = shouldShowWarnings();
// Extract count from active task message during indexing
const indexingCount = extractIndexedCount(activeTask?.message);
// Determine the status content to display
const getStatusContent = () => {
if (isIndexing) {
return (
<div className="flex items-center gap-2 w-full max-w-[200px]">
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
{indexingCount !== null ? <>{indexingCount.toLocaleString()} indexed</> : "Syncing..."}
</span>
{/* Indeterminate progress bar with animation */}
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
<div className="absolute h-full bg-primary rounded-full animate-progress-indeterminate" />
</div>
// Determine the status content to display
const getStatusContent = () => {
if (isIndexing) {
return (
<div className="flex items-center gap-2 w-full max-w-[200px]">
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
Syncing...
</span>
{/* Indeterminate progress bar with animation */}
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
<div className="absolute h-full bg-primary rounded-full animate-progress-indeterminate" />
</div>
);
}
</div>
);
}
if (isConnected) {
// Don't show last indexed in overview tabs - only show in accounts list view
return null;
}
if (isConnected) {
// Don't show last indexed in overview tabs - only show in accounts list view
return null;
}
return description;
};
return description;
};
const cardContent = (
<div

View file

@ -0,0 +1,82 @@
"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
/**
* Hook to track which connectors are currently indexing using local state.
*
* This provides a better UX than polling by:
* 1. Setting indexing state immediately when user triggers indexing (optimistic)
* 2. Clearing indexing state when Electric SQL detects last_indexed_at changed
*
* The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state.
*/
export function useIndexingConnectors(connectors: SearchSourceConnector[]) {
// Set of connector IDs that are currently indexing
const [indexingConnectorIds, setIndexingConnectorIds] = useState<Set<number>>(new Set());
// Track previous last_indexed_at values to detect changes
const previousLastIndexedAtRef = useRef<Map<number, string | null>>(new Map());
// Detect when last_indexed_at changes (indexing completed) via Electric SQL
useEffect(() => {
const previousValues = previousLastIndexedAtRef.current;
const newIndexingIds = new Set(indexingConnectorIds);
let hasChanges = false;
for (const connector of connectors) {
const previousValue = previousValues.get(connector.id);
const currentValue = connector.last_indexed_at;
// If last_indexed_at changed and connector was in indexing state, clear it
if (
previousValue !== undefined && // We've seen this connector before
previousValue !== currentValue && // Value changed
indexingConnectorIds.has(connector.id) // It was marked as indexing
) {
newIndexingIds.delete(connector.id);
hasChanges = true;
}
// Update previous value tracking
previousValues.set(connector.id, currentValue);
}
if (hasChanges) {
setIndexingConnectorIds(newIndexingIds);
}
}, [connectors, indexingConnectorIds]);
// Add a connector to the indexing set (called when indexing starts)
const startIndexing = useCallback((connectorId: number) => {
setIndexingConnectorIds((prev) => {
const next = new Set(prev);
next.add(connectorId);
return next;
});
}, []);
// Remove a connector from the indexing set (called manually if needed)
const stopIndexing = useCallback((connectorId: number) => {
setIndexingConnectorIds((prev) => {
const next = new Set(prev);
next.delete(connectorId);
return next;
});
}, []);
// Check if a connector is currently indexing
const isIndexing = useCallback(
(connectorId: number) => indexingConnectorIds.has(connectorId),
[indexingConnectorIds]
);
return {
indexingConnectorIds,
startIndexing,
stopIndexing,
isIndexing,
};
}

View file

@ -8,7 +8,6 @@ import { Button } from "@/components/ui/button";
import { TabsContent } from "@/components/ui/tabs";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { OAUTH_CONNECTORS } from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
@ -20,7 +19,6 @@ interface ActiveConnectorsTabProps {
activeDocumentTypes: Array<[string, number]>;
connectors: SearchSourceConnector[];
indexingConnectorIds: Set<number>;
logsSummary: LogSummary | undefined;
searchSpaceId: string;
onTabChange: (value: string) => void;
onManage?: (connector: SearchSourceConnector) => void;
@ -33,7 +31,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
activeDocumentTypes,
connectors,
indexingConnectorIds,
logsSummary,
searchSpaceId,
onTabChange,
onManage,
@ -224,9 +221,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
{/* Non-OAuth Connectors - Individual Cards */}
{filteredNonOAuthConnectors.map((connector) => {
const isIndexing = indexingConnectorIds.has(connector.id);
const activeTask = logsSummary?.active_tasks?.find(
(task: LogActiveTask) => task.connector_id === connector.id
);
const documentCount = getDocumentCountForConnector(
connector.connector_type,
documentTypeCounts
@ -259,12 +253,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
{isIndexing && (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
{activeTask?.message && (
<span className="text-muted-foreground truncate max-w-[150px]">
{activeTask.message}
</span>
)}
Syncing...
</p>
)}
<p className="text-[10px] text-muted-foreground mt-1">

View file

@ -1,10 +1,7 @@
"use client";
import { Plus } from "lucide-react";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { ConnectorCard } from "../components/connector-card";
import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
@ -30,7 +27,6 @@ interface AllConnectorsTabProps {
allConnectors: SearchSourceConnector[] | undefined;
documentTypeCounts?: Record<string, number>;
indexingConnectorIds?: Set<number>;
logsSummary?: LogSummary;
onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void;
onConnectNonOAuth?: (connectorType: string) => void;
onCreateWebcrawler?: () => void;
@ -41,13 +37,11 @@ interface AllConnectorsTabProps {
export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
searchQuery,
searchSpaceId,
connectedTypes,
connectingId,
allConnectors,
documentTypeCounts,
indexingConnectorIds,
logsSummary,
onConnectOAuth,
onConnectNonOAuth,
onCreateWebcrawler,
@ -55,13 +49,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
onManage,
onViewAccountsList,
}) => {
// Helper to find active task for a connector
const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => {
if (!logsSummary?.active_tasks) return undefined;
return logsSummary.active_tasks.find(
(task: LogActiveTask) => task.connector_id === connectorId
);
};
// Filter connectors based on search
const filteredOAuth = OAUTH_CONNECTORS.filter(
@ -111,11 +98,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
// Check if any account is currently indexing
const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id));
// Get active task from any indexing account
const activeTask = typeConnectors
.map((c) => getActiveTaskForConnector(c.id))
.find((task) => task !== undefined);
return (
<ConnectorCard
key={connector.id}
@ -128,7 +110,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
documentCount={documentCount}
accountCount={typeConnectors.length}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={() => onConnectOAuth(connector)}
onManage={
isConnected && onViewAccountsList
@ -166,9 +147,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
documentTypeCounts
);
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
const activeTask = actualConnector
? getActiveTaskForConnector(actualConnector.id)
: undefined;
const handleConnect = onConnectNonOAuth
? () => onConnectNonOAuth(connector.connectorType)
@ -185,7 +163,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
isConnecting={isConnecting}
documentCount={documentCount}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={handleConnect}
onManage={
actualConnector && onManage ? () => onManage(actualConnector) : undefined
@ -226,9 +203,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
? getDocumentCountForConnector(crawler.connectorType, documentTypeCounts)
: undefined;
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
const activeTask = actualConnector
? getActiveTaskForConnector(actualConnector.id)
: undefined;
const handleConnect =
isYouTube && onCreateYouTubeCrawler
@ -254,7 +228,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
isConnecting={isConnecting}
documentCount={documentCount}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={handleConnect}
onManage={
actualConnector && onManage ? () => onManage(actualConnector) : undefined

View file

@ -6,7 +6,6 @@ import type { FC } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { useConnectorStatus } from "../hooks/use-connector-status";
import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
@ -16,7 +15,6 @@ interface ConnectorAccountsListViewProps {
connectorTitle: string;
connectors: SearchSourceConnector[];
indexingConnectorIds: Set<number>;
logsSummary: LogSummary | undefined;
onBack: () => void;
onManage: (connector: SearchSourceConnector) => void;
onAddAccount: () => void;
@ -60,7 +58,6 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
connectorTitle,
connectors,
indexingConnectorIds,
logsSummary,
onBack,
onManage,
onAddAccount,
@ -137,9 +134,6 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
{typeConnectors.map((connector) => {
const isIndexing = indexingConnectorIds.has(connector.id);
const activeTask = logsSummary?.active_tasks?.find(
(task: LogActiveTask) => task.connector_id === connector.id
);
return (
<div
@ -168,12 +162,7 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
{isIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
{activeTask?.message && (
<span className="text-muted-foreground truncate max-w-[100px]">
{activeTask.message}
</span>
)}
Syncing...
</p>
) : (
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap truncate">