mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-06 20:15:17 +02:00
feat: Implement real-time indexing state management for connectors
- Introduced a new custom hook, useIndexingConnectors, to track indexing states of connectors without polling. - Updated ConnectorIndicator and related components to utilize the new indexing state management, enhancing user experience with immediate feedback during indexing. - Removed deprecated log summary references and replaced them with real-time updates from Electric SQL. - Adjusted UI components to reflect changes in indexing status, improving clarity and responsiveness.
This commit is contained in:
parent
48b67d9bc1
commit
f250fa177a
7 changed files with 156 additions and 124 deletions
|
|
@ -3,17 +3,14 @@
|
||||||
import { useAtomValue } from "jotai";
|
import { useAtomValue } from "jotai";
|
||||||
import { Cable, Loader2 } from "lucide-react";
|
import { Cable, Loader2 } from "lucide-react";
|
||||||
import { useSearchParams } from "next/navigation";
|
import { useSearchParams } from "next/navigation";
|
||||||
import { type FC, useMemo } from "react";
|
import type { FC } from "react";
|
||||||
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
|
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
|
||||||
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
|
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
|
||||||
import { Dialog, DialogContent } from "@/components/ui/dialog";
|
import { Dialog, DialogContent } from "@/components/ui/dialog";
|
||||||
import { Tabs, TabsContent } from "@/components/ui/tabs";
|
import { Tabs, TabsContent } from "@/components/ui/tabs";
|
||||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
import { useLogsSummary } from "@/hooks/use-logs";
|
|
||||||
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
|
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
|
||||||
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
|
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
|
||||||
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
|
|
||||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header";
|
import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header";
|
||||||
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
|
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
|
||||||
|
|
@ -21,6 +18,7 @@ import { ConnectorEditView } from "./connector-popup/connector-configs/views/con
|
||||||
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
|
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
|
||||||
import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants";
|
import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants";
|
||||||
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
|
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
|
||||||
|
import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-connectors";
|
||||||
import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab";
|
import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab";
|
||||||
import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab";
|
import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab";
|
||||||
import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view";
|
import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view";
|
||||||
|
|
@ -36,12 +34,6 @@ export const ConnectorIndicator: FC = () => {
|
||||||
// Check if YouTube view is active
|
// Check if YouTube view is active
|
||||||
const isYouTubeView = searchParams.get("view") === "youtube";
|
const isYouTubeView = searchParams.get("view") === "youtube";
|
||||||
|
|
||||||
// Track active indexing tasks
|
|
||||||
const { summary: logsSummary } = useLogsSummary(searchSpaceId ? Number(searchSpaceId) : 0, 24, {
|
|
||||||
enablePolling: true,
|
|
||||||
refetchInterval: 5000,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Use the custom hook for dialog state management
|
// Use the custom hook for dialog state management
|
||||||
const {
|
const {
|
||||||
isOpen,
|
isOpen,
|
||||||
|
|
@ -118,17 +110,10 @@ export const ConnectorIndicator: FC = () => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Document type counts now update in real-time via Electric SQL - no polling needed!
|
// Track indexing state locally - clears automatically when Electric SQL detects last_indexed_at changed
|
||||||
|
const { indexingConnectorIds, startIndexing } = useIndexingConnectors(
|
||||||
// Get connector IDs that are currently being indexed
|
connectors as SearchSourceConnector[]
|
||||||
const indexingConnectorIds = useMemo(() => {
|
|
||||||
if (!logsSummary?.active_tasks) return new Set<number>();
|
|
||||||
return new Set(
|
|
||||||
logsSummary.active_tasks
|
|
||||||
.filter((task) => task.source?.includes("connector_indexing") && task.connector_id != null)
|
|
||||||
.map((task) => task.connector_id as number)
|
|
||||||
);
|
);
|
||||||
}, [logsSummary?.active_tasks]);
|
|
||||||
|
|
||||||
const isLoading = connectorsLoading || documentTypesLoading;
|
const isLoading = connectorsLoading || documentTypesLoading;
|
||||||
|
|
||||||
|
|
@ -143,8 +128,9 @@ export const ConnectorIndicator: FC = () => {
|
||||||
const activeConnectorsCount = connectors.length; // Only actual connectors, not document types
|
const activeConnectorsCount = connectors.length; // Only actual connectors, not document types
|
||||||
|
|
||||||
// Check which connectors are already connected
|
// Check which connectors are already connected
|
||||||
|
// Using Electric SQL + PGlite for real-time connector updates
|
||||||
const connectedTypes = new Set(
|
const connectedTypes = new Set(
|
||||||
(allConnectors || []).map((c: SearchSourceConnector) => c.connector_type)
|
(connectors || []).map((c: SearchSourceConnector) => c.connector_type)
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!searchSpaceId) return null;
|
if (!searchSpaceId) return null;
|
||||||
|
|
@ -188,9 +174,8 @@ export const ConnectorIndicator: FC = () => {
|
||||||
<ConnectorAccountsListView
|
<ConnectorAccountsListView
|
||||||
connectorType={viewingAccountsType.connectorType}
|
connectorType={viewingAccountsType.connectorType}
|
||||||
connectorTitle={viewingAccountsType.connectorTitle}
|
connectorTitle={viewingAccountsType.connectorTitle}
|
||||||
connectors={(allConnectors || []) as SearchSourceConnector[]}
|
connectors={(connectors || []) as SearchSourceConnector[]} // Using Electric SQL + PGlite for real-time connector updates (all connector types)
|
||||||
indexingConnectorIds={indexingConnectorIds}
|
indexingConnectorIds={indexingConnectorIds}
|
||||||
logsSummary={logsSummary}
|
|
||||||
onBack={handleBackFromAccountsList}
|
onBack={handleBackFromAccountsList}
|
||||||
onManage={handleStartEdit}
|
onManage={handleStartEdit}
|
||||||
onAddAccount={() => {
|
onAddAccount={() => {
|
||||||
|
|
@ -228,13 +213,18 @@ export const ConnectorIndicator: FC = () => {
|
||||||
onEndDateChange={setEndDate}
|
onEndDateChange={setEndDate}
|
||||||
onPeriodicEnabledChange={setPeriodicEnabled}
|
onPeriodicEnabledChange={setPeriodicEnabled}
|
||||||
onFrequencyChange={setFrequencyMinutes}
|
onFrequencyChange={setFrequencyMinutes}
|
||||||
onSave={() => handleSaveConnector(() => refreshConnectors())}
|
onSave={() => {
|
||||||
|
startIndexing(editingConnector.id);
|
||||||
|
handleSaveConnector(() => refreshConnectors());
|
||||||
|
}}
|
||||||
onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())}
|
onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())}
|
||||||
onBack={handleBackFromEdit}
|
onBack={handleBackFromEdit}
|
||||||
onQuickIndex={
|
onQuickIndex={
|
||||||
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
|
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
|
||||||
? () =>
|
? () => {
|
||||||
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type)
|
startIndexing(editingConnector.id);
|
||||||
|
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type);
|
||||||
|
}
|
||||||
: undefined
|
: undefined
|
||||||
}
|
}
|
||||||
onConfigChange={setConnectorConfig}
|
onConfigChange={setConnectorConfig}
|
||||||
|
|
@ -261,7 +251,12 @@ export const ConnectorIndicator: FC = () => {
|
||||||
onPeriodicEnabledChange={setPeriodicEnabled}
|
onPeriodicEnabledChange={setPeriodicEnabled}
|
||||||
onFrequencyChange={setFrequencyMinutes}
|
onFrequencyChange={setFrequencyMinutes}
|
||||||
onConfigChange={setIndexingConnectorConfig}
|
onConfigChange={setIndexingConnectorConfig}
|
||||||
onStartIndexing={() => handleStartIndexing(() => refreshConnectors())}
|
onStartIndexing={() => {
|
||||||
|
if (indexingConfig.connectorId) {
|
||||||
|
startIndexing(indexingConfig.connectorId);
|
||||||
|
}
|
||||||
|
handleStartIndexing(() => refreshConnectors());
|
||||||
|
}}
|
||||||
onSkip={handleSkipIndexing}
|
onSkip={handleSkipIndexing}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
|
|
@ -290,10 +285,9 @@ export const ConnectorIndicator: FC = () => {
|
||||||
searchSpaceId={searchSpaceId}
|
searchSpaceId={searchSpaceId}
|
||||||
connectedTypes={connectedTypes}
|
connectedTypes={connectedTypes}
|
||||||
connectingId={connectingId}
|
connectingId={connectingId}
|
||||||
allConnectors={allConnectors}
|
allConnectors={connectors}
|
||||||
documentTypeCounts={documentTypeCounts}
|
documentTypeCounts={documentTypeCounts}
|
||||||
indexingConnectorIds={indexingConnectorIds}
|
indexingConnectorIds={indexingConnectorIds}
|
||||||
logsSummary={logsSummary}
|
|
||||||
onConnectOAuth={handleConnectOAuth}
|
onConnectOAuth={handleConnectOAuth}
|
||||||
onConnectNonOAuth={handleConnectNonOAuth}
|
onConnectNonOAuth={handleConnectNonOAuth}
|
||||||
onCreateWebcrawler={handleCreateWebcrawler}
|
onCreateWebcrawler={handleCreateWebcrawler}
|
||||||
|
|
@ -310,7 +304,6 @@ export const ConnectorIndicator: FC = () => {
|
||||||
activeDocumentTypes={activeDocumentTypes}
|
activeDocumentTypes={activeDocumentTypes}
|
||||||
connectors={connectors as SearchSourceConnector[]}
|
connectors={connectors as SearchSourceConnector[]}
|
||||||
indexingConnectorIds={indexingConnectorIds}
|
indexingConnectorIds={indexingConnectorIds}
|
||||||
logsSummary={logsSummary}
|
|
||||||
searchSpaceId={searchSpaceId}
|
searchSpaceId={searchSpaceId}
|
||||||
onTabChange={handleTabChange}
|
onTabChange={handleTabChange}
|
||||||
onManage={handleStartEdit}
|
onManage={handleStartEdit}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ import { FileText, Loader2 } from "lucide-react";
|
||||||
import type { FC } from "react";
|
import type { FC } from "react";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||||
import type { LogActiveTask } from "@/contracts/types/log.types";
|
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
import { useConnectorStatus } from "../hooks/use-connector-status";
|
import { useConnectorStatus } from "../hooks/use-connector-status";
|
||||||
import { ConnectorStatusBadge } from "./connector-status-badge";
|
import { ConnectorStatusBadge } from "./connector-status-badge";
|
||||||
|
|
@ -20,22 +19,10 @@ interface ConnectorCardProps {
|
||||||
documentCount?: number;
|
documentCount?: number;
|
||||||
accountCount?: number;
|
accountCount?: number;
|
||||||
isIndexing?: boolean;
|
isIndexing?: boolean;
|
||||||
activeTask?: LogActiveTask;
|
|
||||||
onConnect?: () => void;
|
onConnect?: () => void;
|
||||||
onManage?: () => void;
|
onManage?: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract a number from the active task message for display
|
|
||||||
* Looks for patterns like "45 indexed", "Processing 123", etc.
|
|
||||||
*/
|
|
||||||
function extractIndexedCount(message: string | undefined): number | null {
|
|
||||||
if (!message) return null;
|
|
||||||
// Try to find a number in the message
|
|
||||||
const match = message.match(/(\d+)/);
|
|
||||||
return match ? parseInt(match[1], 10) : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Format document count (e.g., "1.2k docs", "500 docs", "1.5M docs")
|
* Format document count (e.g., "1.2k docs", "500 docs", "1.5M docs")
|
||||||
*/
|
*/
|
||||||
|
|
@ -60,7 +47,6 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
|
||||||
documentCount,
|
documentCount,
|
||||||
accountCount,
|
accountCount,
|
||||||
isIndexing = false,
|
isIndexing = false,
|
||||||
activeTask,
|
|
||||||
onConnect,
|
onConnect,
|
||||||
onManage,
|
onManage,
|
||||||
}) => {
|
}) => {
|
||||||
|
|
@ -73,16 +59,13 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
|
||||||
const statusMessage = getConnectorStatusMessage(connectorType);
|
const statusMessage = getConnectorStatusMessage(connectorType);
|
||||||
const showWarnings = shouldShowWarnings();
|
const showWarnings = shouldShowWarnings();
|
||||||
|
|
||||||
// Extract count from active task message during indexing
|
|
||||||
const indexingCount = extractIndexedCount(activeTask?.message);
|
|
||||||
|
|
||||||
// Determine the status content to display
|
// Determine the status content to display
|
||||||
const getStatusContent = () => {
|
const getStatusContent = () => {
|
||||||
if (isIndexing) {
|
if (isIndexing) {
|
||||||
return (
|
return (
|
||||||
<div className="flex items-center gap-2 w-full max-w-[200px]">
|
<div className="flex items-center gap-2 w-full max-w-[200px]">
|
||||||
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
|
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
|
||||||
{indexingCount !== null ? <>{indexingCount.toLocaleString()} indexed</> : "Syncing..."}
|
Syncing...
|
||||||
</span>
|
</span>
|
||||||
{/* Indeterminate progress bar with animation */}
|
{/* Indeterminate progress bar with animation */}
|
||||||
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
|
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,82 @@
|
||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hook to track which connectors are currently indexing using local state.
|
||||||
|
*
|
||||||
|
* This provides a better UX than polling by:
|
||||||
|
* 1. Setting indexing state immediately when user triggers indexing (optimistic)
|
||||||
|
* 2. Clearing indexing state when Electric SQL detects last_indexed_at changed
|
||||||
|
*
|
||||||
|
* The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state.
|
||||||
|
*/
|
||||||
|
export function useIndexingConnectors(connectors: SearchSourceConnector[]) {
|
||||||
|
// Set of connector IDs that are currently indexing
|
||||||
|
const [indexingConnectorIds, setIndexingConnectorIds] = useState<Set<number>>(new Set());
|
||||||
|
|
||||||
|
// Track previous last_indexed_at values to detect changes
|
||||||
|
const previousLastIndexedAtRef = useRef<Map<number, string | null>>(new Map());
|
||||||
|
|
||||||
|
// Detect when last_indexed_at changes (indexing completed) via Electric SQL
|
||||||
|
useEffect(() => {
|
||||||
|
const previousValues = previousLastIndexedAtRef.current;
|
||||||
|
const newIndexingIds = new Set(indexingConnectorIds);
|
||||||
|
let hasChanges = false;
|
||||||
|
|
||||||
|
for (const connector of connectors) {
|
||||||
|
const previousValue = previousValues.get(connector.id);
|
||||||
|
const currentValue = connector.last_indexed_at;
|
||||||
|
|
||||||
|
// If last_indexed_at changed and connector was in indexing state, clear it
|
||||||
|
if (
|
||||||
|
previousValue !== undefined && // We've seen this connector before
|
||||||
|
previousValue !== currentValue && // Value changed
|
||||||
|
indexingConnectorIds.has(connector.id) // It was marked as indexing
|
||||||
|
) {
|
||||||
|
newIndexingIds.delete(connector.id);
|
||||||
|
hasChanges = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update previous value tracking
|
||||||
|
previousValues.set(connector.id, currentValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasChanges) {
|
||||||
|
setIndexingConnectorIds(newIndexingIds);
|
||||||
|
}
|
||||||
|
}, [connectors, indexingConnectorIds]);
|
||||||
|
|
||||||
|
// Add a connector to the indexing set (called when indexing starts)
|
||||||
|
const startIndexing = useCallback((connectorId: number) => {
|
||||||
|
setIndexingConnectorIds((prev) => {
|
||||||
|
const next = new Set(prev);
|
||||||
|
next.add(connectorId);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Remove a connector from the indexing set (called manually if needed)
|
||||||
|
const stopIndexing = useCallback((connectorId: number) => {
|
||||||
|
setIndexingConnectorIds((prev) => {
|
||||||
|
const next = new Set(prev);
|
||||||
|
next.delete(connectorId);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Check if a connector is currently indexing
|
||||||
|
const isIndexing = useCallback(
|
||||||
|
(connectorId: number) => indexingConnectorIds.has(connectorId),
|
||||||
|
[indexingConnectorIds]
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
indexingConnectorIds,
|
||||||
|
startIndexing,
|
||||||
|
stopIndexing,
|
||||||
|
isIndexing,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -8,7 +8,6 @@ import { Button } from "@/components/ui/button";
|
||||||
import { TabsContent } from "@/components/ui/tabs";
|
import { TabsContent } from "@/components/ui/tabs";
|
||||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
|
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
import { OAUTH_CONNECTORS } from "../constants/connector-constants";
|
import { OAUTH_CONNECTORS } from "../constants/connector-constants";
|
||||||
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
|
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
|
||||||
|
|
@ -20,7 +19,6 @@ interface ActiveConnectorsTabProps {
|
||||||
activeDocumentTypes: Array<[string, number]>;
|
activeDocumentTypes: Array<[string, number]>;
|
||||||
connectors: SearchSourceConnector[];
|
connectors: SearchSourceConnector[];
|
||||||
indexingConnectorIds: Set<number>;
|
indexingConnectorIds: Set<number>;
|
||||||
logsSummary: LogSummary | undefined;
|
|
||||||
searchSpaceId: string;
|
searchSpaceId: string;
|
||||||
onTabChange: (value: string) => void;
|
onTabChange: (value: string) => void;
|
||||||
onManage?: (connector: SearchSourceConnector) => void;
|
onManage?: (connector: SearchSourceConnector) => void;
|
||||||
|
|
@ -33,7 +31,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
||||||
activeDocumentTypes,
|
activeDocumentTypes,
|
||||||
connectors,
|
connectors,
|
||||||
indexingConnectorIds,
|
indexingConnectorIds,
|
||||||
logsSummary,
|
|
||||||
searchSpaceId,
|
searchSpaceId,
|
||||||
onTabChange,
|
onTabChange,
|
||||||
onManage,
|
onManage,
|
||||||
|
|
@ -224,9 +221,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
||||||
{/* Non-OAuth Connectors - Individual Cards */}
|
{/* Non-OAuth Connectors - Individual Cards */}
|
||||||
{filteredNonOAuthConnectors.map((connector) => {
|
{filteredNonOAuthConnectors.map((connector) => {
|
||||||
const isIndexing = indexingConnectorIds.has(connector.id);
|
const isIndexing = indexingConnectorIds.has(connector.id);
|
||||||
const activeTask = logsSummary?.active_tasks?.find(
|
|
||||||
(task: LogActiveTask) => task.connector_id === connector.id
|
|
||||||
);
|
|
||||||
const documentCount = getDocumentCountForConnector(
|
const documentCount = getDocumentCountForConnector(
|
||||||
connector.connector_type,
|
connector.connector_type,
|
||||||
documentTypeCounts
|
documentTypeCounts
|
||||||
|
|
@ -259,12 +253,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
||||||
{isIndexing && (
|
{isIndexing && (
|
||||||
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
|
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
|
||||||
<Loader2 className="size-3 animate-spin" />
|
<Loader2 className="size-3 animate-spin" />
|
||||||
Indexing...
|
Syncing...
|
||||||
{activeTask?.message && (
|
|
||||||
<span className="text-muted-foreground truncate max-w-[150px]">
|
|
||||||
• {activeTask.message}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
<p className="text-[10px] text-muted-foreground mt-1">
|
<p className="text-[10px] text-muted-foreground mt-1">
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,7 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { Plus } from "lucide-react";
|
|
||||||
import type { FC } from "react";
|
import type { FC } from "react";
|
||||||
import { Button } from "@/components/ui/button";
|
|
||||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
|
|
||||||
import { ConnectorCard } from "../components/connector-card";
|
import { ConnectorCard } from "../components/connector-card";
|
||||||
import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
|
import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
|
||||||
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
|
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
|
||||||
|
|
@ -30,7 +27,6 @@ interface AllConnectorsTabProps {
|
||||||
allConnectors: SearchSourceConnector[] | undefined;
|
allConnectors: SearchSourceConnector[] | undefined;
|
||||||
documentTypeCounts?: Record<string, number>;
|
documentTypeCounts?: Record<string, number>;
|
||||||
indexingConnectorIds?: Set<number>;
|
indexingConnectorIds?: Set<number>;
|
||||||
logsSummary?: LogSummary;
|
|
||||||
onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void;
|
onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void;
|
||||||
onConnectNonOAuth?: (connectorType: string) => void;
|
onConnectNonOAuth?: (connectorType: string) => void;
|
||||||
onCreateWebcrawler?: () => void;
|
onCreateWebcrawler?: () => void;
|
||||||
|
|
@ -41,13 +37,11 @@ interface AllConnectorsTabProps {
|
||||||
|
|
||||||
export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
searchQuery,
|
searchQuery,
|
||||||
searchSpaceId,
|
|
||||||
connectedTypes,
|
connectedTypes,
|
||||||
connectingId,
|
connectingId,
|
||||||
allConnectors,
|
allConnectors,
|
||||||
documentTypeCounts,
|
documentTypeCounts,
|
||||||
indexingConnectorIds,
|
indexingConnectorIds,
|
||||||
logsSummary,
|
|
||||||
onConnectOAuth,
|
onConnectOAuth,
|
||||||
onConnectNonOAuth,
|
onConnectNonOAuth,
|
||||||
onCreateWebcrawler,
|
onCreateWebcrawler,
|
||||||
|
|
@ -55,13 +49,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
onManage,
|
onManage,
|
||||||
onViewAccountsList,
|
onViewAccountsList,
|
||||||
}) => {
|
}) => {
|
||||||
// Helper to find active task for a connector
|
|
||||||
const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => {
|
|
||||||
if (!logsSummary?.active_tasks) return undefined;
|
|
||||||
return logsSummary.active_tasks.find(
|
|
||||||
(task: LogActiveTask) => task.connector_id === connectorId
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
// Filter connectors based on search
|
// Filter connectors based on search
|
||||||
const filteredOAuth = OAUTH_CONNECTORS.filter(
|
const filteredOAuth = OAUTH_CONNECTORS.filter(
|
||||||
|
|
@ -111,11 +98,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
// Check if any account is currently indexing
|
// Check if any account is currently indexing
|
||||||
const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id));
|
const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id));
|
||||||
|
|
||||||
// Get active task from any indexing account
|
|
||||||
const activeTask = typeConnectors
|
|
||||||
.map((c) => getActiveTaskForConnector(c.id))
|
|
||||||
.find((task) => task !== undefined);
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<ConnectorCard
|
<ConnectorCard
|
||||||
key={connector.id}
|
key={connector.id}
|
||||||
|
|
@ -128,7 +110,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
documentCount={documentCount}
|
documentCount={documentCount}
|
||||||
accountCount={typeConnectors.length}
|
accountCount={typeConnectors.length}
|
||||||
isIndexing={isIndexing}
|
isIndexing={isIndexing}
|
||||||
activeTask={activeTask}
|
|
||||||
onConnect={() => onConnectOAuth(connector)}
|
onConnect={() => onConnectOAuth(connector)}
|
||||||
onManage={
|
onManage={
|
||||||
isConnected && onViewAccountsList
|
isConnected && onViewAccountsList
|
||||||
|
|
@ -166,9 +147,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
documentTypeCounts
|
documentTypeCounts
|
||||||
);
|
);
|
||||||
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
|
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
|
||||||
const activeTask = actualConnector
|
|
||||||
? getActiveTaskForConnector(actualConnector.id)
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
const handleConnect = onConnectNonOAuth
|
const handleConnect = onConnectNonOAuth
|
||||||
? () => onConnectNonOAuth(connector.connectorType)
|
? () => onConnectNonOAuth(connector.connectorType)
|
||||||
|
|
@ -185,7 +163,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
isConnecting={isConnecting}
|
isConnecting={isConnecting}
|
||||||
documentCount={documentCount}
|
documentCount={documentCount}
|
||||||
isIndexing={isIndexing}
|
isIndexing={isIndexing}
|
||||||
activeTask={activeTask}
|
|
||||||
onConnect={handleConnect}
|
onConnect={handleConnect}
|
||||||
onManage={
|
onManage={
|
||||||
actualConnector && onManage ? () => onManage(actualConnector) : undefined
|
actualConnector && onManage ? () => onManage(actualConnector) : undefined
|
||||||
|
|
@ -226,9 +203,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
? getDocumentCountForConnector(crawler.connectorType, documentTypeCounts)
|
? getDocumentCountForConnector(crawler.connectorType, documentTypeCounts)
|
||||||
: undefined;
|
: undefined;
|
||||||
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
|
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
|
||||||
const activeTask = actualConnector
|
|
||||||
? getActiveTaskForConnector(actualConnector.id)
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
const handleConnect =
|
const handleConnect =
|
||||||
isYouTube && onCreateYouTubeCrawler
|
isYouTube && onCreateYouTubeCrawler
|
||||||
|
|
@ -254,7 +228,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
isConnecting={isConnecting}
|
isConnecting={isConnecting}
|
||||||
documentCount={documentCount}
|
documentCount={documentCount}
|
||||||
isIndexing={isIndexing}
|
isIndexing={isIndexing}
|
||||||
activeTask={activeTask}
|
|
||||||
onConnect={handleConnect}
|
onConnect={handleConnect}
|
||||||
onManage={
|
onManage={
|
||||||
actualConnector && onManage ? () => onManage(actualConnector) : undefined
|
actualConnector && onManage ? () => onManage(actualConnector) : undefined
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,6 @@ import type { FC } from "react";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
|
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
import { useConnectorStatus } from "../hooks/use-connector-status";
|
import { useConnectorStatus } from "../hooks/use-connector-status";
|
||||||
import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
|
import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
|
||||||
|
|
@ -16,7 +15,6 @@ interface ConnectorAccountsListViewProps {
|
||||||
connectorTitle: string;
|
connectorTitle: string;
|
||||||
connectors: SearchSourceConnector[];
|
connectors: SearchSourceConnector[];
|
||||||
indexingConnectorIds: Set<number>;
|
indexingConnectorIds: Set<number>;
|
||||||
logsSummary: LogSummary | undefined;
|
|
||||||
onBack: () => void;
|
onBack: () => void;
|
||||||
onManage: (connector: SearchSourceConnector) => void;
|
onManage: (connector: SearchSourceConnector) => void;
|
||||||
onAddAccount: () => void;
|
onAddAccount: () => void;
|
||||||
|
|
@ -60,7 +58,6 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
|
||||||
connectorTitle,
|
connectorTitle,
|
||||||
connectors,
|
connectors,
|
||||||
indexingConnectorIds,
|
indexingConnectorIds,
|
||||||
logsSummary,
|
|
||||||
onBack,
|
onBack,
|
||||||
onManage,
|
onManage,
|
||||||
onAddAccount,
|
onAddAccount,
|
||||||
|
|
@ -137,9 +134,6 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
|
||||||
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
|
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
|
||||||
{typeConnectors.map((connector) => {
|
{typeConnectors.map((connector) => {
|
||||||
const isIndexing = indexingConnectorIds.has(connector.id);
|
const isIndexing = indexingConnectorIds.has(connector.id);
|
||||||
const activeTask = logsSummary?.active_tasks?.find(
|
|
||||||
(task: LogActiveTask) => task.connector_id === connector.id
|
|
||||||
);
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
|
|
@ -168,12 +162,7 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
|
||||||
{isIndexing ? (
|
{isIndexing ? (
|
||||||
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
|
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
|
||||||
<Loader2 className="size-3 animate-spin" />
|
<Loader2 className="size-3 animate-spin" />
|
||||||
Indexing...
|
Syncing...
|
||||||
{activeTask?.message && (
|
|
||||||
<span className="text-muted-foreground truncate max-w-[100px]">
|
|
||||||
• {activeTask.message}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</p>
|
</p>
|
||||||
) : (
|
) : (
|
||||||
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap truncate">
|
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap truncate">
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,29 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) {
|
||||||
const syncHandleRef = useRef<SyncHandle | null>(null)
|
const syncHandleRef = useRef<SyncHandle | null>(null)
|
||||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null)
|
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null)
|
||||||
|
|
||||||
|
// Transform connector data from Electric SQL/PGlite to match expected format
|
||||||
|
// Converts Date objects to ISO strings as expected by Zod schema
|
||||||
|
function transformConnector(connector: any): SearchSourceConnector {
|
||||||
|
return {
|
||||||
|
...connector,
|
||||||
|
last_indexed_at: connector.last_indexed_at
|
||||||
|
? typeof connector.last_indexed_at === 'string'
|
||||||
|
? connector.last_indexed_at
|
||||||
|
: new Date(connector.last_indexed_at).toISOString()
|
||||||
|
: null,
|
||||||
|
next_scheduled_at: connector.next_scheduled_at
|
||||||
|
? typeof connector.next_scheduled_at === 'string'
|
||||||
|
? connector.next_scheduled_at
|
||||||
|
: new Date(connector.next_scheduled_at).toISOString()
|
||||||
|
: null,
|
||||||
|
created_at: connector.created_at
|
||||||
|
? typeof connector.created_at === 'string'
|
||||||
|
? connector.created_at
|
||||||
|
: new Date(connector.created_at).toISOString()
|
||||||
|
: new Date().toISOString(), // fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize Electric SQL and start syncing with real-time updates
|
// Initialize Electric SQL and start syncing with real-time updates
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!searchSpaceId) {
|
if (!searchSpaceId) {
|
||||||
|
|
@ -107,20 +130,20 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) {
|
||||||
// Set initial results immediately from the resolved query
|
// Set initial results immediately from the resolved query
|
||||||
if (liveQuery.initialResults?.rows) {
|
if (liveQuery.initialResults?.rows) {
|
||||||
console.log('📋 Initial live query results for connectors:', liveQuery.initialResults.rows.length)
|
console.log('📋 Initial live query results for connectors:', liveQuery.initialResults.rows.length)
|
||||||
setConnectors(liveQuery.initialResults.rows)
|
setConnectors(liveQuery.initialResults.rows.map(transformConnector))
|
||||||
} else if (liveQuery.rows) {
|
} else if (liveQuery.rows) {
|
||||||
// Some versions have rows directly on the result
|
// Some versions have rows directly on the result
|
||||||
console.log('📋 Initial live query results for connectors (direct):', liveQuery.rows.length)
|
console.log('📋 Initial live query results for connectors (direct):', liveQuery.rows.length)
|
||||||
setConnectors(liveQuery.rows)
|
setConnectors(liveQuery.rows.map(transformConnector))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subscribe to changes - this is the correct API!
|
// Subscribe to changes - this is the correct API!
|
||||||
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
||||||
if (typeof liveQuery.subscribe === 'function') {
|
if (typeof liveQuery.subscribe === 'function') {
|
||||||
liveQuery.subscribe((result: { rows: SearchSourceConnector[] }) => {
|
liveQuery.subscribe((result: { rows: any[] }) => {
|
||||||
if (mounted && result.rows) {
|
if (mounted && result.rows) {
|
||||||
console.log('🔄 Connectors updated via live query:', result.rows.length)
|
console.log('🔄 Connectors updated via live query:', result.rows.length)
|
||||||
setConnectors(result.rows)
|
setConnectors(result.rows.map(transformConnector))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -161,7 +184,7 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) {
|
||||||
[searchSpaceId]
|
[searchSpaceId]
|
||||||
)
|
)
|
||||||
console.log('📋 Fetched connectors from PGlite:', result.rows?.length || 0)
|
console.log('📋 Fetched connectors from PGlite:', result.rows?.length || 0)
|
||||||
setConnectors(result.rows || [])
|
setConnectors((result.rows || []).map(transformConnector))
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to fetch connectors from PGlite:', err)
|
console.error('Failed to fetch connectors from PGlite:', err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue