apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-12 09:12:40 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

resolve conflicts

Browse source
This commit is contained in:
Manoj Aggarwal 2026-01-16 15:39:57 -08:00
commit 11b160cee9
85 changed files with 8411 additions and 4637 deletions
Download patch file Download diff file Expand all files Collapse all files

123
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -1,19 +1,16 @@
"use client";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { useAtomValue } from "jotai";
import { Cable, Loader2 } from "lucide-react";
import { useSearchParams } from "next/navigation";
import { type FC, useEffect, useMemo } from "react";
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
import type { FC } from "react";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
import { Dialog, DialogContent } from "@/components/ui/dialog";
import { Tabs, TabsContent } from "@/components/ui/tabs";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { useLogsSummary } from "@/hooks/use-logs";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
import { cn } from "@/lib/utils";
import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header";
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
@ -21,6 +18,7 @@ import { ConnectorEditView } from "./connector-popup/connector-configs/views/con
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants";
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-connectors";
import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab";
import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab";
import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view";
@ -30,18 +28,13 @@ import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view
export const ConnectorIndicator: FC = () => {
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
const searchParams = useSearchParams();
const { data: documentTypeCounts, isLoading: documentTypesLoading } =
useAtomValue(documentTypeCountsAtom);
// Fetch document type counts using Electric SQL + PGlite for real-time updates
const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId);
// Check if YouTube view is active
const isYouTubeView = searchParams.get("view") === "youtube";
// Track active indexing tasks
const { summary: logsSummary } = useLogsSummary(searchSpaceId ? Number(searchSpaceId) : 0, 24, {
enablePolling: true,
refetchInterval: 5000,
});
// Use the custom hook for dialog state management
const {
isOpen,
@ -97,57 +90,35 @@ export const ConnectorIndicator: FC = () => {
setConnectorName,
} = useConnectorDialog();
// Fetch connectors using React Query with conditional refetchInterval
// This automatically refetches when mutations invalidate the cache (event-driven)
// and also polls when dialog is open to catch external changes
// Fetch connectors using Electric SQL + PGlite for real-time updates
// This provides instant updates when connectors change, without polling
const {
data: connectors = [],
isLoading: connectorsLoading,
refetch: refreshConnectors,
} = useQuery({
queryKey: cacheKeys.connectors.all(searchSpaceId || ""),
queryFn: () =>
connectorsApiService.getConnectors({
queryParams: {
search_space_id: searchSpaceId ? Number(searchSpaceId) : undefined,
},
}),
enabled: !!searchSpaceId,
staleTime: 5 * 60 * 1000, // 5 minutes (same as connectorsAtom)
// Poll when dialog is open to catch external changes
refetchInterval: isOpen ? 5000 : false, // 5 seconds when open, no polling when closed
});
connectors: connectorsFromElectric = [],
loading: connectorsLoading,
error: connectorsError,
refreshConnectors: refreshConnectorsElectric,
} = useConnectorsElectric(searchSpaceId);
const queryClient = useQueryClient();
// Fallback to API if Electric fails or is not available
const connectors =
connectorsFromElectric.length > 0 || !connectorsError
? connectorsFromElectric
: allConnectors || [];
// Also refresh document type counts when dialog is open
useEffect(() => {
if (!isOpen || !searchSpaceId) return;
// Manual refresh function that works with both Electric and API
const refreshConnectors = async () => {
if (connectorsFromElectric.length > 0 || !connectorsError) {
await refreshConnectorsElectric();
} else {
// Fallback: use allConnectors from useConnectorDialog (which uses connectorsAtom)
// The connectorsAtom will handle refetching if needed
}
};
const POLL_INTERVAL = 5000; // 5 seconds, same as connectors
const intervalId = setInterval(() => {
// Invalidate document type counts to refresh active document types
queryClient.invalidateQueries({
queryKey: cacheKeys.documents.typeCounts(searchSpaceId),
});
}, POLL_INTERVAL);
// Cleanup interval on unmount or when dialog closes
return () => {
clearInterval(intervalId);
};
}, [isOpen, searchSpaceId, queryClient]);
// Get connector IDs that are currently being indexed
const indexingConnectorIds = useMemo(() => {
if (!logsSummary?.active_tasks) return new Set<number>();
return new Set(
logsSummary.active_tasks
.filter((task) => task.source?.includes("connector_indexing") && task.connector_id != null)
.map((task) => task.connector_id as number)
);
}, [logsSummary?.active_tasks]);
// Track indexing state locally - clears automatically when Electric SQL detects last_indexed_at changed
const { indexingConnectorIds, startIndexing } = useIndexingConnectors(
connectors as SearchSourceConnector[]
);
const isLoading = connectorsLoading || documentTypesLoading;
@ -163,8 +134,9 @@ export const ConnectorIndicator: FC = () => {
const activeConnectorsCount = connectors.length;
// Check which connectors are already connected
// Using Electric SQL + PGlite for real-time connector updates
const connectedTypes = new Set(
(allConnectors || []).map((c: SearchSourceConnector) => c.connector_type)
(connectors || []).map((c: SearchSourceConnector) => c.connector_type)
);
if (!searchSpaceId) return null;
@ -221,9 +193,8 @@ export const ConnectorIndicator: FC = () => {
<ConnectorAccountsListView
connectorType={viewingAccountsType.connectorType}
connectorTitle={viewingAccountsType.connectorTitle}
connectors={(allConnectors || []) as SearchSourceConnector[]}
connectors={(connectors || []) as SearchSourceConnector[]} // Using Electric SQL + PGlite for real-time connector updates (all connector types)
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onBack={handleBackFromAccountsList}
onManage={handleStartEdit}
onAddAccount={() => {
@ -239,7 +210,7 @@ export const ConnectorIndicator: FC = () => {
) : connectingConnectorType ? (
<ConnectorConnectView
connectorType={connectingConnectorType}
onSubmit={handleSubmitConnectForm}
onSubmit={(formData) => handleSubmitConnectForm(formData, startIndexing)}
onBack={handleBackFromConnect}
isSubmitting={isCreatingConnector}
/>
@ -263,13 +234,18 @@ export const ConnectorIndicator: FC = () => {
onEndDateChange={setEndDate}
onPeriodicEnabledChange={setPeriodicEnabled}
onFrequencyChange={setFrequencyMinutes}
onSave={() => handleSaveConnector(() => refreshConnectors())}
onSave={() => {
startIndexing(editingConnector.id);
handleSaveConnector(() => refreshConnectors());
}}
onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())}
onBack={handleBackFromEdit}
onQuickIndex={
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
? () =>
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type)
? () => {
startIndexing(editingConnector.id);
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type);
}
: undefined
}
onConfigChange={setConnectorConfig}
@ -296,7 +272,12 @@ export const ConnectorIndicator: FC = () => {
onPeriodicEnabledChange={setPeriodicEnabled}
onFrequencyChange={setFrequencyMinutes}
onConfigChange={setIndexingConnectorConfig}
onStartIndexing={() => handleStartIndexing(() => refreshConnectors())}
onStartIndexing={() => {
if (indexingConfig.connectorId) {
startIndexing(indexingConfig.connectorId);
}
handleStartIndexing(() => refreshConnectors());
}}
onSkip={handleSkipIndexing}
/>
) : (
@ -325,10 +306,9 @@ export const ConnectorIndicator: FC = () => {
searchSpaceId={searchSpaceId}
connectedTypes={connectedTypes}
connectingId={connectingId}
allConnectors={allConnectors}
allConnectors={connectors}
documentTypeCounts={documentTypeCounts}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onConnectOAuth={handleConnectOAuth}
onConnectNonOAuth={handleConnectNonOAuth}
onCreateWebcrawler={handleCreateWebcrawler}
@ -345,7 +325,6 @@ export const ConnectorIndicator: FC = () => {
activeDocumentTypes={activeDocumentTypes}
connectors={connectors as SearchSourceConnector[]}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
searchSpaceId={searchSpaceId}
onTabChange={handleTabChange}
onManage={handleStartEdit}