diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index fa124f1c2..3bcbe4dc0 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -255,9 +255,10 @@ async def airtable_callback( await session.commit() logger.info(f"Successfully saved Airtable connector for user {user_id}") - # Redirect to the frontend success page + # Redirect to the frontend with success params for indexing config + # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/airtable-connector?success=true" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=airtable-connector" ) except ValidationError as e: diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index fa4ef5466..8bb685450 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -131,8 +131,10 @@ async def calendar_callback( session.add(db_connector) await session.commit() await session.refresh(db_connector) + # Redirect to the frontend with success params for indexing config + # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-calendar-connector?success=true" + f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-calendar-connector" ) except ValidationError as e: await session.rollback() diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 6d37da244..21fcf2c38 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -135,9 +135,10 @@ async def gmail_callback( f"Successfully created Gmail connector for user {user_id} with ID {db_connector.id}" ) - # Redirect to the frontend success page + # Redirect to the frontend with success params for indexing config + # Using query params to auto-open the popup with config view on new-chat page return RedirectResponse( - url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-gmail-connector?success=true" + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/new-chat?modal=connectors&tab=all&success=true&connector=google-gmail-connector" ) except IntegrityError as e: diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 9043ef657..e55c13448 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -1,17 +1,22 @@ "use client"; +import { format, subDays, subYears } from "date-fns"; import { useAtomValue } from "jotai"; import { + ArrowLeft, Cable, + Calendar as CalendarIcon, + Check, ChevronRight, Loader2, Search, } from "lucide-react"; import Link from "next/link"; -import { usePathname, useRouter } from "next/navigation"; -import { type FC, useCallback, useEffect, useState } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { type FC, useCallback, useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon"; +import { indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; @@ -23,14 +28,35 @@ import { DialogTitle, } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { Label } from "@/components/ui/label"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Switch } from "@/components/ui/switch"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { useLogsSummary } from "@/hooks/use-logs"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { authenticatedFetch } from "@/lib/auth-utils"; +import { queryClient } from "@/lib/query-client/client"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { cn } from "@/lib/utils"; +// Type for the indexing configuration state +interface IndexingConfigState { + connectorType: string; + connectorId: number; + connectorTitle: string; +} + // OAuth Connectors (Quick Connect) const OAUTH_CONNECTORS = [ { @@ -165,44 +191,243 @@ import { export const ConnectorIndicator: FC = () => { const router = useRouter(); + const searchParams = useSearchParams(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors( + const { connectors, isLoading: connectorsLoading, refreshConnectors } = useSearchSourceConnectors( false, searchSpaceId ? Number(searchSpaceId) : undefined ); const { data: documentTypeCounts, isLoading: documentTypesLoading } = useAtomValue(documentTypeCountsAtom); - const { data: allConnectors } = useAtomValue(connectorsAtom); - const pathname = usePathname(); + const { data: allConnectors, refetch: refetchAllConnectors } = useAtomValue(connectorsAtom); + const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); const [isOpen, setIsOpen] = useState(false); const [activeTab, setActiveTab] = useState("all"); const [connectingId, setConnectingId] = useState(null); const [isScrolled, setIsScrolled] = useState(false); const [searchQuery, setSearchQuery] = useState(""); + + // Indexing configuration state (shown after OAuth success) + const [indexingConfig, setIndexingConfig] = useState(null); + const [startDate, setStartDate] = useState(undefined); + const [endDate, setEndDate] = useState(undefined); + const [isStartingIndexing, setIsStartingIndexing] = useState(false); + + // Periodic indexing state + const [periodicEnabled, setPeriodicEnabled] = useState(false); + const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); // Default: daily + + // Helper function to get frequency label + const getFrequencyLabel = useCallback((minutes: string): string => { + switch (minutes) { + case "15": return "15 minutes"; + case "60": return "hour"; + case "360": return "6 hours"; + case "720": return "12 hours"; + case "1440": return "day"; + case "10080": return "week"; + default: return `${minutes} minutes`; + } + }, []); + + // Track active indexing tasks + const { summary: logsSummary } = useLogsSummary( + searchSpaceId ? Number(searchSpaceId) : 0, + 24, + { + enablePolling: true, + refetchInterval: 5000, + } + ); + + // Get connector IDs that are currently being indexed + const indexingConnectorIds = useMemo(() => { + if (!logsSummary?.active_tasks) return new Set(); + return new Set( + logsSummary.active_tasks + .filter((task) => task.source?.includes("connector_indexing")) + .map((task) => { + // Extract connector ID from task metadata or source + const match = task.source?.match(/connector[_-]?(\d+)/i); + return match ? parseInt(match[1], 10) : null; + }) + .filter((id): id is number => id !== null) + ); + }, [logsSummary?.active_tasks]); const isLoading = connectorsLoading || documentTypesLoading; - // Synchronize state with URL path + // Synchronize state with URL query params useEffect(() => { - const pathParts = window.location.pathname.split("/"); - const connectorsIdx = pathParts.indexOf("connectors"); + const modalParam = searchParams.get("modal"); + const tabParam = searchParams.get("tab"); + const viewParam = searchParams.get("view"); + const connectorParam = searchParams.get("connector"); - if (connectorsIdx !== -1) { + if (modalParam === "connectors") { if (!isOpen) setIsOpen(true); - // Detect tab from URL: .../connectors/active or .../connectors/all - const tabFromUrl = pathParts[connectorsIdx + 1]; - if (tabFromUrl === "active" || tabFromUrl === "all") { - if (activeTab !== tabFromUrl) setActiveTab(tabFromUrl); + // Detect tab from URL query param + if (tabParam === "active" || tabParam === "all") { + if (activeTab !== tabParam) setActiveTab(tabParam); + } + + // Restore indexing config view from URL if present (e.g., on page refresh) + if (viewParam === "configure" && connectorParam && !indexingConfig) { + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); + if (oauthConnector && allConnectors) { + const existingConnector = allConnectors.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (existingConnector) { + setIndexingConfig({ + connectorType: oauthConnector.connectorType, + connectorId: existingConnector.id, + connectorTitle: oauthConnector.title, + }); + } + } } } else { if (isOpen) setIsOpen(false); } - }, [pathname, isOpen, activeTab]); + }, [searchParams, isOpen, activeTab, indexingConfig, allConnectors]); + + // Detect OAuth success and transition to config view + useEffect(() => { + const success = searchParams.get("success"); + const connectorParam = searchParams.get("connector"); + const modalParam = searchParams.get("modal"); + + if (success === "true" && connectorParam && searchSpaceId && modalParam === "connectors") { + // Find the OAuth connector info + const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === connectorParam); + if (oauthConnector) { + // Refetch connectors to get the newly created connector + refetchAllConnectors().then((result) => { + const newConnector = result.data?.find( + (c: SearchSourceConnector) => c.connector_type === oauthConnector.connectorType + ); + if (newConnector) { + setIndexingConfig({ + connectorType: oauthConnector.connectorType, + connectorId: newConnector.id, + connectorTitle: oauthConnector.title, + }); + setIsOpen(true); + // Update URL to reflect config view (replace success with view=configure) + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.set("view", "configure"); + // Keep connector param for URL restoration + window.history.replaceState({}, "", url.toString()); + } + }); + } + } + }, [searchParams, searchSpaceId, refetchAllConnectors]); + + // Handle starting indexing + const handleStartIndexing = useCallback(async () => { + if (!indexingConfig || !searchSpaceId) return; + + setIsStartingIndexing(true); + try { + const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; + const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; + + // Update periodic indexing settings if enabled + if (periodicEnabled) { + const frequency = parseInt(frequencyMinutes, 10); + await updateConnector({ + id: indexingConfig.connectorId, + data: { + periodic_indexing_enabled: true, + indexing_frequency_minutes: frequency, + }, + }); + } + + await indexConnector({ + connector_id: indexingConfig.connectorId, + queryParams: { + search_space_id: searchSpaceId, + start_date: startDateStr, + end_date: endDateStr, + }, + }); + + toast.success(`${indexingConfig.connectorTitle} indexing started`, { + description: periodicEnabled + ? `Periodic sync enabled every ${getFrequencyLabel(frequencyMinutes)}.` + : "You can continue working while we sync your data.", + }); + + // Close the config view and reset state + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + + // Clear config-related URL params and switch to active tab + const url = new URL(window.location.href); + url.searchParams.delete("view"); + url.searchParams.delete("connector"); + url.searchParams.set("tab", "active"); + window.history.replaceState({}, "", url.toString()); + setActiveTab("active"); + + // Refresh connectors list + refreshConnectors(); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + } catch (error) { + console.error("Error starting indexing:", error); + toast.error("Failed to start indexing"); + } finally { + setIsStartingIndexing(false); + } + }, [indexingConfig, searchSpaceId, startDate, endDate, indexConnector, updateConnector, periodicEnabled, frequencyMinutes, refreshConnectors, getFrequencyLabel]); + + // Handle skipping indexing for now + const handleSkipIndexing = useCallback(() => { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + + // Clear config-related URL params + const url = new URL(window.location.href); + url.searchParams.delete("view"); + url.searchParams.delete("connector"); + window.history.replaceState({}, "", url.toString()); + }, []); + + // Quick date range handlers + const handleLast30Days = useCallback(() => { + const today = new Date(); + setStartDate(subDays(today, 30)); + setEndDate(today); + }, []); + + const handleLastYear = useCallback(() => { + const today = new Date(); + setStartDate(subYears(today, 1)); + setEndDate(today); + }, []); + + const handleClearDates = useCallback(() => { + setStartDate(undefined); + setEndDate(undefined); + }, []); // Get document types that have documents in the search space const activeDocumentTypes = documentTypeCounts - ? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0) + ? Object.entries(documentTypeCounts).filter(([, count]) => count > 0) : []; const hasConnectors = connectors.length > 0; @@ -257,32 +482,43 @@ export const ConnectorIndicator: FC = () => { (open: boolean) => { setIsOpen(open); - const currentPath = window.location.pathname; - const basePath = currentPath.split("/connectors")[0].replace(/\/$/, ""); - if (open) { - // Base state is /connectors/all - const newUrl = `${basePath}/connectors/${activeTab}`; - window.history.pushState({ modal: true }, "", newUrl); + // Add modal query params to current URL + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("tab", activeTab); + window.history.pushState({ modal: true }, "", url.toString()); } else { - // Return to base chat path - window.history.pushState({ modal: false }, "", basePath || "/"); + // Remove modal query params from URL + const url = new URL(window.location.href); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("view"); + window.history.pushState({ modal: false }, "", url.toString()); setIsScrolled(false); setSearchQuery(""); + // Reset indexing config when closing + if (!isStartingIndexing) { + setIndexingConfig(null); + setStartDate(undefined); + setEndDate(undefined); + setPeriodicEnabled(false); + setFrequencyMinutes("1440"); + } } }, - [activeTab] + [activeTab, isStartingIndexing] ); const handleTabChange = useCallback( (value: string) => { setActiveTab(value); - const currentPath = window.location.pathname; - const basePath = currentPath.split("/connectors")[0].replace(/\/$/, ""); - - // Update URL to reflect the new tab state - const newUrl = `${basePath}/connectors/${value}`; - window.history.replaceState({ modal: true }, "", newUrl); + // Update tab query param + const url = new URL(window.location.href); + url.searchParams.set("tab", value); + window.history.replaceState({ modal: true }, "", url.toString()); }, [] ); @@ -323,8 +559,215 @@ export const ConnectorIndicator: FC = () => { )} - - + + {/* Indexing Configuration View - shown after OAuth success */} + {indexingConfig ? ( +
+ {/* Fixed Header */} +
+ {/* Back button */} + + + {/* Success header */} +
+
+ +
+
+

+ {indexingConfig.connectorTitle} Connected! +

+

+ Configure when to start syncing your data +

+
+
+
+ + {/* Scrollable Content */} +
+
+
+

Select Date Range

+

+ Choose how far back you want to sync your data. You can always re-index later with different dates. +

+ +
+ {/* Start Date */} +
+ + + + + + + date > new Date()} + /> + + +
+ + {/* End Date */} +
+ + + + + + + date > new Date() || (startDate ? date < startDate : false)} + /> + + +
+
+ + {/* Quick date range buttons */} +
+ + + +
+
+ + {/* Periodic Indexing Configuration */} +
+
+
+

Enable Periodic Sync

+

+ Automatically re-index at regular intervals +

+
+ +
+ + {periodicEnabled && ( +
+
+ + +
+
+ )} +
+ + {/* Info box */} +
+
+ {getConnectorIcon(indexingConfig.connectorType, "size-4")} +
+
+

Indexing runs in the background

+

+ You can continue using SurfSense while we sync your data. Check the Active tab to see progress. +

+
+
+
+
+ + {/* Fixed Footer - Action buttons */} +
+ + +
+
+ ) : ( + {/* Header */}
{ -
+
{ setSearchQuery(e.target.value)} /> @@ -397,9 +840,9 @@ export const ConnectorIndicator: FC = () => { return (
-
+
{getConnectorIcon(connector.connectorType, "size-6")}
@@ -455,9 +898,9 @@ export const ConnectorIndicator: FC = () => { -
+
{getConnectorIcon(connector.connectorType, "size-6")}
@@ -490,9 +933,9 @@ export const ConnectorIndicator: FC = () => { {activeDocumentTypes.map(([docType, count]) => (
-
+
{getConnectorIcon(docType, "size-6")}
@@ -505,25 +948,64 @@ export const ConnectorIndicator: FC = () => {
))} - {connectors.map((connector) => ( -
-
- {getConnectorIcon(connector.connector_type, "size-6")} + {connectors.map((connector) => { + const isIndexing = indexingConnectorIds.has(connector.id); + const activeTask = logsSummary?.active_tasks?.find( + (task) => task.source?.includes(`connector_${connector.id}`) || task.source?.includes(`connector-${connector.id}`) + ); + + return ( +
+
+ {getConnectorIcon(connector.connector_type, "size-6")} +
+
+

+ {connector.name} +

+ {isIndexing ? ( +

+ + Indexing... + {activeTask?.message && ( + + • {activeTask.message} + + )} +

+ ) : ( +

+ {connector.last_indexed_at + ? `Last indexed: ${format(new Date(connector.last_indexed_at), "MMM d, yyyy")}` + : "Never indexed"} +

+ )} +
+
-
-

- {connector.name} -

-

Status: Active

-
- -
- ))} + ); + })}
) : ( @@ -547,6 +1029,7 @@ export const ConnectorIndicator: FC = () => {
+ )} );