diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 0af368081..413be03c4 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -50,6 +50,7 @@ REGISTRATION_ENABLED=TRUE or FALSE # For Google Auth Only GOOGLE_OAUTH_CLIENT_ID=924507538m GOOGLE_OAUTH_CLIENT_SECRET=GOCSV +GOOGLE_PICKER_API_KEY=your-google-picker-api-key # Google Connector Specific Configurations GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index a03ef5f8a..aaf77a54f 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -235,6 +235,7 @@ class Config: # Google OAuth GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID") GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET") + GOOGLE_PICKER_API_KEY = os.getenv("GOOGLE_PICKER_API_KEY") # Google Calendar redirect URI GOOGLE_CALENDAR_REDIRECT_URI = os.getenv("GOOGLE_CALENDAR_REDIRECT_URI") diff --git a/surfsense_backend/app/connectors/composio_gmail_connector.py b/surfsense_backend/app/connectors/composio_gmail_connector.py index e83ba5cfb..94ee7b14c 100644 --- a/surfsense_backend/app/connectors/composio_gmail_connector.py +++ b/surfsense_backend/app/connectors/composio_gmail_connector.py @@ -10,6 +10,9 @@ from collections.abc import Awaitable, Callable from datetime import UTC, datetime from typing import Any +from bs4 import BeautifulSoup +from markdownify import markdownify as md + from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm import selectinload @@ -130,6 +133,16 @@ class ComposioGmailConnector(ComposioConnector): message_id=message_id, ) + @staticmethod + def _html_to_markdown(html: str) -> str: + """Convert HTML (especially email layouts with nested tables) to clean markdown.""" + soup = BeautifulSoup(html, "html.parser") + for tag in soup.find_all(["style", "script", "img"]): + tag.decompose() + for tag in soup.find_all(["table", "thead", "tbody", "tfoot", "tr", "td", "th"]): + tag.unwrap() + return md(str(soup)).strip() + def format_gmail_message_to_markdown(self, message: dict[str, Any]) -> str: """ Format a Gmail message to markdown. @@ -178,9 +191,10 @@ class ComposioGmailConnector(ComposioConnector): markdown_content += "\n---\n\n" - # Composio provides full message text in 'messageText' + # Composio provides full message text in 'messageText' which is often raw HTML message_text = message.get("messageText", "") if message_text: + message_text = self._html_to_markdown(message_text) markdown_content += f"## Content\n\n{message_text}\n\n" else: # Fallback to snippet if no messageText diff --git a/surfsense_backend/app/connectors/google_gmail_connector.py b/surfsense_backend/app/connectors/google_gmail_connector.py index 7c7262bff..46b825253 100644 --- a/surfsense_backend/app/connectors/google_gmail_connector.py +++ b/surfsense_backend/app/connectors/google_gmail_connector.py @@ -7,9 +7,11 @@ Allows fetching emails from Gmail mailbox using Google OAuth credentials. import base64 import json import logging -import re from typing import Any +from bs4 import BeautifulSoup +from markdownify import markdownify as md + from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -348,6 +350,16 @@ class GoogleGmailConnector: except Exception as e: return [], f"Error fetching recent messages: {e!s}" + @staticmethod + def _html_to_markdown(html: str) -> str: + """Convert HTML (especially email layouts with nested tables) to clean markdown.""" + soup = BeautifulSoup(html, "html.parser") + for tag in soup.find_all(["style", "script", "img"]): + tag.decompose() + for tag in soup.find_all(["table", "thead", "tbody", "tfoot", "tr", "td", "th"]): + tag.unwrap() + return md(str(soup)).strip() + def extract_message_text(self, message: dict[str, Any]) -> str: """ Extract text content from a Gmail message. @@ -387,13 +399,10 @@ class GoogleGmailConnector: ) text_content += decoded_data + "\n" elif mime_type == "text/html" and data and not text_content: - # Use HTML as fallback if no plain text decoded_data = base64.urlsafe_b64decode(data + "===").decode( "utf-8", errors="ignore" ) - # Basic HTML tag removal (you might want to use a proper HTML parser) - - text_content = re.sub(r"<[^>]+>", "", decoded_data) + text_content = self._html_to_markdown(decoded_data) return text_content.strip() diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index e808635e6..800eb5629 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -52,7 +52,9 @@ from app.schemas import ( SearchSourceConnectorRead, SearchSourceConnectorUpdate, ) -from app.services.composio_service import ComposioService +import asyncio + +from app.services.composio_service import ComposioService, get_composio_service from app.services.notification_service import NotificationService from app.tasks.connector_indexers import ( index_airtable_records, @@ -3054,3 +3056,86 @@ async def test_mcp_server_connection( "message": f"Failed to test connection: {e!s}", "tools": [], } + + +# --------------------------------------------------------------------------- +# Google Picker token endpoint (unified for native & Composio Drive) +# --------------------------------------------------------------------------- + +DRIVE_CONNECTOR_TYPES = { + SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, +} + + +@router.get("/connectors/{connector_id}/drive-picker-token") +async def get_drive_picker_token( + connector_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """Return an OAuth access token + client ID for the Google Picker API.""" + result = await session.execute( + select(SearchSourceConnector).filter(SearchSourceConnector.id == connector_id) + ) + connector = result.scalars().first() + if not connector: + raise HTTPException(status_code=404, detail="Connector not found") + + await check_permission( + session, + user, + connector.search_space_id, + Permission.CONNECTORS_READ.value, + "You don't have permission to access this connector", + ) + + if connector.connector_type not in DRIVE_CONNECTOR_TYPES: + raise HTTPException( + status_code=400, + detail="This endpoint is only for Google Drive connectors", + ) + + picker_api_key = config.GOOGLE_PICKER_API_KEY + if not picker_api_key: + raise HTTPException( + status_code=500, + detail="GOOGLE_PICKER_API_KEY is not configured on the server", + ) + + try: + if connector.connector_type == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR: + from app.connectors.google_drive.credentials import get_valid_credentials + + credentials = await get_valid_credentials(session, connector_id) + return { + "access_token": credentials.token, + "client_id": config.GOOGLE_OAUTH_CLIENT_ID, + "picker_api_key": picker_api_key, + } + + # Composio path + composio_account_id = (connector.config or {}).get( + "composio_connected_account_id" + ) + if not composio_account_id: + raise HTTPException( + status_code=400, + detail="Composio connected account not found. Please reconnect.", + ) + service = get_composio_service() + access_token = await asyncio.to_thread(service.get_access_token, composio_account_id) + return { + "access_token": access_token, + "client_id": config.GOOGLE_OAUTH_CLIENT_ID, + "picker_api_key": picker_api_key, + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get Drive picker token: {e!s}", exc_info=True) + raise HTTPException( + status_code=500, + detail="Failed to retrieve access token. Check server logs for details.", + ) from e diff --git a/surfsense_backend/app/services/composio_service.py b/surfsense_backend/app/services/composio_service.py index 763347d5a..3930d38ad 100644 --- a/surfsense_backend/app/services/composio_service.py +++ b/surfsense_backend/app/services/composio_service.py @@ -247,6 +247,19 @@ class ComposioService: ) return False + def get_access_token(self, connected_account_id: str) -> str: + """Retrieve the raw OAuth access token for a Composio connected account.""" + account = self.client.connected_accounts.get(nanoid=connected_account_id) + token = getattr(getattr(account, "state", None), "val", None) + if token is None: + raise ValueError( + f"No state.val on connected account {connected_account_id}" + ) + access_token = getattr(token, "access_token", None) + if not access_token: + raise ValueError(f"No access_token in state.val for {connected_account_id}") + return access_token + async def execute_tool( self, connected_account_id: str, diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index f388acd18..cf1235c65 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -4,7 +4,7 @@ import { useAtomValue } from "jotai"; import { AlertTriangle, Cable, Settings } from "lucide-react"; import Link from "next/link"; import { useSearchParams } from "next/navigation"; -import { type FC, forwardRef, useImperativeHandle, useMemo } from "react"; +import { type FC, forwardRef, useEffect, useImperativeHandle, useMemo, useState } from "react"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { statusInboxItemsAtom } from "@/atoms/inbox/status-inbox.atom"; import { @@ -21,6 +21,7 @@ import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsContent } from "@/components/ui/tabs"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { useConnectorsElectric } from "@/hooks/use-connectors-electric"; +import { PICKER_CLOSE_EVENT, PICKER_OPEN_EVENT } from "@/hooks/use-google-picker"; import { cn } from "@/lib/utils"; import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header"; import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view"; @@ -143,6 +144,18 @@ export const ConnectorIndicator = forwardRef { + const onOpen = () => setPickerOpen(true); + const onClose = () => setPickerOpen(false); + window.addEventListener(PICKER_OPEN_EVENT, onOpen); + window.addEventListener(PICKER_CLOSE_EVENT, onClose); + return () => { + window.removeEventListener(PICKER_OPEN_EVENT, onOpen); + window.removeEventListener(PICKER_CLOSE_EVENT, onClose); + }; + }, []); + // Fetch connectors using Electric SQL + PGlite for real-time updates // This provides instant updates when connectors change, without polling const { @@ -202,7 +215,14 @@ export const ConnectorIndicator = forwardRef + { + if (!open && pickerOpen) return; + handleOpenChange(open); + }} + modal={!pickerOpen} + > {showTrigger && ( ; } diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx index 8480ab53c..3f38eb1e6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx @@ -8,12 +8,13 @@ import { FileText, FolderClosed, Image, + Loader2, Presentation, X, } from "lucide-react"; import type { FC } from "react"; -import { useEffect, useState } from "react"; -import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree"; +import { useCallback, useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; import { Label } from "@/components/ui/label"; import { Select, @@ -23,9 +24,10 @@ import { SelectValue, } from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; +import { type PickerResult, useGooglePicker } from "@/hooks/use-google-picker"; import type { ConnectorConfigProps } from "../index"; -interface SelectedFolder { +interface SelectedItem { id: string; name: string; } @@ -42,10 +44,8 @@ const DEFAULT_INDEXING_OPTIONS: IndexingOptions = { include_subfolders: true, }; -// Helper to get appropriate icon for file type based on file name function getFileIconFromName(fileName: string, className: string = "size-3.5 shrink-0") { const lowerName = fileName.toLowerCase(); - // Spreadsheets if ( lowerName.endsWith(".xlsx") || lowerName.endsWith(".xls") || @@ -54,7 +54,6 @@ function getFileIconFromName(fileName: string, className: string = "size-3.5 shr ) { return ; } - // Presentations if ( lowerName.endsWith(".pptx") || lowerName.endsWith(".ppt") || @@ -62,7 +61,6 @@ function getFileIconFromName(fileName: string, className: string = "size-3.5 shr ) { return ; } - // Documents (word, text only - not PDF) if ( lowerName.endsWith(".docx") || lowerName.endsWith(".doc") || @@ -73,7 +71,6 @@ function getFileIconFromName(fileName: string, className: string = "size-3.5 shr ) { return ; } - // Images if ( lowerName.endsWith(".png") || lowerName.endsWith(".jpg") || @@ -84,29 +81,22 @@ function getFileIconFromName(fileName: string, className: string = "size-3.5 shr ) { return ; } - // Default (including PDF) return ; } export const GoogleDriveConfig: FC = ({ connector, onConfigChange }) => { - // Initialize with existing selected folders and files from connector config - const existingFolders = - (connector.config?.selected_folders as SelectedFolder[] | undefined) || []; - const existingFiles = (connector.config?.selected_files as SelectedFolder[] | undefined) || []; + const existingFolders = (connector.config?.selected_folders as SelectedItem[] | undefined) || []; + const existingFiles = (connector.config?.selected_files as SelectedItem[] | undefined) || []; const existingIndexingOptions = (connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS; - const [selectedFolders, setSelectedFolders] = useState(existingFolders); - const [selectedFiles, setSelectedFiles] = useState(existingFiles); + const [selectedFolders, setSelectedFolders] = useState(existingFolders); + const [selectedFiles, setSelectedFiles] = useState(existingFiles); const [indexingOptions, setIndexingOptions] = useState(existingIndexingOptions); - const [isEditMode] = useState(() => existingFolders.length > 0 || existingFiles.length > 0); - const [isFolderTreeOpen, setIsFolderTreeOpen] = useState(!isEditMode); - - // Update selected folders and files when connector config changes useEffect(() => { - const folders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || []; - const files = (connector.config?.selected_files as SelectedFolder[] | undefined) || []; + const folders = (connector.config?.selected_folders as SelectedItem[] | undefined) || []; + const files = (connector.config?.selected_files as SelectedItem[] | undefined) || []; const options = (connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS; @@ -116,8 +106,8 @@ export const GoogleDriveConfig: FC = ({ connector, onConfi }, [connector.config]); const updateConfig = ( - folders: SelectedFolder[], - files: SelectedFolder[], + folders: SelectedItem[], + files: SelectedItem[], options: IndexingOptions ) => { if (onConfigChange) { @@ -130,15 +120,26 @@ export const GoogleDriveConfig: FC = ({ connector, onConfi } }; - const handleSelectFolders = (folders: SelectedFolder[]) => { - setSelectedFolders(folders); - updateConfig(folders, selectedFiles, indexingOptions); - }; + const handlePicked = useCallback( + (result: PickerResult) => { + const folders = result.folders.map((f) => ({ id: f.id, name: f.name })); + const files = result.files.map((f) => ({ id: f.id, name: f.name })); + setSelectedFolders(folders); + setSelectedFiles(files); + updateConfig(folders, files, indexingOptions); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [indexingOptions, connector.config] + ); - const handleSelectFiles = (files: SelectedFolder[]) => { - setSelectedFiles(files); - updateConfig(selectedFolders, files, indexingOptions); - }; + const { + openPicker, + loading: pickerLoading, + error: pickerError, + } = useGooglePicker({ + connectorId: connector.id, + onPicked: handlePicked, + }); const handleIndexingOptionChange = (key: keyof IndexingOptions, value: number | boolean) => { const newOptions = { ...indexingOptions, [key]: value }; @@ -147,13 +148,13 @@ export const GoogleDriveConfig: FC = ({ connector, onConfi }; const handleRemoveFolder = (folderId: string) => { - const newFolders = selectedFolders.filter((folder) => folder.id !== folderId); + const newFolders = selectedFolders.filter((f) => f.id !== folderId); setSelectedFolders(newFolders); updateConfig(newFolders, selectedFiles, indexingOptions); }; const handleRemoveFile = (fileId: string) => { - const newFiles = selectedFiles.filter((file) => file.id !== fileId); + const newFiles = selectedFiles.filter((f) => f.id !== fileId); setSelectedFiles(newFiles); updateConfig(selectedFolders, newFiles, indexingOptions); }; @@ -228,39 +229,18 @@ export const GoogleDriveConfig: FC = ({ connector, onConfi )} - {isEditMode ? ( -
- - {isFolderTreeOpen && ( - - )} -
- ) : ( - - )} + + + {pickerError &&

{pickerError}

} {/* Indexing Options */} diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 566b2fdbc..9889708d7 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -2,7 +2,6 @@ import { EnumConnectorName } from "@/contracts/enums/connector"; // OAuth Connectors (Quick Connect) export const OAUTH_CONNECTORS = [ - // // Uncomment for managed Google Connections // { // id: "google-drive-connector", // title: "Google Drive", @@ -241,5 +240,95 @@ export const COMPOSIO_TOOLKITS = [ }, ] as const; +export interface AutoIndexConfig { + daysBack: number; + daysForward: number; + frequencyMinutes: number; + syncDescription: string; +} + +export const AUTO_INDEX_DEFAULTS: Record = { + [EnumConnectorName.GOOGLE_GMAIL_CONNECTOR]: { + daysBack: 30, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 30 days of emails.", + }, + [EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR]: { + daysBack: 30, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 30 days of emails.", + }, + [EnumConnectorName.SLACK_CONNECTOR]: { + daysBack: 30, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 30 days of messages.", + }, + [EnumConnectorName.DISCORD_CONNECTOR]: { + daysBack: 30, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 30 days of messages.", + }, + [EnumConnectorName.TEAMS_CONNECTOR]: { + daysBack: 30, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 30 days of messages.", + }, + [EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR]: { + daysBack: 90, + daysForward: 90, + frequencyMinutes: 1440, + syncDescription: "Syncing 90 days of past and upcoming events.", + }, + [EnumConnectorName.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR]: { + daysBack: 90, + daysForward: 90, + frequencyMinutes: 1440, + syncDescription: "Syncing 90 days of past and upcoming events.", + }, + [EnumConnectorName.LINEAR_CONNECTOR]: { + daysBack: 90, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 90 days of issues.", + }, + [EnumConnectorName.JIRA_CONNECTOR]: { + daysBack: 90, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 90 days of issues.", + }, + [EnumConnectorName.CLICKUP_CONNECTOR]: { + daysBack: 90, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your last 90 days of tasks.", + }, + [EnumConnectorName.NOTION_CONNECTOR]: { + daysBack: 365, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your pages.", + }, + [EnumConnectorName.CONFLUENCE_CONNECTOR]: { + daysBack: 365, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your documentation.", + }, + [EnumConnectorName.AIRTABLE_CONNECTOR]: { + daysBack: 365, + daysForward: 0, + frequencyMinutes: 1440, + syncDescription: "Syncing your bases.", + }, +}; + +export const AUTO_INDEX_CONNECTOR_TYPES = new Set(Object.keys(AUTO_INDEX_DEFAULTS)); + // Re-export IndexingConfigState from schemas for backward compatibility export type { IndexingConfigState } from "./connector-popup.schemas"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index 4585057ff..8cbf08bb9 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -28,6 +28,8 @@ import { cacheKeys } from "@/lib/query-client/cache-keys"; import { queryClient } from "@/lib/query-client/client"; import type { IndexingConfigState } from "../constants/connector-constants"; import { + AUTO_INDEX_CONNECTOR_TYPES, + AUTO_INDEX_DEFAULTS, COMPOSIO_CONNECTORS, OAUTH_CONNECTORS, OTHER_CONNECTORS, @@ -80,6 +82,7 @@ export const useConnectorDialog = () => { const [connectingConnectorType, setConnectingConnectorType] = useState(null); const [isCreatingConnector, setIsCreatingConnector] = useState(false); const isCreatingConnectorRef = useRef(false); + const isAutoIndexingRef = useRef(false); // Accounts list view state (for OAuth connectors with multiple accounts) const [viewingAccountsType, setViewingAccountsType] = useState<{ @@ -119,6 +122,71 @@ export const useConnectorDialog = () => { } }, []); + const handleAutoIndex = useCallback( + async ( + connector: SearchSourceConnector, + connectorTitle: string, + connectorType: string + ) => { + if (!searchSpaceId || isAutoIndexingRef.current) return; + isAutoIndexingRef.current = true; + + const defaults = AUTO_INDEX_DEFAULTS[connectorType]; + const now = new Date(); + const startDate = new Date(now); + startDate.setDate(startDate.getDate() - (defaults?.daysBack ?? 365)); + const endDate = new Date(now); + endDate.setDate(endDate.getDate() + (defaults?.daysForward ?? 0)); + + const toastId = "auto-index"; + toast.loading(`Setting up ${connectorTitle}...`, { id: toastId }); + + try { + await updateConnector({ + id: connector.id, + data: { + periodic_indexing_enabled: true, + indexing_frequency_minutes: defaults?.frequencyMinutes ?? 1440, + }, + }); + + await indexConnector({ + connector_id: connector.id, + queryParams: { + search_space_id: searchSpaceId, + start_date: format(startDate, "yyyy-MM-dd"), + end_date: format(endDate, "yyyy-MM-dd"), + }, + }); + + trackIndexWithDateRangeStarted( + Number(searchSpaceId), + connectorType, + connector.id, + { hasStartDate: true, hasEndDate: true } + ); + + toast.success(`${connectorTitle} connected!`, { + id: toastId, + description: defaults?.syncDescription ?? "Syncing started.", + }); + } catch (error) { + console.error("Auto-index failed:", error); + toast.error(`${connectorTitle} connected, but sync failed`, { + id: toastId, + description: "You can start syncing from settings.", + }); + } finally { + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), + }); + await refetchAllConnectors(); + isAutoIndexingRef.current = false; + } + }, + [searchSpaceId, indexConnector, updateConnector, refetchAllConnectors] + ); + // Synchronize state with URL query params useEffect(() => { try { @@ -336,8 +404,29 @@ export const useConnectorDialog = () => { } if (params.success === "true" && searchSpaceId && params.modal === "connectors") { - refetchAllConnectors().then((result) => { - if (!result.data) return; + // For auto-index connectors: close modal and show loading toast before refetch + const earlyConnector = params.connector + ? OAUTH_CONNECTORS.find((c) => c.id === params.connector) || + COMPOSIO_CONNECTORS.find((c) => c.id === params.connector) + : null; + + if (earlyConnector && AUTO_INDEX_CONNECTOR_TYPES.has(earlyConnector.connectorType)) { + toast.loading(`Setting up ${earlyConnector.title}...`, { id: "auto-index" }); + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.delete("connector"); + url.searchParams.delete("connectorId"); + url.searchParams.delete("view"); + url.searchParams.delete("modal"); + url.searchParams.delete("tab"); + router.replace(url.pathname + url.search, { scroll: false }); + } + + refetchAllConnectors().then(async (result) => { + if (!result.data) { + toast.dismiss("auto-index"); + return; + } let newConnector: SearchSourceConnector | undefined; let oauthConnector: @@ -376,31 +465,45 @@ export const useConnectorDialog = () => { if (newConnector && oauthConnector) { const connectorValidation = searchSourceConnector.safeParse(newConnector); if (connectorValidation.success) { - // Track connector connected event for OAuth/Composio connectors trackConnectorConnected( Number(searchSpaceId), oauthConnector.connectorType, newConnector.id ); - const config = validateIndexingConfigState({ - connectorType: oauthConnector.connectorType, - connectorId: newConnector.id, - connectorTitle: oauthConnector.title, - }); - setIndexingConfig(config); - setIndexingConnector(newConnector); - setIndexingConnectorConfig(newConnector.config); - setIsOpen(true); - const url = new URL(window.location.href); - url.searchParams.delete("success"); - url.searchParams.set("connectorId", newConnector.id.toString()); - url.searchParams.set("view", "configure"); - window.history.replaceState({}, "", url.toString()); + if ( + newConnector.is_indexable && + AUTO_INDEX_CONNECTOR_TYPES.has(oauthConnector.connectorType) + ) { + await handleAutoIndex( + newConnector, + oauthConnector.title, + oauthConnector.connectorType + ); + } else { + toast.dismiss("auto-index"); + const config = validateIndexingConfigState({ + connectorType: oauthConnector.connectorType, + connectorId: newConnector.id, + connectorTitle: oauthConnector.title, + }); + setIndexingConfig(config); + setIndexingConnector(newConnector); + setIndexingConnectorConfig(newConnector.config); + setIsOpen(true); + const url = new URL(window.location.href); + url.searchParams.delete("success"); + url.searchParams.set("connectorId", newConnector.id.toString()); + url.searchParams.set("view", "configure"); + window.history.replaceState({}, "", url.toString()); + } } else { console.warn("Invalid connector data after OAuth:", connectorValidation.error); + toast.dismiss("auto-index"); toast.error("Failed to validate connector data"); } + } else { + toast.dismiss("auto-index"); } }); } @@ -408,7 +511,7 @@ export const useConnectorDialog = () => { // Invalid query params - log but don't crash console.warn("Invalid connector popup query params in OAuth success handler:", error); } - }, [searchParams, searchSpaceId, refetchAllConnectors, setIsOpen]); + }, [searchParams, searchSpaceId, refetchAllConnectors, setIsOpen, handleAutoIndex, router]); // Handle OAuth connection const handleConnectOAuth = useCallback( @@ -479,6 +582,7 @@ export const useConnectorDialog = () => { periodic_indexing_enabled: false, indexing_frequency_minutes: null, next_scheduled_at: null, + enable_summary: false, }, queryParams: { search_space_id: searchSpaceId, @@ -583,6 +687,7 @@ export const useConnectorDialog = () => { connector_type: connectorData.connector_type as EnumConnectorName, is_active: true, next_scheduled_at: connectorData.next_scheduled_at as string | null, + enable_summary: false, }, queryParams: { search_space_id: searchSpaceId, @@ -1592,6 +1697,7 @@ export const useConnectorDialog = () => { handleCreateWebcrawler, handleCreateYouTubeCrawler, handleSubmitConnectForm, + handleAutoIndex, handleStartIndexing, handleSkipIndexing, handleStartEdit, diff --git a/surfsense_web/components/markdown-viewer.tsx b/surfsense_web/components/markdown-viewer.tsx index 7e0fc17b1..e22df8998 100644 --- a/surfsense_web/components/markdown-viewer.tsx +++ b/surfsense_web/components/markdown-viewer.tsx @@ -1,6 +1,5 @@ import { createCodePlugin } from "@streamdown/code"; import { createMathPlugin } from "@streamdown/math"; -import Image from "next/image"; import { Streamdown, type StreamdownProps } from "streamdown"; import "katex/dist/katex.min.css"; import { cn } from "@/lib/utils"; @@ -126,12 +125,12 @@ export function MarkdownViewer({ content, className }: MarkdownViewerProps) { ), hr: ({ ...props }) =>
, img: ({ src, alt, width: _w, height: _h, ...props }) => ( - {alt ), diff --git a/surfsense_web/hooks/use-google-picker.ts b/surfsense_web/hooks/use-google-picker.ts new file mode 100644 index 000000000..fa2a159b9 --- /dev/null +++ b/surfsense_web/hooks/use-google-picker.ts @@ -0,0 +1,186 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { connectorsApiService } from "@/lib/apis/connectors-api.service"; + +export interface PickerItem { + id: string; + name: string; + mimeType: string; +} + +export interface PickerResult { + folders: PickerItem[]; + files: PickerItem[]; +} + +interface UseGooglePickerOptions { + connectorId: number; + onPicked: (result: PickerResult) => void; +} + +const PICKER_SCRIPT_URL = "https://apis.google.com/js/api.js"; +const FOLDER_MIME = "application/vnd.google-apps.folder"; +export const PICKER_OPEN_EVENT = "google-picker-open"; +export const PICKER_CLOSE_EVENT = "google-picker-close"; + +let scriptLoadPromise: Promise | null = null; +let pickerApiPromise: Promise | null = null; + +function loadPickerScript(): Promise { + if (scriptLoadPromise) return scriptLoadPromise; + if (typeof window !== "undefined" && window.gapi) { + scriptLoadPromise = Promise.resolve(); + return scriptLoadPromise; + } + + scriptLoadPromise = new Promise((resolve, reject) => { + const script = document.createElement("script"); + script.src = PICKER_SCRIPT_URL; + script.async = true; + script.defer = true; + script.onload = () => resolve(); + script.onerror = () => { + scriptLoadPromise = null; + reject(new Error("Failed to load Google Picker script")); + }; + document.head.appendChild(script); + }); + return scriptLoadPromise; +} + +function loadPickerApi(): Promise { + if (pickerApiPromise) return pickerApiPromise; + + pickerApiPromise = new Promise((resolve, reject) => { + gapi.load("picker", { + callback: () => resolve(), + onerror: () => { + pickerApiPromise = null; + reject(new Error("Failed to load Google Picker API")); + }, + }); + }); + return pickerApiPromise; +} + +export function useGooglePicker({ connectorId, onPicked }: UseGooglePickerOptions) { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const onPickedRef = useRef(onPicked); + onPickedRef.current = onPicked; + const openingRef = useRef(false); + const pickerRef = useRef(null); + + const closePicker = useCallback(() => { + if (!pickerRef.current) return; + window.dispatchEvent(new Event(PICKER_CLOSE_EVENT)); + pickerRef.current.dispose(); + pickerRef.current = null; + openingRef.current = false; + }, []); + + useEffect(() => { + const onEscape = (e: KeyboardEvent) => { + if (e.key === "Escape" && pickerRef.current) { + closePicker(); + } + }; + window.addEventListener("keydown", onEscape); + return () => { + window.removeEventListener("keydown", onEscape); + if (pickerRef.current) { + pickerRef.current.dispose(); + pickerRef.current = null; + } + openingRef.current = false; + }; + }, [closePicker]); + + const openPicker = useCallback(async () => { + if (openingRef.current) return; + openingRef.current = true; + setLoading(true); + setError(null); + + try { + const [tokenData] = await Promise.all([ + connectorsApiService.getDrivePickerToken(connectorId), + loadPickerScript().then(() => loadPickerApi()), + ]); + + const { access_token, picker_api_key } = tokenData; + + const docsView = new google.picker.DocsView(google.picker.ViewId.DOCS) + .setIncludeFolders(true) + .setSelectFolderEnabled(true); + + const builder = new google.picker.PickerBuilder() + .addView(docsView) + .enableFeature(google.picker.Feature.MULTISELECT_ENABLED) + .setOAuthToken(access_token) + .setOrigin(window.location.protocol + "//" + window.location.host) + .setTitle("Select files and folders to index"); + + if (picker_api_key) { + builder.setDeveloperKey(picker_api_key); + } + + const picker = builder + .setCallback((data: google.picker.ResponseObject) => { + const action = data[google.picker.Response.ACTION]; + + if (action === google.picker.Action.PICKED) { + const docs = data[google.picker.Response.DOCUMENTS]; + if (docs) { + const folders: PickerItem[] = []; + const files: PickerItem[] = []; + + for (const doc of docs) { + const mimeType = doc[google.picker.Document.MIME_TYPE] ?? ""; + const item: PickerItem = { + id: doc[google.picker.Document.ID], + name: doc[google.picker.Document.NAME] ?? "Untitled", + mimeType, + }; + if (mimeType === FOLDER_MIME) { + folders.push(item); + } else { + files.push(item); + } + } + + onPickedRef.current({ folders, files }); + } + } + + if (action === google.picker.Action.ERROR) { + setError("Google Drive encountered an error. Please try again."); + } + + if ( + action === google.picker.Action.PICKED || + action === google.picker.Action.CANCEL || + action === google.picker.Action.ERROR + ) { + closePicker(); + } + }) + .build(); + + pickerRef.current = picker; + window.dispatchEvent(new Event(PICKER_OPEN_EVENT)); + picker.setVisible(true); + } catch (err) { + window.dispatchEvent(new Event(PICKER_CLOSE_EVENT)); + openingRef.current = false; + const msg = err instanceof Error ? err.message : "Failed to open Google Picker"; + setError(msg); + console.error("Google Picker error:", err); + } finally { + setLoading(false); + } + }, [connectorId, closePicker]); + + return { openPicker, closePicker, loading, error }; +} diff --git a/surfsense_web/lib/apis/connectors-api.service.ts b/surfsense_web/lib/apis/connectors-api.service.ts index 4a6d67d80..fafe1a8fa 100644 --- a/surfsense_web/lib/apis/connectors-api.service.ts +++ b/surfsense_web/lib/apis/connectors-api.service.ts @@ -266,6 +266,17 @@ class ConnectorsApiService { ); }; + /** + * Get Google Picker token (access_token + client_id + picker_api_key) for a Drive connector + */ + getDrivePickerToken = async (connectorId: number) => { + return baseApiService.get<{ + access_token: string; + client_id: string; + picker_api_key: string | null; + }>(`/api/v1/connectors/${connectorId}/drive-picker-token`); + }; + // ============================================================================= // MCP Connector Methods // ============================================================================= diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 0789e304e..89c1757a1 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -144,6 +144,8 @@ "@tailwindcss/postcss": "^4.1.11", "@tailwindcss/typography": "^0.5.16", "@types/canvas-confetti": "^1.9.0", + "@types/gapi": "^0.0.47", + "@types/google.picker": "^0.0.52", "@types/node": "^20.19.9", "@types/pg": "^8.15.5", "@types/react": "^19.1.8", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index b78cc44c8..5dda2b6cb 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -372,6 +372,12 @@ importers: '@types/canvas-confetti': specifier: ^1.9.0 version: 1.9.0 + '@types/gapi': + specifier: ^0.0.47 + version: 0.0.47 + '@types/google.picker': + specifier: ^0.0.52 + version: 0.0.52 '@types/node': specifier: ^20.19.9 version: 20.19.33 @@ -3807,6 +3813,12 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/gapi@0.0.47': + resolution: {integrity: sha512-/ZsLuq6BffMgbKMtZyDZ8vwQvTyKhKQ1G2K6VyWCgtHHhfSSXbk4+4JwImZiTjWNXfI2q1ZStAwFFHSkNoTkHA==} + + '@types/google.picker@0.0.52': + resolution: {integrity: sha512-k0HyW8HxJePomM2r0JWq9nE9XG6qY93lVpoVnaV4WjQggDHrGwDKq3G8CGpcBWhQlJBTxX9jDIrI7RQnqjM63w==} + '@types/hast@2.3.10': resolution: {integrity: sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==} @@ -10486,6 +10498,10 @@ snapshots: '@types/estree@1.0.8': {} + '@types/gapi@0.0.47': {} + + '@types/google.picker@0.0.52': {} + '@types/hast@2.3.10': dependencies: '@types/unist': 2.0.11