apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-08 20:25:19 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: enhance Composio Google Drive integration with folder and file selection

Browse source 
- Added a new endpoint to list folders and files in a user's Composio Google Drive, supporting hierarchical structure.
- Implemented UI components for selecting specific folders and files to index, improving user control over indexing options.
- Introduced indexing options for maximum files per folder and inclusion of subfolders, allowing for customizable indexing behavior.
- Enhanced error handling and logging for Composio Drive operations, ensuring better visibility into issues during file retrieval and indexing.
- Updated the Composio configuration component to reflect new selection capabilities and indexing options.
This commit is contained in:
Anish Sarkar 2026-01-23 05:17:28 +05:30
parent e6a4ac7c9c
commit 7ec7ed5c3b
11 changed files with 1069 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

122
surfsense_backend/app/routes/composio_routes.py
View file

@ -8,6 +8,7 @@ Endpoints:
- GET /composio/toolkits - List available Composio toolkits
- GET /auth/composio/connector/add - Initiate OAuth for a specific toolkit
- GET /auth/composio/connector/callback - Handle OAuth callback
- GET /connectors/{connector_id}/composio-drive/folders - List folders/files for Composio Google Drive
"""
import asyncio
@ -369,3 +370,124 @@ async def composio_callback(
raise HTTPException(
status_code=500, detail=f"Failed to complete Composio OAuth: {e!s}"
) from e
@router.get("/connectors/{connector_id}/composio-drive/folders")
async def list_composio_drive_folders(
connector_id: int,
parent_id: str | None = None,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
List folders AND files in user's Google Drive via Composio with hierarchical support.
This is called at index time from the manage connector page to display
the complete file system (folders and files). Only folders are selectable.
Args:
connector_id: ID of the Composio Google Drive connector
parent_id: Optional parent folder ID to list contents (None for root)
Returns:
JSON with list of items: {
"items": [
{"id": str, "name": str, "mimeType": str, "isFolder": bool, ...},
...
]
}
"""
if not ComposioService.is_enabled():
raise HTTPException(
status_code=503,
detail="Composio integration is not enabled.",
)
try:
# Get connector and verify ownership
result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.id == connector_id,
SearchSourceConnector.user_id == user.id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR,
)
)
connector = result.scalars().first()
if not connector:
raise HTTPException(
status_code=404,
detail="Composio Google Drive connector not found or access denied",
)
# Get Composio connected account ID from config
composio_connected_account_id = connector.config.get("composio_connected_account_id")
if not composio_connected_account_id:
raise HTTPException(
status_code=400,
detail="Composio connected account not found. Please reconnect the connector.",
)
# Initialize Composio service and fetch files
service = ComposioService()
entity_id = f"surfsense_{user.id}"
# Fetch files/folders from Composio Google Drive
files, next_token, error = await service.get_drive_files(
connected_account_id=composio_connected_account_id,
entity_id=entity_id,
folder_id=parent_id,
page_size=100,
)
if error:
logger.error(f"Failed to list Composio Drive files: {error}")
raise HTTPException(
status_code=500, detail=f"Failed to list folder contents: {error}"
)
# Transform files to match the expected format with isFolder field
items = []
for file_info in files:
file_id = file_info.get("id", "") or file_info.get("fileId", "")
file_name = file_info.get("name", "") or file_info.get("fileName", "") or "Untitled"
mime_type = file_info.get("mimeType", "") or file_info.get("mime_type", "")
if not file_id:
continue
is_folder = mime_type == "application/vnd.google-apps.folder"
items.append({
"id": file_id,
"name": file_name,
"mimeType": mime_type,
"isFolder": is_folder,
"parents": file_info.get("parents", []),
"size": file_info.get("size"),
"iconLink": file_info.get("iconLink"),
})
# Sort: folders first, then files, both alphabetically
folders = sorted([item for item in items if item["isFolder"]], key=lambda x: x["name"].lower())
files_list = sorted([item for item in items if not item["isFolder"]], key=lambda x: x["name"].lower())
items = folders + files_list
folder_count = len(folders)
file_count = len(files_list)
logger.info(
f"✅ Listed {len(items)} total items ({folder_count} folders, {file_count} files) for Composio connector {connector_id}"
+ (f" in folder {parent_id}" if parent_id else " in ROOT")
)
return {"items": items}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error listing Composio Drive contents: {e!s}", exc_info=True)
raise HTTPException(
status_code=500, detail=f"Failed to list Drive contents: {e!s}"
) from e

40
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -897,8 +897,46 @@ async def index_connector_content(
)
response_message = "Web page indexing started in the background."
elif connector.connector_type == SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR:
from app.tasks.celery_tasks.connector_tasks import (
index_composio_connector_task,
)
# For Composio Google Drive, if drive_items is provided, update connector config
# This allows the UI to pass folder/file selection like the regular Google Drive connector
if drive_items and drive_items.has_items():
# Update connector config with the selected folders/files
config = connector.config or {}
config["selected_folders"] = [{"id": f.id, "name": f.name} for f in drive_items.folders]
config["selected_files"] = [{"id": f.id, "name": f.name} for f in drive_items.files]
if drive_items.indexing_options:
config["indexing_options"] = {
"max_files_per_folder": drive_items.indexing_options.max_files_per_folder,
"incremental_sync": drive_items.indexing_options.incremental_sync,
"include_subfolders": drive_items.indexing_options.include_subfolders,
}
connector.config = config
from sqlalchemy.orm.attributes import flag_modified
flag_modified(connector, "config")
await session.commit()
await session.refresh(connector)
logger.info(
f"Triggering Composio Google Drive indexing for connector {connector_id} into search space {search_space_id}, "
f"folders: {len(drive_items.folders)}, files: {len(drive_items.files)}"
)
else:
logger.info(
f"Triggering Composio Google Drive indexing for connector {connector_id} into search space {search_space_id} "
f"using existing config (from {indexing_from} to {indexing_to})"
)
index_composio_connector_task.delay(
connector_id, search_space_id, str(user.id), indexing_from, indexing_to
)
response_message = "Composio Google Drive indexing started in the background."
elif connector.connector_type in [
SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR,
SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR,
SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR,
]:

6
surfsense_backend/app/services/composio_service.py
View file

@ -397,7 +397,11 @@ class ComposioService:
"page_size": min(page_size, 100),
}
if folder_id:
params["folder_id"] = folder_id
# List contents of a specific folder (exclude shortcuts - we don't have access to them)
params["q"] = f"'{folder_id}' in parents and trashed = false and mimeType != 'application/vnd.google-apps.shortcut'"
else:
# List root-level items only (My Drive root), exclude shortcuts
params["q"] = "'root' in parents and trashed = false and mimeType != 'application/vnd.google-apps.shortcut'"
if page_token:
params["page_token"] = page_token

195
surfsense_backend/app/tasks/composio_indexer.py
View file

@ -252,37 +252,123 @@ async def _index_composio_google_drive(
update_last_indexed: bool = True,
max_items: int = 1000,
) -> tuple[int, str]:
"""Index Google Drive files via Composio."""
"""Index Google Drive files via Composio.
Supports folder/file selection via connector config:
- selected_folders: List of {id, name} for folders to index
- selected_files: List of {id, name} for individual files to index
- indexing_options: {max_files_per_folder, incremental_sync, include_subfolders}
"""
try:
composio_connector = ComposioConnector(session, connector_id)
connector_config = await composio_connector.get_config()
# Get folder/file selection configuration
selected_folders = connector_config.get("selected_folders", [])
selected_files = connector_config.get("selected_files", [])
indexing_options = connector_config.get("indexing_options", {})
max_files_per_folder = indexing_options.get("max_files_per_folder", 100)
include_subfolders = indexing_options.get("include_subfolders", True)
await task_logger.log_task_progress(
log_entry,
f"Fetching Google Drive files via Composio for connector {connector_id}",
{"stage": "fetching_files"},
{"stage": "fetching_files", "selected_folders": len(selected_folders), "selected_files": len(selected_files)},
)
# Fetch files
all_files = []
page_token = None
while len(all_files) < max_items:
files, next_token, error = await composio_connector.list_drive_files(
page_token=page_token,
page_size=min(100, max_items - len(all_files)),
)
# If specific folders/files are selected, fetch from those
if selected_folders or selected_files:
# Fetch files from selected folders
for folder in selected_folders:
folder_id = folder.get("id")
folder_name = folder.get("name", "Unknown")
if not folder_id:
continue
# Handle special case for "root" folder
actual_folder_id = None if folder_id == "root" else folder_id
logger.info(f"Fetching files from folder: {folder_name} ({folder_id})")
# Fetch files from this folder
folder_files = []
page_token = None
while len(folder_files) < max_files_per_folder:
files, next_token, error = await composio_connector.list_drive_files(
folder_id=actual_folder_id,
page_token=page_token,
page_size=min(100, max_files_per_folder - len(folder_files)),
)
if error:
await task_logger.log_task_failure(
log_entry, f"Failed to fetch Drive files: {error}", {}
if error:
logger.warning(f"Failed to fetch files from folder {folder_name}: {error}")
break
# Process files
for file_info in files:
mime_type = file_info.get("mimeType", "") or file_info.get("mime_type", "")
# If it's a folder and include_subfolders is enabled, recursively fetch
if mime_type == "application/vnd.google-apps.folder":
if include_subfolders:
# Add subfolder files recursively
subfolder_files = await _fetch_folder_files_recursively(
composio_connector,
file_info.get("id"),
max_files=max_files_per_folder,
current_count=len(folder_files),
)
folder_files.extend(subfolder_files)
else:
folder_files.append(file_info)
if not next_token:
break
page_token = next_token
all_files.extend(folder_files[:max_files_per_folder])
logger.info(f"Found {len(folder_files)} files in folder {folder_name}")
# Add specifically selected files
for selected_file in selected_files:
file_id = selected_file.get("id")
file_name = selected_file.get("name", "Unknown")
if not file_id:
continue
# Add file info (we'll fetch content later during indexing)
all_files.append({
"id": file_id,
"name": file_name,
"mimeType": "", # Will be determined later
})
else:
# No selection specified - fetch all files (original behavior)
page_token = None
while len(all_files) < max_items:
files, next_token, error = await composio_connector.list_drive_files(
page_token=page_token,
page_size=min(100, max_items - len(all_files)),
)
return 0, f"Failed to fetch Drive files: {error}"
all_files.extend(files)
if error:
await task_logger.log_task_failure(
log_entry, f"Failed to fetch Drive files: {error}", {}
)
return 0, f"Failed to fetch Drive files: {error}"
if not next_token:
break
page_token = next_token
all_files.extend(files)
if not next_token:
break
page_token = next_token
if not all_files:
success_msg = "No Google Drive files found"
@ -479,6 +565,81 @@ async def _index_composio_google_drive(
return 0, f"Failed to index Google Drive via Composio: {e!s}"
async def _fetch_folder_files_recursively(
composio_connector: ComposioConnector,
folder_id: str,
max_files: int = 100,
current_count: int = 0,
depth: int = 0,
max_depth: int = 10,
) -> list[dict[str, Any]]:
"""
Recursively fetch files from a Google Drive folder via Composio.
Args:
composio_connector: The Composio connector instance
folder_id: Google Drive folder ID
max_files: Maximum number of files to fetch
current_count: Current number of files already fetched
depth: Current recursion depth
max_depth: Maximum recursion depth to prevent infinite loops
Returns:
List of file info dictionaries
"""
if depth >= max_depth:
logger.warning(f"Max recursion depth reached for folder {folder_id}")
return []
if current_count >= max_files:
return []
all_files = []
page_token = None
try:
while len(all_files) + current_count < max_files:
files, next_token, error = await composio_connector.list_drive_files(
folder_id=folder_id,
page_token=page_token,
page_size=min(100, max_files - len(all_files) - current_count),
)
if error:
logger.warning(f"Error fetching files from subfolder {folder_id}: {error}")
break
for file_info in files:
mime_type = file_info.get("mimeType", "") or file_info.get("mime_type", "")
if mime_type == "application/vnd.google-apps.folder":
# Recursively fetch from subfolders
subfolder_files = await _fetch_folder_files_recursively(
composio_connector,
file_info.get("id"),
max_files=max_files,
current_count=current_count + len(all_files),
depth=depth + 1,
max_depth=max_depth,
)
all_files.extend(subfolder_files)
else:
all_files.append(file_info)
if len(all_files) + current_count >= max_files:
break
if not next_token:
break
page_token = next_token
return all_files[:max_files - current_count]
except Exception as e:
logger.error(f"Error in recursive folder fetch: {e!s}")
return all_files
async def _process_gmail_message_batch(
session: AsyncSession,
messages: list[dict[str, Any]],

294
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/composio-config.tsx
View file

@ -1,7 +1,20 @@
"use client";
import { File, FileSpreadsheet, FileText, FolderClosed, Image, Presentation } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { ComposioDriveFolderTree } from "@/components/connectors/composio-drive-folder-tree";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { cn } from "@/lib/utils";
@ -11,11 +24,134 @@ interface ComposioConfigProps {
onNameChange?: (name: string) => void;
}
export const ComposioConfig: FC<ComposioConfigProps> = ({ connector }) => {
interface SelectedFolder {
id: string;
name: string;
}
interface IndexingOptions {
max_files_per_folder: number;
incremental_sync: boolean;
include_subfolders: boolean;
}
const DEFAULT_INDEXING_OPTIONS: IndexingOptions = {
max_files_per_folder: 100,
incremental_sync: true,
include_subfolders: true,
};
// Helper to get appropriate icon for file type based on file name
function getFileIconFromName(fileName: string, className: string = "size-3.5 shrink-0") {
const lowerName = fileName.toLowerCase();
// Spreadsheets
if (
lowerName.endsWith(".xlsx") ||
lowerName.endsWith(".xls") ||
lowerName.endsWith(".csv") ||
lowerName.includes("spreadsheet")
) {
return <FileSpreadsheet className={`${className} text-green-500`} />;
}
// Presentations
if (
lowerName.endsWith(".pptx") ||
lowerName.endsWith(".ppt") ||
lowerName.includes("presentation")
) {
return <Presentation className={`${className} text-orange-500`} />;
}
// Documents (word, text only - not PDF)
if (
lowerName.endsWith(".docx") ||
lowerName.endsWith(".doc") ||
lowerName.endsWith(".txt") ||
lowerName.includes("document") ||
lowerName.includes("word") ||
lowerName.includes("text")
) {
return <FileText className={`${className} text-gray-500`} />;
}
// Images
if (
lowerName.endsWith(".png") ||
lowerName.endsWith(".jpg") ||
lowerName.endsWith(".jpeg") ||
lowerName.endsWith(".gif") ||
lowerName.endsWith(".webp") ||
lowerName.endsWith(".svg")
) {
return <Image className={`${className} text-purple-500`} />;
}
// Default (including PDF)
return <File className={`${className} text-gray-500`} />;
}
export const ComposioConfig: FC<ComposioConfigProps> = ({ connector, onConfigChange }) => {
const toolkitId = connector.config?.toolkit_id as string;
const isIndexable = connector.config?.is_indexable as boolean;
const composioAccountId = connector.config?.composio_connected_account_id as string;
// Check if this is a Google Drive Composio connector
const isGoogleDrive = toolkitId === "googledrive";
// Initialize with existing selected folders and files from connector config
const existingFolders =
(connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const existingFiles = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const existingIndexingOptions =
(connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS;
const [selectedFolders, setSelectedFolders] = useState<SelectedFolder[]>(existingFolders);
const [selectedFiles, setSelectedFiles] = useState<SelectedFolder[]>(existingFiles);
const [showFolderSelector, setShowFolderSelector] = useState(false);
const [indexingOptions, setIndexingOptions] = useState<IndexingOptions>(existingIndexingOptions);
// Update selected folders and files when connector config changes
useEffect(() => {
const folders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const files = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const options =
(connector.config?.indexing_options as IndexingOptions | undefined) ||
DEFAULT_INDEXING_OPTIONS;
setSelectedFolders(folders);
setSelectedFiles(files);
setIndexingOptions(options);
}, [connector.config]);
const updateConfig = (
folders: SelectedFolder[],
files: SelectedFolder[],
options: IndexingOptions
) => {
if (onConfigChange) {
onConfigChange({
...connector.config,
selected_folders: folders,
selected_files: files,
indexing_options: options,
});
}
};
const handleSelectFolders = (folders: SelectedFolder[]) => {
setSelectedFolders(folders);
updateConfig(folders, selectedFiles, indexingOptions);
};
const handleSelectFiles = (files: SelectedFolder[]) => {
setSelectedFiles(files);
updateConfig(selectedFolders, files, indexingOptions);
};
const handleIndexingOptionChange = (key: keyof IndexingOptions, value: number | boolean) => {
const newOptions = { ...indexingOptions, [key]: value };
setIndexingOptions(newOptions);
updateConfig(selectedFolders, selectedFiles, newOptions);
};
const totalSelected = selectedFolders.length + selectedFiles.length;
return (
<div className="space-y-6">
{/* Connection Details */}
@ -52,6 +188,162 @@ export const ComposioConfig: FC<ComposioConfigProps> = ({ connector }) => {
)}
</div>
</div>
{/* Google Drive specific: Folder & File Selection */}
{isGoogleDrive && isIndexable && (
<>
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Folder & File Selection</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Select specific folders and/or individual files to index.
</p>
</div>
{totalSelected > 0 && (
<div className="p-2 sm:p-3 bg-muted rounded-lg text-xs sm:text-sm space-y-1 sm:space-y-2">
<p className="font-medium">
Selected {totalSelected} item{totalSelected > 1 ? "s" : ""}: {(() => {
const parts: string[] = [];
if (selectedFolders.length > 0) {
parts.push(
`${selectedFolders.length} folder${selectedFolders.length > 1 ? "s" : ""}`
);
}
if (selectedFiles.length > 0) {
parts.push(`${selectedFiles.length} file${selectedFiles.length > 1 ? "s" : ""}`);
}
return parts.length > 0 ? `(${parts.join(" ")})` : "";
})()}
</p>
<div className="max-h-20 sm:max-h-24 overflow-y-auto space-y-1">
{selectedFolders.map((folder) => (
<p
key={folder.id}
className="text-xs sm:text-sm text-muted-foreground truncate flex items-center gap-1.5"
title={folder.name}
>
<FolderClosed className="size-3.5 shrink-0 text-gray-500" />
{folder.name}
</p>
))}
{selectedFiles.map((file) => (
<p
key={file.id}
className="text-xs sm:text-sm text-muted-foreground truncate flex items-center gap-1.5"
title={file.name}
>
{getFileIconFromName(file.name)}
{file.name}
</p>
))}
</div>
</div>
)}
{showFolderSelector ? (
<div className="space-y-2 sm:space-y-3">
<ComposioDriveFolderTree
connectorId={connector.id}
selectedFolders={selectedFolders}
onSelectFolders={handleSelectFolders}
selectedFiles={selectedFiles}
onSelectFiles={handleSelectFiles}
/>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => setShowFolderSelector(false)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
Done Selecting
</Button>
</div>
) : (
<Button
type="button"
variant="outline"
onClick={() => setShowFolderSelector(true)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
{totalSelected > 0 ? "Change Selection" : "Select Folders & Files"}
</Button>
)}
</div>
{/* Indexing Options */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Indexing Options</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure how files are indexed from your Google Drive.
</p>
</div>
{/* Max files per folder */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="max-files" className="text-sm font-medium">
Max files per folder
</Label>
<p className="text-xs text-muted-foreground">
Maximum number of files to index from each folder
</p>
</div>
<Select
value={indexingOptions.max_files_per_folder.toString()}
onValueChange={(value) =>
handleIndexingOptionChange("max_files_per_folder", parseInt(value, 10))
}
>
<SelectTrigger
id="max-files"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select limit" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="50" className="text-xs sm:text-sm">
50 files
</SelectItem>
<SelectItem value="100" className="text-xs sm:text-sm">
100 files
</SelectItem>
<SelectItem value="250" className="text-xs sm:text-sm">
250 files
</SelectItem>
<SelectItem value="500" className="text-xs sm:text-sm">
500 files
</SelectItem>
<SelectItem value="1000" className="text-xs sm:text-sm">
1000 files
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Include subfolders toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="include-subfolders" className="text-sm font-medium">
Include subfolders
</Label>
<p className="text-xs text-muted-foreground">
Recursively index files in subfolders of selected folders
</p>
</div>
<Switch
id="include-subfolders"
checked={indexingOptions.include_subfolders}
onCheckedChange={(checked) => handleIndexingOptionChange("include_subfolders", checked)}
/>
</div>
</div>
</>
)}
</div>
);
};

7
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -224,8 +224,11 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{/* Periodic sync - shown for all indexable connectors */}
{(() => {
// Check if Google Drive has folders/files selected
// Check if Google Drive (regular or Composio) has folders/files selected
const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR";
const isComposioGoogleDrive =
connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR";
const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive;
const selectedFolders =
(connector.config?.selected_folders as
| Array<{ id: string; name: string }>
@ -235,7 +238,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
| Array<{ id: string; name: string }>
| undefined) || [];
const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0;
const isDisabled = isGoogleDrive && !hasItemsSelected;
const isDisabled = requiresFolderSelection && !hasItemsSelected;
return (
<PeriodicSyncConfig

8
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -1130,8 +1130,12 @@ export const useConnectorDialog = () => {
return;
}
// Prevent periodic indexing for Google Drive without folders/files selected
if (periodicEnabled && editingConnector.connector_type === "GOOGLE_DRIVE_CONNECTOR") {
// Prevent periodic indexing for Google Drive (regular or Composio) without folders/files selected
if (
periodicEnabled &&
(editingConnector.connector_type === "GOOGLE_DRIVE_CONNECTOR" ||
editingConnector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR")
) {
const selectedFolders = (connectorConfig || editingConnector.config)?.selected_folders as
| Array<{ id: string; name: string }>
| undefined;

365
surfsense_web/components/connectors/composio-drive-folder-tree.tsx Normal file
View file

@ -0,0 +1,365 @@
"use client";
import {
ChevronDown,
ChevronRight,
File,
FileSpreadsheet,
FileText,
FolderClosed,
FolderOpen,
HardDrive,
Image,
Loader2,
Presentation,
} from "lucide-react";
import { useState } from "react";
import { Checkbox } from "@/components/ui/checkbox";
import { ScrollArea } from "@/components/ui/scroll-area";
import { useComposioDriveFolders } from "@/hooks/use-composio-drive-folders";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
import { cn } from "@/lib/utils";
interface DriveItem {
id: string;
name: string;
mimeType: string;
isFolder: boolean;
parents?: string[];
size?: number;
iconLink?: string;
}
interface ItemTreeNode {
item: DriveItem;
children: DriveItem[] | null; // null = not loaded, [] = loaded but empty
isExpanded: boolean;
isLoading: boolean;
}
interface SelectedFolder {
id: string;
name: string;
}
interface ComposioDriveFolderTreeProps {
connectorId: number;
selectedFolders: SelectedFolder[];
onSelectFolders: (folders: SelectedFolder[]) => void;
selectedFiles?: SelectedFolder[];
onSelectFiles?: (files: SelectedFolder[]) => void;
}
// Helper to get appropriate icon for file type
function getFileIcon(mimeType: string, className: string = "h-4 w-4") {
if (mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
return <FileSpreadsheet className={`${className} text-green-500`} />;
}
if (mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
return <Presentation className={`${className} text-orange-500`} />;
}
if (mimeType.includes("document") || mimeType.includes("word") || mimeType.includes("text")) {
return <FileText className={`${className} text-gray-500`} />;
}
if (mimeType.includes("image")) {
return <Image className={`${className} text-purple-500`} />;
}
return <File className={`${className} text-gray-500`} />;
}
export function ComposioDriveFolderTree({
connectorId,
selectedFolders,
onSelectFolders,
selectedFiles = [],
onSelectFiles = () => {},
}: ComposioDriveFolderTreeProps) {
const [itemStates, setItemStates] = useState<Map<string, ItemTreeNode>>(new Map());
const { data: rootData, isLoading: isLoadingRoot } = useComposioDriveFolders({
connectorId,
});
const rootItems = rootData?.items || [];
const isFolderSelected = (folderId: string): boolean => {
return selectedFolders.some((f) => f.id === folderId);
};
const isFileSelected = (fileId: string): boolean => {
return selectedFiles.some((f) => f.id === fileId);
};
const toggleFolderSelection = (folderId: string, folderName: string) => {
if (isFolderSelected(folderId)) {
onSelectFolders(selectedFolders.filter((f) => f.id !== folderId));
} else {
onSelectFolders([...selectedFolders, { id: folderId, name: folderName }]);
}
};
const toggleFileSelection = (fileId: string, fileName: string) => {
if (isFileSelected(fileId)) {
onSelectFiles(selectedFiles.filter((f) => f.id !== fileId));
} else {
onSelectFiles([...selectedFiles, { id: fileId, name: fileName }]);
}
};
/**
* Find an item by ID across all loaded items (root and nested).
*/
const findItem = (itemId: string): DriveItem | undefined => {
const state = itemStates.get(itemId);
if (state?.item) return state.item;
const rootItem = rootItems.find((item) => item.id === itemId);
if (rootItem) return rootItem;
for (const [, nodeState] of itemStates) {
if (nodeState.children) {
const found = nodeState.children.find((child) => child.id === itemId);
if (found) return found;
}
}
return undefined;
};
/**
* Load and display contents of a specific folder.
*/
const loadFolderContents = async (folderId: string) => {
try {
setItemStates((prev) => {
const newMap = new Map(prev);
const existing = newMap.get(folderId);
if (existing) {
newMap.set(folderId, { ...existing, isLoading: true });
} else {
const item = findItem(folderId);
if (item) {
newMap.set(folderId, {
item,
children: null,
isExpanded: false,
isLoading: true,
});
}
}
return newMap;
});
const data = await connectorsApiService.listComposioDriveFolders({
connector_id: connectorId,
parent_id: folderId,
});
const items = data.items || [];
setItemStates((prev) => {
const newMap = new Map(prev);
const existing = newMap.get(folderId);
const item = existing?.item || findItem(folderId);
if (item) {
newMap.set(folderId, {
item,
children: items,
isExpanded: true,
isLoading: false,
});
} else {
console.error(`Could not find item for folderId: ${folderId}`);
}
return newMap;
});
} catch (error) {
console.error("Error loading folder contents:", error);
setItemStates((prev) => {
const newMap = new Map(prev);
const existing = newMap.get(folderId);
if (existing) {
newMap.set(folderId, { ...existing, isLoading: false });
}
return newMap;
});
}
};
/**
* Toggle folder expand/collapse state.
*/
const toggleFolder = async (item: DriveItem) => {
if (!item.isFolder) return;
const state = itemStates.get(item.id);
if (!state || state.children === null) {
await loadFolderContents(item.id);
} else {
setItemStates((prev) => {
const newMap = new Map(prev);
newMap.set(item.id, {
...state,
isExpanded: !state.isExpanded,
});
return newMap;
});
}
};
/**
* Render a single item (folder or file) with its children.
*/
const renderItem = (item: DriveItem, level: number = 0) => {
const state = itemStates.get(item.id);
const isExpanded = state?.isExpanded || false;
const isLoading = state?.isLoading || false;
const children = state?.children;
const isFolder = item.isFolder;
const isSelected = isFolder ? isFolderSelected(item.id) : isFileSelected(item.id);
const childFolders = children?.filter((c) => c.isFolder) || [];
const childFiles = children?.filter((c) => !c.isFolder) || [];
const indentSize = 0.75; // Smaller indent for mobile
return (
<div
key={item.id}
className="w-full sm:ml-[calc(var(--level)*1.25rem)]"
style={
{ marginLeft: `${level * indentSize}rem`, "--level": level } as React.CSSProperties & {
"--level"?: number;
}
}
>
<div
className={cn(
"flex items-center group gap-1 sm:gap-2 h-auto py-1 sm:py-2 px-1 sm:px-2 rounded-md",
isFolder && "hover:bg-accent cursor-pointer",
!isFolder && "cursor-default opacity-60",
isSelected && "bg-accent/50"
)}
>
{isFolder ? (
<button
type="button"
className="flex items-center justify-center w-3 h-3 sm:w-4 sm:h-4 shrink-0 bg-transparent border-0 p-0 cursor-pointer"
onClick={(e) => {
e.stopPropagation();
toggleFolder(item);
}}
aria-label={isExpanded ? `Collapse ${item.name}` : `Expand ${item.name}`}
>
{isLoading ? (
<Loader2 className="h-2.5 w-2.5 sm:h-3 sm:w-3 animate-spin" />
) : isExpanded ? (
<ChevronDown className="h-3 w-3 sm:h-4 sm:w-4" />
) : (
<ChevronRight className="h-3 w-3 sm:h-4 sm:w-4" />
)}
</button>
) : (
<span className="w-3 h-3 sm:w-4 sm:h-4 shrink-0" />
)}
<Checkbox
checked={isSelected}
onCheckedChange={() => {
if (isFolder) {
toggleFolderSelection(item.id, item.name);
} else {
toggleFileSelection(item.id, item.name);
}
}}
className="shrink-0 h-3.5 w-3.5 sm:h-4 sm:w-4 border-slate-400/20 dark:border-white/20"
onClick={(e) => e.stopPropagation()}
/>
<div className="shrink-0">
{isFolder ? (
isExpanded ? (
<FolderOpen className="h-3 w-3 sm:h-4 sm:w-4 text-gray-500" />
) : (
<FolderClosed className="h-3 w-3 sm:h-4 sm:w-4 text-gray-500" />
)
) : (
getFileIcon(item.mimeType, "h-3 w-3 sm:h-4 sm:w-4")
)}
</div>
{isFolder ? (
<button
type="button"
className="truncate flex-1 text-left text-xs sm:text-sm min-w-0 bg-transparent border-0 p-0 cursor-pointer"
onClick={() => toggleFolder(item)}
>
{item.name}
</button>
) : (
<span className="truncate flex-1 text-left text-xs sm:text-sm min-w-0">
{item.name}
</span>
)}
</div>
{isExpanded && isFolder && children && (
<div className="w-full">
{childFolders.map((child) => renderItem(child, level + 1))}
{childFiles.map((child) => renderItem(child, level + 1))}
{children.length === 0 && (
<div className="text-[10px] sm:text-xs text-muted-foreground py-1 sm:py-2 pl-1 sm:pl-2">
Empty folder
</div>
)}
</div>
)}
</div>
);
};
return (
<div className="border border-slate-400/20 dark:border-white/20 rounded-md w-full overflow-hidden">
<ScrollArea className="h-[300px] sm:h-[450px] w-full">
<div className="p-1 sm:p-2 pr-2 sm:pr-4 w-full overflow-x-hidden">
<div className="mb-1 sm:mb-2 pb-1 sm:pb-2 border-b border-slate-400/20 dark:border-white/20">
<div className="flex items-center gap-1 sm:gap-2 h-auto py-1 sm:py-2 px-1 sm:px-2 rounded-md hover:bg-accent cursor-pointer">
<Checkbox
checked={isFolderSelected("root")}
onCheckedChange={() => toggleFolderSelection("root", "My Drive")}
className="shrink-0 h-3.5 w-3.5 sm:h-4 sm:w-4 border-slate-400/20 dark:border-white/20"
/>
<HardDrive className="h-3 w-3 sm:h-4 sm:w-4 text-primary shrink-0" />
<button
type="button"
className="font-semibold truncate text-xs sm:text-sm cursor-pointer bg-transparent border-0 p-0 text-left"
onClick={() => toggleFolderSelection("root", "My Drive")}
>
My Drive
</button>
</div>
</div>
{isLoadingRoot && (
<div className="flex items-center justify-center py-4 sm:py-8">
<Loader2 className="h-4 w-4 sm:h-6 sm:w-6 animate-spin text-muted-foreground" />
</div>
)}
<div className="w-full overflow-x-hidden">
{!isLoadingRoot && rootItems.map((item) => renderItem(item, 0))}
</div>
{!isLoadingRoot && rootItems.length === 0 && (
<div className="text-center text-xs sm:text-sm text-muted-foreground py-4 sm:py-8">
No files or folders found in your Google Drive
</div>
)}
</div>
</ScrollArea>
</div>
);
}

29
surfsense_web/hooks/use-composio-drive-folders.ts Normal file
View file

@ -0,0 +1,29 @@
import { useQuery } from "@tanstack/react-query";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
interface UseComposioDriveFoldersOptions {
connectorId: number;
parentId?: string;
enabled?: boolean;
}
export function useComposioDriveFolders({
connectorId,
parentId,
enabled = true,
}: UseComposioDriveFoldersOptions) {
return useQuery({
queryKey: cacheKeys.connectors.composioDrive.folders(connectorId, parentId),
queryFn: async () => {
return connectorsApiService.listComposioDriveFolders({
connector_id: connectorId,
parent_id: parentId,
});
},
enabled: enabled && !!connectorId,
staleTime: 5 * 60 * 1000, // 5 minutes
retry: 2,
});
}

23
surfsense_web/lib/apis/connectors-api.service.ts
View file

@ -233,6 +233,29 @@ class ConnectorsApiService {
);
};
/**
* List Composio Google Drive folders and files
*/
listComposioDriveFolders = async (request: ListGoogleDriveFoldersRequest) => {
const parsedRequest = listGoogleDriveFoldersRequest.safeParse(request);
if (!parsedRequest.success) {
console.error("Invalid request:", parsedRequest.error);
const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
throw new ValidationError(`Invalid request: ${errorMessage}`);
}
const { connector_id, parent_id } = parsedRequest.data;
const queryParams = parent_id ? `?parent_id=${encodeURIComponent(parent_id)}` : "";
return baseApiService.get(
`/api/v1/connectors/${connector_id}/composio-drive/folders${queryParams}`,
listGoogleDriveFoldersResponse
);
};
// =============================================================================
// MCP Connector Methods
// =============================================================================

4
surfsense_web/lib/query-client/cache-keys.ts
View file

@ -71,6 +71,10 @@ export const cacheKeys = {
folders: (connectorId: number, parentId?: string) =>
["connectors", "google-drive", connectorId, "folders", parentId] as const,
},
composioDrive: {
folders: (connectorId: number, parentId?: string) =>
["connectors", "composio-drive", connectorId, "folders", parentId] as const,
},
},
comments: {
byMessage: (messageId: number) => ["comments", "message", messageId] as const,