apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: enhance Google Drive indexing with new options

Browse source 
- Updated the Google Drive indexing functionality to include indexing options such as max files per folder, incremental sync, and inclusion of subfolders.
- Modified the API to accept a new 'indexing_options' parameter in the request body.
- Enhanced the UI to allow users to configure these options when selecting folders and files for indexing.
- Updated related components and tasks to support the new indexing options, ensuring a more flexible and efficient indexing process.
This commit is contained in:
Anish Sarkar 2026-01-17 12:33:57 +05:30
parent cf53338119
commit a3112a24fe
9 changed files with 381 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

9
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -1716,7 +1716,7 @@ async def run_google_drive_indexing(
connector_id: int,
search_space_id: int,
user_id: str,
items_dict: dict, # Dictionary with 'folders' and 'files' lists
items_dict: dict, # Dictionary with 'folders', 'files', and 'indexing_options'
):
"""Runs the Google Drive indexing task for folders and files with notifications."""
from uuid import UUID
@ -1730,6 +1730,7 @@ async def run_google_drive_indexing(
# Parse the structured data
items = GoogleDriveIndexRequest(**items_dict)
indexing_options = items.indexing_options
total_indexed = 0
errors = []
@ -1765,7 +1766,7 @@ async def run_google_drive_indexing(
stage="fetching",
)
# Index each folder
# Index each folder with indexing options
for folder in items.folders:
try:
indexed_count, error_message = await index_google_drive_files(
@ -1775,8 +1776,10 @@ async def run_google_drive_indexing(
user_id,
folder_id=folder.id,
folder_name=folder.name,
use_delta_sync=True,
use_delta_sync=indexing_options.incremental_sync,
update_last_indexed=False,
max_files=indexing_options.max_files_per_folder,
include_subfolders=indexing_options.include_subfolders,
)
if error_message:
errors.append(f"Folder '{folder.name}': {error_message}")

3
surfsense_backend/app/schemas/__init__.py
View file

@ -10,7 +10,7 @@ from .documents import (
ExtensionDocumentMetadata,
PaginatedResponse,
)
from .google_drive import DriveItem, GoogleDriveIndexRequest
from .google_drive import DriveItem, GoogleDriveIndexingOptions, GoogleDriveIndexRequest
from .logs import LogBase, LogCreate, LogFilter, LogRead, LogUpdate
from .new_chat import (
ChatMessage,
@ -90,6 +90,7 @@ __all__ = [
"DocumentsCreate",
# Google Drive schemas
"DriveItem",
"GoogleDriveIndexingOptions",
"ExtensionDocumentContent",
"ExtensionDocumentMetadata",
"GlobalNewLLMConfigRead",

23
surfsense_backend/app/schemas/google_drive.py
View file

@ -10,6 +10,25 @@ class DriveItem(BaseModel):
name: str = Field(..., description="Item display name")
class GoogleDriveIndexingOptions(BaseModel):
"""Indexing options for Google Drive connector."""
max_files_per_folder: int = Field(
default=100,
ge=1,
le=1000,
description="Maximum number of files to index from each folder (1-1000)",
)
incremental_sync: bool = Field(
default=True,
description="Only sync changes since last index (faster). Disable for a full re-index.",
)
include_subfolders: bool = Field(
default=True,
description="Recursively index files in subfolders of selected folders",
)
class GoogleDriveIndexRequest(BaseModel):
"""Request body for indexing Google Drive content."""
@ -19,6 +38,10 @@ class GoogleDriveIndexRequest(BaseModel):
files: list[DriveItem] = Field(
default_factory=list, description="List of specific files to index"
)
indexing_options: GoogleDriveIndexingOptions = Field(
default_factory=GoogleDriveIndexingOptions,
description="Indexing configuration options",
)
def has_items(self) -> bool:
"""Check if any items are selected."""

4
surfsense_backend/app/tasks/celery_tasks/connector_tasks.py
View file

@ -461,7 +461,7 @@ def index_google_drive_files_task(
connector_id: int,
search_space_id: int,
user_id: str,
items_dict: dict, # Dictionary with 'folders' and 'files' lists
items_dict: dict, # Dictionary with 'folders', 'files', and 'indexing_options'
):
"""Celery task to index Google Drive folders and files."""
import asyncio
@ -486,7 +486,7 @@ async def _index_google_drive_files(
connector_id: int,
search_space_id: int,
user_id: str,
items_dict: dict, # Dictionary with 'folders' and 'files' lists
items_dict: dict, # Dictionary with 'folders', 'files', and 'indexing_options'
):
"""Index Google Drive folders and files with new session."""
from app.routes.search_source_connectors_routes import (

45
surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py
View file

@ -72,6 +72,7 @@ async def _check_and_trigger_schedules():
index_elasticsearch_documents_task,
index_github_repos_task,
index_google_calendar_events_task,
index_google_drive_files_task,
index_google_gmail_messages_task,
index_jira_issues_task,
index_linear_issues_task,
@ -96,6 +97,7 @@ async def _check_and_trigger_schedules():
SearchSourceConnectorType.LUMA_CONNECTOR: index_luma_events_task,
SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: index_elasticsearch_documents_task,
SearchSourceConnectorType.WEBCRAWLER_CONNECTOR: index_crawled_urls_task,
SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR: index_google_drive_files_task,
}
# Trigger indexing for each due connector
@ -106,13 +108,42 @@ async def _check_and_trigger_schedules():
f"Triggering periodic indexing for connector {connector.id} "
f"({connector.connector_type.value})"
)
task.delay(
connector.id,
connector.search_space_id,
str(connector.user_id),
None, # start_date - uses last_indexed_at
None, # end_date - uses now
)
# Special handling for Google Drive - uses config for folder/file selection
if connector.connector_type == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR:
config = connector.config or {}
selected_folders = config.get("selected_folders", [])
selected_files = config.get("selected_files", [])
indexing_options = config.get("indexing_options", {
"max_files_per_folder": 100,
"incremental_sync": True,
"include_subfolders": True,
})
if selected_folders or selected_files:
task.delay(
connector.id,
connector.search_space_id,
str(connector.user_id),
{
"folders": selected_folders,
"files": selected_files,
"indexing_options": indexing_options,
},
)
else:
logger.warning(
f"Google Drive connector {connector.id} has no folders or files selected, skipping periodic indexing"
)
continue
else:
task.delay(
connector.id,
connector.search_space_id,
str(connector.user_id),
None, # start_date - uses last_indexed_at
None, # end_date - uses now
)
# Update next_scheduled_at for next run
from datetime import timedelta

105
surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py
View file

@ -37,6 +37,7 @@ async def index_google_drive_files(
use_delta_sync: bool = True,
update_last_indexed: bool = True,
max_files: int = 500,
include_subfolders: bool = False,
) -> tuple[int, str | None]:
"""
Index Google Drive files for a specific connector.
@ -51,6 +52,7 @@ async def index_google_drive_files(
use_delta_sync: Whether to use change tracking for incremental sync
update_last_indexed: Whether to update last_indexed_at timestamp
max_files: Maximum number of files to index
include_subfolders: Whether to recursively index files in subfolders
Returns:
Tuple of (number_of_indexed_files, error_message)
@ -144,6 +146,7 @@ async def index_google_drive_files(
task_logger=task_logger,
log_entry=log_entry,
max_files=max_files,
include_subfolders=include_subfolders,
)
else:
logger.info(f"Using full scan for connector {connector_id}")
@ -159,6 +162,7 @@ async def index_google_drive_files(
task_logger=task_logger,
log_entry=log_entry,
max_files=max_files,
include_subfolders=include_subfolders,
)
documents_indexed, documents_skipped = result
@ -375,60 +379,80 @@ async def _index_full_scan(
task_logger: TaskLoggingService,
log_entry: any,
max_files: int,
include_subfolders: bool = False,
) -> tuple[int, int]:
"""Perform full scan indexing of a folder."""
await task_logger.log_task_progress(
log_entry,
f"Starting full scan of folder: {folder_name}",
{"stage": "full_scan", "folder_id": folder_id},
f"Starting full scan of folder: {folder_name} (include_subfolders={include_subfolders})",
{"stage": "full_scan", "folder_id": folder_id, "include_subfolders": include_subfolders},
)
documents_indexed = 0
documents_skipped = 0
page_token = None
files_processed = 0
while files_processed < max_files:
files, next_token, error = await get_files_in_folder(
drive_client, folder_id, include_subfolders=False, page_token=page_token
)
# Queue of folders to process: (folder_id, folder_name)
folders_to_process = [(folder_id, folder_name)]
if error:
logger.error(f"Error listing files: {error}")
break
while folders_to_process and files_processed < max_files:
current_folder_id, current_folder_name = folders_to_process.pop(0)
logger.info(f"Processing folder: {current_folder_name} ({current_folder_id})")
page_token = None
if not files:
break
for file in files:
if files_processed >= max_files:
break
files_processed += 1
indexed, skipped = await _process_single_file(
drive_client=drive_client,
session=session,
file=file,
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
task_logger=task_logger,
log_entry=log_entry,
while files_processed < max_files:
# Get files and folders in current folder
# include_subfolders=True here so we get folder items to queue them
files, next_token, error = await get_files_in_folder(
drive_client, current_folder_id, include_subfolders=True, page_token=page_token
)
documents_indexed += indexed
documents_skipped += skipped
if error:
logger.error(f"Error listing files in {current_folder_name}: {error}")
break
if documents_indexed % 10 == 0 and documents_indexed > 0:
await session.commit()
logger.info(
f"Committed batch: {documents_indexed} files indexed so far"
if not files:
break
for file in files:
if files_processed >= max_files:
break
mime_type = file.get("mimeType", "")
# If this is a folder and include_subfolders is enabled, queue it for processing
if mime_type == "application/vnd.google-apps.folder":
if include_subfolders:
folders_to_process.append((file["id"], file.get("name", "Unknown")))
logger.debug(f"Queued subfolder: {file.get('name', 'Unknown')}")
continue
# Process the file
files_processed += 1
indexed, skipped = await _process_single_file(
drive_client=drive_client,
session=session,
file=file,
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
task_logger=task_logger,
log_entry=log_entry,
)
page_token = next_token
if not page_token:
break
documents_indexed += indexed
documents_skipped += skipped
if documents_indexed % 10 == 0 and documents_indexed > 0:
await session.commit()
logger.info(
f"Committed batch: {documents_indexed} files indexed so far"
)
page_token = next_token
if not page_token:
break
logger.info(
f"Full scan complete: {documents_indexed} indexed, {documents_skipped} skipped"
@ -448,8 +472,13 @@ async def _index_with_delta_sync(
task_logger: TaskLoggingService,
log_entry: any,
max_files: int,
include_subfolders: bool = False,
) -> tuple[int, int]:
"""Perform delta sync indexing using change tracking."""
"""Perform delta sync indexing using change tracking.
Note: include_subfolders is accepted for API consistency but delta sync
automatically tracks changes across all folders including subfolders.
"""
await task_logger.log_task_progress(
log_entry,
f"Starting delta sync from token: {start_page_token[:20]}...",

279
surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx
View file

@ -6,6 +6,15 @@ import { useEffect, useState } from "react";
import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import type { ConnectorConfigProps } from "../index";
interface SelectedFolder {
@ -13,128 +22,238 @@ interface SelectedFolder {
name: string;
}
interface IndexingOptions {
max_files_per_folder: number;
incremental_sync: boolean;
include_subfolders: boolean;
}
const DEFAULT_INDEXING_OPTIONS: IndexingOptions = {
max_files_per_folder: 100,
incremental_sync: true,
include_subfolders: true,
};
export const GoogleDriveConfig: FC<ConnectorConfigProps> = ({ connector, onConfigChange }) => {
// Initialize with existing selected folders and files from connector config
const existingFolders =
(connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const existingFiles = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const existingIndexingOptions = (connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS;
const [selectedFolders, setSelectedFolders] = useState<SelectedFolder[]>(existingFolders);
const [selectedFiles, setSelectedFiles] = useState<SelectedFolder[]>(existingFiles);
const [showFolderSelector, setShowFolderSelector] = useState(false);
const [indexingOptions, setIndexingOptions] = useState<IndexingOptions>(existingIndexingOptions);
// Update selected folders and files when connector config changes
useEffect(() => {
const folders = (connector.config?.selected_folders as SelectedFolder[] | undefined) || [];
const files = (connector.config?.selected_files as SelectedFolder[] | undefined) || [];
const options = (connector.config?.indexing_options as IndexingOptions | undefined) || DEFAULT_INDEXING_OPTIONS;
setSelectedFolders(folders);
setSelectedFiles(files);
setIndexingOptions(options);
}, [connector.config]);
const handleSelectFolders = (folders: SelectedFolder[]) => {
setSelectedFolders(folders);
const updateConfig = (
folders: SelectedFolder[],
files: SelectedFolder[],
options: IndexingOptions
) => {
if (onConfigChange) {
// Store folder IDs and names in config for indexing
onConfigChange({
...connector.config,
selected_folders: folders,
selected_files: selectedFiles, // Preserve existing files
selected_files: files,
indexing_options: options,
});
}
};
const handleSelectFolders = (folders: SelectedFolder[]) => {
setSelectedFolders(folders);
updateConfig(folders, selectedFiles, indexingOptions);
};
const handleSelectFiles = (files: SelectedFolder[]) => {
setSelectedFiles(files);
if (onConfigChange) {
// Store file IDs and names in config for indexing
onConfigChange({
...connector.config,
selected_folders: selectedFolders, // Preserve existing folders
selected_files: files,
});
}
updateConfig(selectedFolders, files, indexingOptions);
};
const handleIndexingOptionChange = (key: keyof IndexingOptions, value: number | boolean) => {
const newOptions = { ...indexingOptions, [key]: value };
setIndexingOptions(newOptions);
updateConfig(selectedFolders, selectedFiles, newOptions);
};
const totalSelected = selectedFolders.length + selectedFiles.length;
return (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Folder & File Selection</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Select specific folders and/or individual files to index. Only files directly in each
folder will be processed—subfolders must be selected separately.
</p>
</div>
{totalSelected > 0 && (
<div className="p-2 sm:p-3 bg-muted rounded-lg text-xs sm:text-sm space-y-1 sm:space-y-2">
<p className="font-medium">
Selected {totalSelected} item{totalSelected > 1 ? "s" : ""}:
{selectedFolders.length > 0 &&
` ${selectedFolders.length} folder${selectedFolders.length > 1 ? "s" : ""}`}
{selectedFiles.length > 0 &&
` ${selectedFiles.length} file${selectedFiles.length > 1 ? "s" : ""}`}
<div className="space-y-4">
{/* Folder & File Selection */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Folder & File Selection</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Select specific folders and/or individual files to index.
</p>
<div className="max-h-20 sm:max-h-24 overflow-y-auto space-y-1">
{selectedFolders.map((folder) => (
<p
key={folder.id}
className="text-xs sm:text-sm text-muted-foreground truncate"
title={folder.name}
>
📁 {folder.name}
</p>
))}
{selectedFiles.map((file) => (
<p
key={file.id}
className="text-xs sm:text-sm text-muted-foreground truncate"
title={file.name}
>
📄 {file.name}
</p>
))}
</div>
</div>
)}
{showFolderSelector ? (
<div className="space-y-2 sm:space-y-3">
<GoogleDriveFolderTree
connectorId={connector.id}
selectedFolders={selectedFolders}
onSelectFolders={handleSelectFolders}
selectedFiles={selectedFiles}
onSelectFiles={handleSelectFiles}
/>
{totalSelected > 0 && (
<div className="p-2 sm:p-3 bg-muted rounded-lg text-xs sm:text-sm space-y-1 sm:space-y-2">
<p className="font-medium">
Selected {totalSelected} item{totalSelected > 1 ? "s" : ""}:
{selectedFolders.length > 0 &&
` ${selectedFolders.length} folder${selectedFolders.length > 1 ? "s" : ""}`}
{selectedFiles.length > 0 &&
` ${selectedFiles.length} file${selectedFiles.length > 1 ? "s" : ""}`}
</p>
<div className="max-h-20 sm:max-h-24 overflow-y-auto space-y-1">
{selectedFolders.map((folder) => (
<p
key={folder.id}
className="text-xs sm:text-sm text-muted-foreground truncate"
title={folder.name}
>
📁 {folder.name}
</p>
))}
{selectedFiles.map((file) => (
<p
key={file.id}
className="text-xs sm:text-sm text-muted-foreground truncate"
title={file.name}
>
📄 {file.name}
</p>
))}
</div>
</div>
)}
{showFolderSelector ? (
<div className="space-y-2 sm:space-y-3">
<GoogleDriveFolderTree
connectorId={connector.id}
selectedFolders={selectedFolders}
onSelectFolders={handleSelectFolders}
selectedFiles={selectedFiles}
onSelectFiles={handleSelectFiles}
/>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => setShowFolderSelector(false)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
Done Selecting
</Button>
</div>
) : (
<Button
type="button"
variant="outline"
size="sm"
onClick={() => setShowFolderSelector(false)}
onClick={() => setShowFolderSelector(true)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
Done Selecting
{totalSelected > 0 ? "Change Selection" : "Select Folders & Files"}
</Button>
</div>
) : (
<Button
type="button"
variant="outline"
onClick={() => setShowFolderSelector(true)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-white/10 text-xs sm:text-sm h-8 sm:h-9"
>
{totalSelected > 0 ? "Change Selection" : "Select Folders & Files"}
</Button>
)}
)}
</div>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center gap-2 [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0" />
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
Folder and file selection is used when indexing. You can change this selection when you
start indexing.
</AlertDescription>
</Alert>
{/* Indexing Options */}
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Indexing Options</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure how files are indexed from your Google Drive.
</p>
</div>
{/* Max files per folder */}
<div className="space-y-2">
<div className="flex items-center justify-between">
<div className="space-y-0.5">
<Label htmlFor="max-files" className="text-sm font-medium">
Max files per folder
</Label>
<p className="text-xs text-muted-foreground">
Maximum number of files to index from each folder
</p>
</div>
<Select
value={indexingOptions.max_files_per_folder.toString()}
onValueChange={(value) =>
handleIndexingOptionChange("max_files_per_folder", parseInt(value, 10))
}
>
<SelectTrigger
id="max-files"
className="w-[140px] bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select limit" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="50" className="text-xs sm:text-sm">
50 files
</SelectItem>
<SelectItem value="100" className="text-xs sm:text-sm">
100 files
</SelectItem>
<SelectItem value="250" className="text-xs sm:text-sm">
250 files
</SelectItem>
<SelectItem value="500" className="text-xs sm:text-sm">
500 files
</SelectItem>
<SelectItem value="1000" className="text-xs sm:text-sm">
1000 files
</SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Incremental sync toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="incremental-sync" className="text-sm font-medium">
Incremental sync
</Label>
<p className="text-xs text-muted-foreground">
Only sync changes since last index (faster). Disable for a full re-index.
</p>
</div>
<Switch
id="incremental-sync"
checked={indexingOptions.incremental_sync}
onCheckedChange={(checked) =>
handleIndexingOptionChange("incremental_sync", checked)
}
/>
</div>
{/* Include subfolders toggle */}
<div className="flex items-center justify-between pt-2 border-t border-slate-400/20">
<div className="space-y-0.5">
<Label htmlFor="include-subfolders" className="text-sm font-medium">
Include subfolders
</Label>
<p className="text-xs text-muted-foreground">
Recursively index files in subfolders of selected folders
</p>
</div>
<Switch
id="include-subfolders"
checked={indexingOptions.include_subfolders}
onCheckedChange={(checked) =>
handleIndexingOptionChange("include_subfolders", checked)
}
/>
</div>
</div>
</div>
);
};

16
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -218,15 +218,13 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
/>
)}
{/* Periodic sync - not shown for Google Drive */}
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && (
<PeriodicSyncConfig
enabled={periodicEnabled}
frequencyMinutes={frequencyMinutes}
onEnabledChange={onPeriodicEnabledChange}
onFrequencyChange={onFrequencyChange}
/>
)}
{/* Periodic sync - shown for all indexable connectors */}
<PeriodicSyncConfig
enabled={periodicEnabled}
frequencyMinutes={frequencyMinutes}
onEnabledChange={onPeriodicEnabledChange}
onFrequencyChange={onFrequencyChange}
/>
</>
)}

75
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -203,11 +203,9 @@ export const useConnectorDialog = () => {
setEditingConnector(connector);
setConnectorConfig(connector.config);
setConnectorName(connector.name);
// Load existing periodic sync settings (disabled for Google Drive and non-indexable connectors)
// Load existing periodic sync settings (disabled for non-indexable connectors)
setPeriodicEnabled(
connector.connector_type === "GOOGLE_DRIVE_CONNECTOR" || !connector.is_indexable
? false
: connector.periodic_indexing_enabled
!connector.is_indexable ? false : connector.periodic_indexing_enabled
);
setFrequencyMinutes(connector.indexing_frequency_minutes?.toString() || "1440");
// Reset dates - user can set new ones for re-indexing
@ -817,20 +815,14 @@ export const useConnectorDialog = () => {
const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined;
// Update connector with periodic sync settings and config changes
// Note: Periodic sync is disabled for Google Drive connectors
if (periodicEnabled || indexingConnectorConfig) {
const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : undefined;
await updateConnector({
id: indexingConfig.connectorId,
data: {
...(periodicEnabled &&
indexingConfig.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && {
periodic_indexing_enabled: true,
indexing_frequency_minutes: frequency,
}),
...(indexingConfig.connectorType === "GOOGLE_DRIVE_CONNECTOR" && {
periodic_indexing_enabled: false,
indexing_frequency_minutes: null,
...(periodicEnabled && {
periodic_indexing_enabled: true,
indexing_frequency_minutes: frequency,
}),
...(indexingConnectorConfig && {
config: indexingConnectorConfig,
@ -847,11 +839,18 @@ export const useConnectorDialog = () => {
const selectedFiles = indexingConnectorConfig.selected_files as
| Array<{ id: string; name: string }>
| undefined;
const indexingOptions = indexingConnectorConfig.indexing_options as
| {
max_files_per_folder: number;
incremental_sync: boolean;
include_subfolders: boolean;
}
| undefined;
if (
(selectedFolders && selectedFolders.length > 0) ||
(selectedFiles && selectedFiles.length > 0)
) {
// Index with folder/file selection
// Index with folder/file selection and indexing options
await indexConnector({
connector_id: indexingConfig.connectorId,
queryParams: {
@ -860,6 +859,11 @@ export const useConnectorDialog = () => {
body: {
folders: selectedFolders || [],
files: selectedFiles || [],
indexing_options: indexingOptions || {
max_files_per_folder: 100,
incremental_sync: true,
include_subfolders: true,
},
},
});
} else {
@ -899,7 +903,7 @@ export const useConnectorDialog = () => {
);
// Track periodic indexing started if enabled
if (periodicEnabled && indexingConfig.connectorType !== "GOOGLE_DRIVE_CONNECTOR") {
if (periodicEnabled) {
trackPeriodicIndexingStarted(
Number(searchSpaceId),
indexingConfig.connectorType,
@ -993,12 +997,8 @@ export const useConnectorDialog = () => {
setEditingConnector(connector);
setConnectorName(connector.name);
// Load existing periodic sync settings (disabled for Google Drive and non-indexable connectors)
setPeriodicEnabled(
connector.connector_type === "GOOGLE_DRIVE_CONNECTOR" || !connector.is_indexable
? false
: connector.periodic_indexing_enabled
);
// Load existing periodic sync settings (disabled for non-indexable connectors)
setPeriodicEnabled(!connector.is_indexable ? false : connector.periodic_indexing_enabled);
setFrequencyMinutes(connector.indexing_frequency_minutes?.toString() || "1440");
// Reset dates - user can set new ones for re-indexing
setStartDate(undefined);
@ -1053,23 +1053,14 @@ export const useConnectorDialog = () => {
const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined;
// Update connector with periodic sync settings, config changes, and name
// Note: Periodic sync is disabled for Google Drive connectors and non-indexable connectors
const frequency =
periodicEnabled && editingConnector.is_indexable ? parseInt(frequencyMinutes, 10) : null;
await updateConnector({
id: editingConnector.id,
data: {
name: connectorName || editingConnector.name,
periodic_indexing_enabled:
editingConnector.connector_type === "GOOGLE_DRIVE_CONNECTOR" ||
!editingConnector.is_indexable
? false
: periodicEnabled,
indexing_frequency_minutes:
editingConnector.connector_type === "GOOGLE_DRIVE_CONNECTOR" ||
!editingConnector.is_indexable
? null
: frequency,
periodic_indexing_enabled: !editingConnector.is_indexable ? false : periodicEnabled,
indexing_frequency_minutes: !editingConnector.is_indexable ? null : frequency,
config: connectorConfig || editingConnector.config,
},
});
@ -1087,6 +1078,13 @@ export const useConnectorDialog = () => {
const selectedFiles = (connectorConfig || editingConnector.config)?.selected_files as
| Array<{ id: string; name: string }>
| undefined;
const indexingOptions = (connectorConfig || editingConnector.config)?.indexing_options as
| {
max_files_per_folder: number;
incremental_sync: boolean;
include_subfolders: boolean;
}
| undefined;
if (
(selectedFolders && selectedFolders.length > 0) ||
(selectedFiles && selectedFiles.length > 0)
@ -1099,6 +1097,11 @@ export const useConnectorDialog = () => {
body: {
folders: selectedFolders || [],
files: selectedFiles || [],
indexing_options: indexingOptions || {
max_files_per_folder: 100,
incremental_sync: true,
include_subfolders: true,
},
},
});
const totalItems = (selectedFolders?.length || 0) + (selectedFiles?.length || 0);
@ -1142,12 +1145,8 @@ export const useConnectorDialog = () => {
);
}
// Track periodic indexing if enabled (for non-Google Drive connectors)
if (
periodicEnabled &&
editingConnector.is_indexable &&
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
) {
// Track periodic indexing if enabled
if (periodicEnabled && editingConnector.is_indexable) {
trackPeriodicIndexingStarted(
Number(searchSpaceId),
editingConnector.connector_type,