feat: enhance date handling and indexing logic across connectors

- Added normalization for "undefined" strings to None in date parameters to prevent parsing errors.
- Improved date range validation to ensure start_date is strictly before end_date, adjusting end_date if necessary.
- Updated Google Calendar and Composio connector indexing logic to handle duplicate content more effectively, logging warnings for skipped events.
- Enhanced error handling during final commits to manage integrity errors gracefully.
- Refactored date handling in various connector indexers for consistency and reliability.
This commit is contained in:
Anish Sarkar 2026-01-23 23:03:29 +05:30
parent 08f16b43d7
commit d20bb385b5
9 changed files with 83 additions and 13 deletions

View file

@ -259,7 +259,13 @@ export const ConnectorIndicator: FC = () => {
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
? () => {
startIndexing(editingConnector.id);
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type, stopIndexing);
handleQuickIndexConnector(
editingConnector.id,
editingConnector.connector_type,
stopIndexing,
startDate,
endDate
);
}
: undefined
}

View file

@ -272,8 +272,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
Re-indexing runs in the background
</p>
<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
You can continue using SurfSense while we sync your data. Check the Active tab
to see progress.
You can continue using SurfSense while we sync your data. Check inbox for updates.
</p>
</div>
</div>

View file

@ -189,8 +189,7 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
<div className="text-xs sm:text-sm">
<p className="font-medium text-xs sm:text-sm">Indexing runs in the background</p>
<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
You can continue using SurfSense while we sync your data. Check the Active tab
to see progress.
You can continue using SurfSense while we sync your data. Check inbox for updates.
</p>
</div>
</div>

View file

@ -1400,9 +1400,15 @@ export const useConnectorDialog = () => {
[editingConnector, searchSpaceId, deleteConnector, router, cameFromMCPList]
);
// Handle quick index (index without date picker, uses backend defaults)
// Handle quick index (index with selected date range, or backend defaults if none selected)
const handleQuickIndexConnector = useCallback(
async (connectorId: number, connectorType?: string, stopIndexing?: (id: number) => void) => {
async (
connectorId: number,
connectorType?: string,
stopIndexing?: (id: number) => void,
startDate?: Date,
endDate?: Date
) => {
if (!searchSpaceId) return;
// Track quick index clicked event
@ -1411,10 +1417,16 @@ export const useConnectorDialog = () => {
}
try {
// Format dates if provided, otherwise pass undefined (backend will use defaults)
const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined;
const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined;
await indexConnector({
connector_id: connectorId,
queryParams: {
search_space_id: searchSpaceId,
start_date: startDateStr,
end_date: endDateStr,
},
});
toast.success("Indexing started", {