feat: improve indexing logic and duplicate handling in connectors

- Enhanced Google Calendar and Composio connector indexing to track and log duplicate content, preventing re-indexing of already processed events.
- Implemented robust error handling during final commits to manage integrity errors gracefully, ensuring successful indexing despite potential duplicates.
- Updated notification service to differentiate between actual errors and warnings for duplicate content, improving user feedback.
- Refactored date handling to ensure valid date ranges and adjusted end dates when necessary for better indexing accuracy.
This commit is contained in:
Anish Sarkar 2026-01-23 23:36:14 +05:30
parent d20bb385b5
commit c48ba36fa4
6 changed files with 198 additions and 35 deletions

View file

@ -100,10 +100,14 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
// Reset local quick indexing state when indexing completes or fails
useEffect(() => {
if (!isIndexing) {
setIsQuickIndexing(false);
if (!isIndexing && isQuickIndexing) {
// Small delay to ensure smooth transition
const timer = setTimeout(() => {
setIsQuickIndexing(false);
}, 100);
return () => clearTimeout(timer);
}
}, [isIndexing]);
}, [isIndexing, isQuickIndexing]);
const handleDisconnectClick = () => {
setShowDisconnectConfirm(true);
@ -119,11 +123,11 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
};
const handleQuickIndex = useCallback(() => {
if (onQuickIndex) {
if (onQuickIndex && !isQuickIndexing && !isIndexing) {
setIsQuickIndexing(true);
onQuickIndex();
}
}, [onQuickIndex]);
}, [onQuickIndex, isQuickIndexing, isIndexing]);
return (
<div className="flex-1 flex flex-col min-h-0 overflow-hidden">

View file

@ -1409,7 +1409,12 @@ export const useConnectorDialog = () => {
startDate?: Date,
endDate?: Date
) => {
if (!searchSpaceId) return;
if (!searchSpaceId) {
if (stopIndexing) {
stopIndexing(connectorId);
}
return;
}
// Track quick index clicked event
if (connectorType) {
@ -1437,6 +1442,8 @@ export const useConnectorDialog = () => {
queryClient.invalidateQueries({
queryKey: cacheKeys.logs.summary(Number(searchSpaceId)),
});
// Note: Don't call stopIndexing here - let useIndexingConnectors hook
// detect when last_indexed_at changes via Electric SQL
} catch (error) {
console.error("Error indexing connector content:", error);
toast.error(error instanceof Error ? error.message : "Failed to start indexing");
@ -1446,7 +1453,7 @@ export const useConnectorDialog = () => {
}
}
},
[searchSpaceId, indexConnector]
[searchSpaceId, indexConnector, queryClient]
);
// Handle going back from edit view