mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-27 19:25:15 +02:00
feat: improve indexing logic and duplicate handling in connectors
- Enhanced Google Calendar and Composio connector indexing to track and log duplicate content, preventing re-indexing of already processed events. - Implemented robust error handling during final commits to manage integrity errors gracefully, ensuring successful indexing despite potential duplicates. - Updated notification service to differentiate between actual errors and warnings for duplicate content, improving user feedback. - Refactored date handling to ensure valid date ranges and adjusted end dates when necessary for better indexing accuracy.
This commit is contained in:
parent
d20bb385b5
commit
c48ba36fa4
6 changed files with 198 additions and 35 deletions
|
|
@ -100,10 +100,14 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
|
|||
|
||||
// Reset local quick indexing state when indexing completes or fails
|
||||
useEffect(() => {
|
||||
if (!isIndexing) {
|
||||
setIsQuickIndexing(false);
|
||||
if (!isIndexing && isQuickIndexing) {
|
||||
// Small delay to ensure smooth transition
|
||||
const timer = setTimeout(() => {
|
||||
setIsQuickIndexing(false);
|
||||
}, 100);
|
||||
return () => clearTimeout(timer);
|
||||
}
|
||||
}, [isIndexing]);
|
||||
}, [isIndexing, isQuickIndexing]);
|
||||
|
||||
const handleDisconnectClick = () => {
|
||||
setShowDisconnectConfirm(true);
|
||||
|
|
@ -119,11 +123,11 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
|
|||
};
|
||||
|
||||
const handleQuickIndex = useCallback(() => {
|
||||
if (onQuickIndex) {
|
||||
if (onQuickIndex && !isQuickIndexing && !isIndexing) {
|
||||
setIsQuickIndexing(true);
|
||||
onQuickIndex();
|
||||
}
|
||||
}, [onQuickIndex]);
|
||||
}, [onQuickIndex, isQuickIndexing, isIndexing]);
|
||||
|
||||
return (
|
||||
<div className="flex-1 flex flex-col min-h-0 overflow-hidden">
|
||||
|
|
|
|||
|
|
@ -1409,7 +1409,12 @@ export const useConnectorDialog = () => {
|
|||
startDate?: Date,
|
||||
endDate?: Date
|
||||
) => {
|
||||
if (!searchSpaceId) return;
|
||||
if (!searchSpaceId) {
|
||||
if (stopIndexing) {
|
||||
stopIndexing(connectorId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Track quick index clicked event
|
||||
if (connectorType) {
|
||||
|
|
@ -1437,6 +1442,8 @@ export const useConnectorDialog = () => {
|
|||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.logs.summary(Number(searchSpaceId)),
|
||||
});
|
||||
// Note: Don't call stopIndexing here - let useIndexingConnectors hook
|
||||
// detect when last_indexed_at changes via Electric SQL
|
||||
} catch (error) {
|
||||
console.error("Error indexing connector content:", error);
|
||||
toast.error(error instanceof Error ? error.message : "Failed to start indexing");
|
||||
|
|
@ -1446,7 +1453,7 @@ export const useConnectorDialog = () => {
|
|||
}
|
||||
}
|
||||
},
|
||||
[searchSpaceId, indexConnector]
|
||||
[searchSpaceId, indexConnector, queryClient]
|
||||
);
|
||||
|
||||
// Handle going back from edit view
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue