mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-04-28 10:26:33 +02:00
feat: enhance date handling and indexing logic across connectors
- Added normalization for "undefined" strings to None in date parameters to prevent parsing errors. - Improved date range validation to ensure start_date is strictly before end_date, adjusting end_date if necessary. - Updated Google Calendar and Composio connector indexing logic to handle duplicate content more effectively, logging warnings for skipped events. - Enhanced error handling during final commits to manage integrity errors gracefully. - Refactored date handling in various connector indexers for consistency and reliability.
This commit is contained in:
parent
08f16b43d7
commit
d20bb385b5
9 changed files with 83 additions and 13 deletions
|
|
@ -112,6 +112,13 @@ def calculate_date_range(
|
|||
Returns:
|
||||
Tuple of (start_date_str, end_date_str)
|
||||
"""
|
||||
# Normalize "undefined" strings to None (from frontend)
|
||||
# This prevents parsing errors and ensures consistent behavior across all indexers
|
||||
if start_date == "undefined" or start_date == "":
|
||||
start_date = None
|
||||
if end_date == "undefined" or end_date == "":
|
||||
end_date = None
|
||||
|
||||
if start_date is not None and end_date is not None:
|
||||
return start_date, end_date
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ Google Calendar connector indexer.
|
|||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytz
|
||||
from dateutil.parser import isoparse
|
||||
from google.oauth2.credentials import Credentials
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
|
@ -205,6 +207,23 @@ async def index_google_calendar_events(
|
|||
# Use provided dates (including future dates)
|
||||
start_date_str = start_date
|
||||
end_date_str = end_date
|
||||
|
||||
# If start_date and end_date are the same, adjust end_date to be one day later
|
||||
# to ensure valid date range (start_date must be strictly before end_date)
|
||||
if start_date_str == end_date_str:
|
||||
# Parse the date and add one day to ensure valid range
|
||||
dt = isoparse(end_date_str)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=pytz.UTC)
|
||||
else:
|
||||
dt = dt.astimezone(pytz.UTC)
|
||||
# Add one day to end_date to make it strictly after start_date
|
||||
dt_end = dt + timedelta(days=1)
|
||||
end_date_str = dt_end.strftime("%Y-%m-%d")
|
||||
logger.info(
|
||||
f"Adjusted end_date from {end_date} to {end_date_str} "
|
||||
f"to ensure valid date range (start_date must be strictly before end_date)"
|
||||
)
|
||||
|
||||
await task_logger.log_task_progress(
|
||||
log_entry,
|
||||
|
|
|
|||
|
|
@ -116,6 +116,13 @@ async def index_luma_events(
|
|||
|
||||
luma_client = LumaConnector(api_key=api_key)
|
||||
|
||||
# Handle 'undefined' string from frontend (treat as None)
|
||||
# This prevents "time data 'undefined' does not match format" errors
|
||||
if start_date == "undefined" or start_date == "":
|
||||
start_date = None
|
||||
if end_date == "undefined" or end_date == "":
|
||||
end_date = None
|
||||
|
||||
# Calculate date range
|
||||
# For calendar connectors, allow future dates to index upcoming events
|
||||
if start_date is None or end_date is None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue