Merge pull request #680 from AnishSarkar22/fix/index-future-date

feat: Index future dates for calendar based connectors & bug fixes

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -543,7 +543,7 @@ async def index_connector_content(
     ),
     end_date: str = Query(
         None,
-        description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date",
+        description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date. For calendar connectors (Google Calendar, Luma), future dates can be selected to index upcoming events.",
     ),
     drive_items: GoogleDriveIndexRequest | None = Body(
         None,
@@ -617,7 +617,19 @@ async def index_connector_content(
         else:
             indexing_from = start_date
 
-        indexing_to = end_date if end_date else today_str
+        # For calendar connectors, default to today but allow future dates if explicitly provided
+        if connector.connector_type in [
+            SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR,
+            SearchSourceConnectorType.LUMA_CONNECTOR,
+        ]:
+            # Default to today if no end_date provided (users can manually select future dates)
+            if end_date is None:
+                indexing_to = today_str
+            else:
+                indexing_to = end_date
+        else:
+            # For non-calendar connectors, cap at today
+            indexing_to = end_date if end_date else today_str
 
         if connector.connector_type == SearchSourceConnectorType.SLACK_CONNECTOR:
             from app.tasks.celery_tasks.connector_tasks import (

surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py

@@ -45,8 +45,9 @@ async def index_google_calendar_events(
         connector_id: ID of the Google Calendar connector
         search_space_id: ID of the search space to store documents in
         user_id: User ID
-        start_date: Start date for indexing (YYYY-MM-DD format)
-        end_date: End date for indexing (YYYY-MM-DD format)
+        start_date: Start date for indexing (YYYY-MM-DD format). Can be in the past or future.
+        end_date: End date for indexing (YYYY-MM-DD format). Can be in the future to index upcoming events.
+                  Defaults to today if not provided.
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
 
     Returns:
@@ -165,8 +166,10 @@ async def index_google_calendar_events(
             end_date = None
 
         # Calculate date range
+        # For calendar connectors, allow future dates to index upcoming events
         if start_date is None or end_date is None:
             # Fall back to calculating dates based on last_indexed_at
+            # Default to today (users can manually select future dates if needed)
             calculated_end_date = datetime.now()
 
             # Use last_indexed_at as start date if available, otherwise use 30 days ago
@@ -178,19 +181,13 @@ async def index_google_calendar_events(
                     else connector.last_indexed_at
                 )
 
-                # Check if last_indexed_at is in the future or after end_date
-                if last_indexed_naive > calculated_end_date:
-                    logger.warning(
-                        f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 30 days ago instead."
-                    )
-                    calculated_start_date = calculated_end_date - timedelta(days=30)
-                else:
-                    calculated_start_date = last_indexed_naive
-                    logger.info(
-                        f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date"
-                    )
+                # Allow future dates - use last_indexed_at as start date
+                calculated_start_date = last_indexed_naive
+                logger.info(
+                    f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date"
+                )
             else:
-                calculated_start_date = calculated_end_date - timedelta(
+                calculated_start_date = datetime.now() - timedelta(
                     days=30
                 )  # Use 30 days as default for calendar events
                 logger.info(
@@ -205,7 +202,7 @@ async def index_google_calendar_events(
                 end_date if end_date else calculated_end_date.strftime("%Y-%m-%d")
             )
         else:
-            # Use provided dates
+            # Use provided dates (including future dates)
             start_date_str = start_date
             end_date_str = end_date
 

surfsense_backend/app/tasks/connector_indexers/luma_indexer.py

@@ -45,8 +45,9 @@ async def index_luma_events(
         connector_id: ID of the Luma connector
         search_space_id: ID of the search space to store documents in
         user_id: User ID
-        start_date: Start date for indexing (YYYY-MM-DD format)
-        end_date: End date for indexing (YYYY-MM-DD format)
+        start_date: Start date for indexing (YYYY-MM-DD format). Can be in the past or future.
+        end_date: End date for indexing (YYYY-MM-DD format). Can be in the future to index upcoming events.
+                  Defaults to today if not provided.
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
 
     Returns:
@@ -116,8 +117,10 @@ async def index_luma_events(
         luma_client = LumaConnector(api_key=api_key)
 
         # Calculate date range
+        # For calendar connectors, allow future dates to index upcoming events
         if start_date is None or end_date is None:
             # Fall back to calculating dates based on last_indexed_at
+            # Default to today (users can manually select future dates if needed)
             calculated_end_date = datetime.now()
 
             # Use last_indexed_at as start date if available, otherwise use 30 days ago
@@ -129,19 +132,13 @@ async def index_luma_events(
                     else connector.last_indexed_at
                 )
 
-                # Check if last_indexed_at is in the future or after end_date
-                if last_indexed_naive > calculated_end_date:
-                    logger.warning(
-                        f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 30 days ago instead."
-                    )
-                    calculated_start_date = calculated_end_date - timedelta(days=30)
-                else:
-                    calculated_start_date = last_indexed_naive
-                    logger.info(
-                        f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date"
-                    )
+                # Allow future dates - use last_indexed_at as start date
+                calculated_start_date = last_indexed_naive
+                logger.info(
+                    f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date"
+                )
             else:
-                calculated_start_date = calculated_end_date - timedelta(days=30)
+                calculated_start_date = datetime.now() - timedelta(days=30)
                 logger.info(
                     f"No last_indexed_at found, using {calculated_start_date.strftime('%Y-%m-%d')} (30 days ago) as start date"
                 )
@@ -154,7 +151,7 @@ async def index_luma_events(
                 end_date if end_date else calculated_end_date.strftime("%Y-%m-%d")
             )
         else:
-            # Use provided dates
+            # Use provided dates (including future dates)
             start_date_str = start_date
             end_date_str = end_date