From 0e6669ac4ee366a30a923278a1cb418b9949d521 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 21 Oct 2025 21:11:41 -0700 Subject: [PATCH] fix: celery_app path and gmail indexing --- CODE_OF_CONDUCT.md | 2 +- .../app/connectors/google_gmail_connector.py | 33 +++++++++++++++---- .../routes/search_source_connectors_routes.py | 12 +++++-- .../app/tasks/celery_tasks/connector_tasks.py | 19 +++++++++-- .../google_gmail_indexer.py | 15 +++------ surfsense_backend/celery_worker.py | 6 ---- surfsense_web/app/(home)/privacy/page.tsx | 2 +- surfsense_web/app/(home)/terms/page.tsx | 2 +- .../[search_space_id]/onboard/page.tsx | 4 +-- 9 files changed, 61 insertions(+), 34 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index bd4dc0e63..14176e8f0 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -60,7 +60,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -vermarohanfinal@gmail.com. +rohan@surfsense.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the diff --git a/surfsense_backend/app/connectors/google_gmail_connector.py b/surfsense_backend/app/connectors/google_gmail_connector.py index 637680560..402337448 100644 --- a/surfsense_backend/app/connectors/google_gmail_connector.py +++ b/surfsense_backend/app/connectors/google_gmail_connector.py @@ -227,23 +227,42 @@ class GoogleGmailConnector: async def get_recent_messages( self, max_results: int = 50, - days_back: int = 30, + start_date: str | None = None, + end_date: str | None = None, ) -> tuple[list[dict[str, Any]], str | None]: """ - Fetch recent messages from Gmail within specified days. + Fetch recent messages from Gmail within specified date range. Args: max_results: Maximum number of messages to fetch (default: 50) - days_back: Number of days to look back (default: 30) + start_date: Start date in YYYY-MM-DD format (default: 30 days ago) + end_date: End date in YYYY-MM-DD format (default: today) Returns: Tuple containing (messages list with details, error message or None) """ try: - # Calculate date query from datetime import datetime, timedelta - cutoff_date = datetime.now() - timedelta(days=days_back) - date_query = cutoff_date.strftime("%Y/%m/%d") - query = f"after:{date_query}" + # Build date query + query_parts = [] + + if start_date: + # Parse start_date from YYYY-MM-DD to Gmail query format YYYY/MM/DD + start_dt = datetime.strptime(start_date, "%Y-%m-%d") + start_query = start_dt.strftime("%Y/%m/%d") + query_parts.append(f"after:{start_query}") + else: + # Default to 30 days ago + cutoff_date = datetime.now() - timedelta(days=30) + date_query = cutoff_date.strftime("%Y/%m/%d") + query_parts.append(f"after:{date_query}") + + if end_date: + # Parse end_date from YYYY-MM-DD to Gmail query format YYYY/MM/DD + end_dt = datetime.strptime(end_date, "%Y-%m-%d") + end_query = end_dt.strftime("%Y/%m/%d") + query_parts.append(f"before:{end_query}") + + query = " ".join(query_parts) # Get messages list messages_list, error = await self.get_messages_list( diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index fbed978b4..d0ee44f51 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -1250,14 +1250,22 @@ async def run_google_gmail_indexing( ): """Runs the Google Gmail indexing task and updates the timestamp.""" try: + # Convert days_back to start_date string in YYYY-MM-DD format + from datetime import datetime, timedelta + + start_date_obj = datetime.now() - timedelta(days=days_back) + start_date = start_date_obj.strftime("%Y-%m-%d") + end_date = None # No end date, index up to current time + indexed_count, error_message = await index_google_gmail_messages( session, connector_id, search_space_id, user_id, - max_messages, - days_back, + start_date=start_date, + end_date=end_date, update_last_indexed=False, + max_messages=max_messages, ) if error_message: logger.error( diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index e1a0b708e..5e6907499 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -445,14 +445,27 @@ async def _index_google_gmail_messages( end_date: str, ): """Index Google Gmail messages with new session.""" + from datetime import datetime + from app.routes.search_source_connectors_routes import ( run_google_gmail_indexing, ) - # Parse dates to get max_messages and days_back - # For now, we'll use default values + # Parse dates to calculate days_back max_messages = 100 - days_back = 30 + days_back = 30 # Default + + if start_date: + try: + # Parse start_date (format: YYYY-MM-DD) + start_dt = datetime.strptime(start_date, "%Y-%m-%d") + # Calculate days back from now + days_back = (datetime.now() - start_dt).days + # Ensure at least 1 day + days_back = max(1, days_back) + except ValueError: + # If parsing fails, use default + days_back = 30 async with get_celery_session_maker()() as session: await run_google_gmail_indexing( diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 9d3823741..7e821dc9f 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -40,7 +40,7 @@ async def index_google_gmail_messages( start_date: str | None = None, end_date: str | None = None, update_last_indexed: bool = True, - max_messages: int = 100, + max_messages: int = 1000, ) -> tuple[int, str]: """ Index Gmail messages for a specific connector. @@ -60,14 +60,6 @@ async def index_google_gmail_messages( """ task_logger = TaskLoggingService(session, search_space_id) - # Calculate days back based on start_date - if start_date: - try: - start_date_obj = datetime.strptime(start_date, "%Y-%m-%d") - days_back = (datetime.now() - start_date_obj).days - except ValueError: - days_back = 30 # Default to 30 days if start_date is invalid - # Log task start log_entry = await task_logger.log_task_start( task_name="google_gmail_messages_indexing", @@ -77,7 +69,8 @@ async def index_google_gmail_messages( "connector_id": connector_id, "user_id": str(user_id), "max_messages": max_messages, - "days_back": days_back, + "start_date": start_date, + "end_date": end_date, }, ) @@ -135,7 +128,7 @@ async def index_google_gmail_messages( # Fetch recent Google gmail messages logger.info(f"Fetching recent emails for connector {connector_id}") messages, error = await gmail_connector.get_recent_messages( - max_results=max_messages, days_back=days_back + max_results=max_messages, start_date=start_date, end_date=end_date ) if error: diff --git a/surfsense_backend/celery_worker.py b/surfsense_backend/celery_worker.py index 2d416a82f..dcc7599ba 100644 --- a/surfsense_backend/celery_worker.py +++ b/surfsense_backend/celery_worker.py @@ -1,11 +1,5 @@ """Celery worker startup script.""" -import os -import sys - -# Add the app directory to the Python path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "app")) - from app.celery_app import celery_app if __name__ == "__main__": diff --git a/surfsense_web/app/(home)/privacy/page.tsx b/surfsense_web/app/(home)/privacy/page.tsx index ffe1ba9e3..970f50e27 100644 --- a/surfsense_web/app/(home)/privacy/page.tsx +++ b/surfsense_web/app/(home)/privacy/page.tsx @@ -181,7 +181,7 @@ export default function PrivacyPolicy() { contact us at:

- Email: vermarohanfinal@gmail.com + Email: rohan@surfsense.com

diff --git a/surfsense_web/app/(home)/terms/page.tsx b/surfsense_web/app/(home)/terms/page.tsx index e1f317a4c..dbc4c0253 100644 --- a/surfsense_web/app/(home)/terms/page.tsx +++ b/surfsense_web/app/(home)/terms/page.tsx @@ -216,7 +216,7 @@ export default function TermsOfService() {

12. Contact Us

If you have any questions about these Terms, please contact us at:

- Email: vermarohanfinal@gmail.com + Email: rohan@surfsense.com

diff --git a/surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx index 0ff0fb205..d7c2d0b3a 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx @@ -122,11 +122,11 @@ const OnboardPage = () => { {/* Header */}
- +

Welcome to SurfSense

- Let's configure your SurfSense to get started + Let's configure your LLM configurations to get started