mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-06 20:15:17 +02:00
feat(routes): add Google Drive indexing support with folder selection
- Accept folder_id and folder_name as indexing parameters - Hide date range for Google Drive connectors - Create wrapper function to avoid circular imports - Trigger Google Drive indexing Celery task
This commit is contained in:
parent
7b8900d51f
commit
358abdf02f
1 changed files with 72 additions and 0 deletions
|
|
@ -45,6 +45,7 @@ from app.tasks.connector_indexers import (
|
||||||
index_github_repos,
|
index_github_repos,
|
||||||
index_google_calendar_events,
|
index_google_calendar_events,
|
||||||
index_google_gmail_messages,
|
index_google_gmail_messages,
|
||||||
|
index_google_drive_files,
|
||||||
index_jira_issues,
|
index_jira_issues,
|
||||||
index_linear_issues,
|
index_linear_issues,
|
||||||
index_luma_events,
|
index_luma_events,
|
||||||
|
|
@ -542,6 +543,14 @@ async def index_connector_content(
|
||||||
None,
|
None,
|
||||||
description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date",
|
description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date",
|
||||||
),
|
),
|
||||||
|
folder_id: str = Query(
|
||||||
|
None,
|
||||||
|
description="[Google Drive only] Folder ID to index. If not provided, uses the connector's saved selected_folder_id",
|
||||||
|
),
|
||||||
|
folder_name: str = Query(
|
||||||
|
None,
|
||||||
|
description="[Google Drive only] Folder name for display purposes",
|
||||||
|
),
|
||||||
session: AsyncSession = Depends(get_async_session),
|
session: AsyncSession = Depends(get_async_session),
|
||||||
user: User = Depends(current_active_user),
|
user: User = Depends(current_active_user),
|
||||||
):
|
):
|
||||||
|
|
@ -747,6 +756,25 @@ async def index_connector_content(
|
||||||
)
|
)
|
||||||
response_message = "Google Gmail indexing started in the background."
|
response_message = "Google Gmail indexing started in the background."
|
||||||
|
|
||||||
|
elif (
|
||||||
|
connector.connector_type == SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR
|
||||||
|
):
|
||||||
|
from app.tasks.celery_tasks.connector_tasks import (
|
||||||
|
index_google_drive_files_task,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Triggering Google Drive indexing for connector {connector_id} into search space {search_space_id}, folder: {folder_name or 'default'}"
|
||||||
|
)
|
||||||
|
index_google_drive_files_task.delay(
|
||||||
|
connector_id,
|
||||||
|
search_space_id,
|
||||||
|
str(user.id),
|
||||||
|
folder_id,
|
||||||
|
folder_name,
|
||||||
|
)
|
||||||
|
response_message = "Google Drive indexing started in the background."
|
||||||
|
|
||||||
elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
|
elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
|
||||||
from app.tasks.celery_tasks.connector_tasks import (
|
from app.tasks.celery_tasks.connector_tasks import (
|
||||||
index_discord_messages_task,
|
index_discord_messages_task,
|
||||||
|
|
@ -1515,6 +1543,50 @@ async def run_google_gmail_indexing(
|
||||||
# Optionally update status in DB to indicate failure
|
# Optionally update status in DB to indicate failure
|
||||||
|
|
||||||
|
|
||||||
|
async def run_google_drive_indexing(
|
||||||
|
session: AsyncSession,
|
||||||
|
connector_id: int,
|
||||||
|
search_space_id: int,
|
||||||
|
user_id: str,
|
||||||
|
folder_id: str,
|
||||||
|
folder_name: str,
|
||||||
|
):
|
||||||
|
"""Runs the Google Drive indexing task and updates the timestamp."""
|
||||||
|
try:
|
||||||
|
from app.tasks.connector_indexers.google_drive_indexer import (
|
||||||
|
index_google_drive_files,
|
||||||
|
)
|
||||||
|
|
||||||
|
indexed_count, error_message = await index_google_drive_files(
|
||||||
|
session,
|
||||||
|
connector_id,
|
||||||
|
search_space_id,
|
||||||
|
user_id,
|
||||||
|
folder_id,
|
||||||
|
folder_name,
|
||||||
|
use_delta_sync=True,
|
||||||
|
update_last_indexed=False,
|
||||||
|
)
|
||||||
|
if error_message:
|
||||||
|
logger.error(
|
||||||
|
f"Google Drive indexing failed for connector {connector_id}: {error_message}"
|
||||||
|
)
|
||||||
|
# Optionally update status in DB to indicate failure
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
f"Google Drive indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
|
||||||
|
)
|
||||||
|
# Update the last indexed timestamp only on success
|
||||||
|
await update_connector_last_indexed(session, connector_id)
|
||||||
|
await session.commit() # Commit timestamp update
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
f"Critical error in run_google_drive_indexing for connector {connector_id}: {e}",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
# Optionally update status in DB to indicate failure
|
||||||
|
|
||||||
|
|
||||||
# Add new helper functions for luma indexing
|
# Add new helper functions for luma indexing
|
||||||
async def run_luma_indexing_with_new_session(
|
async def run_luma_indexing_with_new_session(
|
||||||
connector_id: int,
|
connector_id: int,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue