mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-06 20:15:17 +02:00
feat(routes): support multiple Google Drive folder indexing
- Accept comma-separated folder_ids and folder_names - Loop through each folder and index sequentially - Collect total indexed count and errors - Update timestamp only on full success
This commit is contained in:
parent
27a4bcdfc2
commit
634eeb887e
1 changed files with 38 additions and 18 deletions
|
|
@ -1548,35 +1548,55 @@ async def run_google_drive_indexing(
|
||||||
connector_id: int,
|
connector_id: int,
|
||||||
search_space_id: int,
|
search_space_id: int,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
folder_id: str,
|
folder_ids: str, # Comma-separated folder IDs
|
||||||
folder_name: str,
|
folder_names: str, # Comma-separated folder names
|
||||||
):
|
):
|
||||||
"""Runs the Google Drive indexing task and updates the timestamp."""
|
"""Runs the Google Drive indexing task for multiple folders and updates the timestamp."""
|
||||||
try:
|
try:
|
||||||
from app.tasks.connector_indexers.google_drive_indexer import (
|
from app.tasks.connector_indexers.google_drive_indexer import (
|
||||||
index_google_drive_files,
|
index_google_drive_files,
|
||||||
)
|
)
|
||||||
|
|
||||||
indexed_count, error_message = await index_google_drive_files(
|
# Split comma-separated IDs and names into lists
|
||||||
session,
|
folder_id_list = [fid.strip() for fid in folder_ids.split(",")]
|
||||||
connector_id,
|
folder_name_list = [fname.strip() for fname in folder_names.split(",")]
|
||||||
search_space_id,
|
|
||||||
user_id,
|
total_indexed = 0
|
||||||
folder_id,
|
errors = []
|
||||||
folder_name,
|
|
||||||
use_delta_sync=True,
|
# Index each folder
|
||||||
update_last_indexed=False,
|
for folder_id, folder_name in zip(folder_id_list, folder_name_list):
|
||||||
)
|
try:
|
||||||
if error_message:
|
indexed_count, error_message = await index_google_drive_files(
|
||||||
|
session,
|
||||||
|
connector_id,
|
||||||
|
search_space_id,
|
||||||
|
user_id,
|
||||||
|
folder_id,
|
||||||
|
folder_name,
|
||||||
|
use_delta_sync=True,
|
||||||
|
update_last_indexed=False,
|
||||||
|
)
|
||||||
|
if error_message:
|
||||||
|
errors.append(f"{folder_name}: {error_message}")
|
||||||
|
else:
|
||||||
|
total_indexed += indexed_count
|
||||||
|
except Exception as e:
|
||||||
|
errors.append(f"{folder_name}: {str(e)}")
|
||||||
|
logger.error(
|
||||||
|
f"Error indexing folder {folder_name} ({folder_id}): {e}",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if errors:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Google Drive indexing failed for connector {connector_id}: {error_message}"
|
f"Google Drive indexing completed with errors for connector {connector_id}: {'; '.join(errors)}"
|
||||||
)
|
)
|
||||||
# Optionally update status in DB to indicate failure
|
|
||||||
else:
|
else:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Google Drive indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
|
f"Google Drive indexing successful for connector {connector_id}. Indexed {total_indexed} documents from {len(folder_id_list)} folder(s)."
|
||||||
)
|
)
|
||||||
# Update the last indexed timestamp only on success
|
# Update the last indexed timestamp only on full success
|
||||||
await update_connector_last_indexed(session, connector_id)
|
await update_connector_last_indexed(session, connector_id)
|
||||||
await session.commit() # Commit timestamp update
|
await session.commit() # Commit timestamp update
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue