apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-06 20:15:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: added circleback connector

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2025-12-30 09:00:59 -08:00
parent 23870042f3
commit c19d300c9d
27 changed files with 1153 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

2
surfsense_backend/app/tasks/connector_indexers/__init__.py
View file

@ -34,8 +34,8 @@ from .discord_indexer import index_discord_messages
from .elasticsearch_indexer import index_elasticsearch_documents
from .github_indexer import index_github_repos
from .google_calendar_indexer import index_google_calendar_events
from .google_gmail_indexer import index_google_gmail_messages
from .google_drive_indexer import index_google_drive_files
from .google_gmail_indexer import index_google_gmail_messages
from .jira_indexer import index_jira_issues
# Issue tracking and project management

23
surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py
View file

@ -1,7 +1,6 @@
"""Google Drive indexer using Surfsense file processors."""
import logging
from datetime import datetime
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
@ -99,11 +98,15 @@ async def index_google_drive_files(
target_folder_id = folder_id
target_folder_name = folder_name or "Selected Folder"
logger.info(f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})")
logger.info(
f"Indexing Google Drive folder: {target_folder_name} ({target_folder_id})"
)
folder_tokens = connector.config.get("folder_tokens", {})
start_page_token = folder_tokens.get(target_folder_id)
can_use_delta_sync = use_delta_sync and start_page_token and connector.last_indexed_at
can_use_delta_sync = (
use_delta_sync and start_page_token and connector.last_indexed_at
)
if can_use_delta_sync:
logger.info(f"Using delta sync for connector {connector_id}")
@ -151,9 +154,7 @@ async def index_google_drive_files(
await update_connector_last_indexed(session, connector, update_last_indexed)
await session.commit()
logger.info(
f"Successfully committed Google Drive indexing changes to database"
)
logger.info("Successfully committed Google Drive indexing changes to database")
await task_logger.log_task_success(
log_entry,
@ -252,7 +253,9 @@ async def _index_full_scan(
if documents_indexed % 10 == 0 and documents_indexed > 0:
await session.commit()
logger.info(f"Committed batch: {documents_indexed} files indexed so far")
logger.info(
f"Committed batch: {documents_indexed} files indexed so far"
)
page_token = next_token
if not page_token:
@ -391,9 +394,7 @@ async def _process_single_file(
return 0, 1
async def _remove_document(
session: AsyncSession, file_id: str, search_space_id: int
):
async def _remove_document(session: AsyncSession, file_id: str, search_space_id: int):
"""Remove a document that was deleted in Drive."""
unique_identifier_hash = generate_unique_identifier_hash(
DocumentType.GOOGLE_DRIVE_FILE, file_id, search_space_id
@ -406,5 +407,3 @@ async def _remove_document(
if existing_document:
await session.delete(existing_document)
logger.info(f"Removed deleted file document: {file_id}")