mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-22 23:31:12 +02:00
chore: ran both frontend and backend linting
This commit is contained in:
parent
99bd2df463
commit
5bd6bd3d67
21 changed files with 861 additions and 739 deletions
|
|
@ -6,6 +6,7 @@ Revises: 61
|
||||||
Note: Electric SQL replication setup (REPLICA IDENTITY FULL and publication)
|
Note: Electric SQL replication setup (REPLICA IDENTITY FULL and publication)
|
||||||
is handled in app/db.py setup_electric_replication() which runs on app startup.
|
is handled in app/db.py setup_electric_replication() which runs on app startup.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|
|
||||||
|
|
@ -711,15 +711,22 @@ class Notification(BaseModel, TimestampMixin):
|
||||||
__tablename__ = "notifications"
|
__tablename__ = "notifications"
|
||||||
|
|
||||||
user_id = Column(
|
user_id = Column(
|
||||||
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False, index=True
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("user.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
)
|
)
|
||||||
search_space_id = Column(
|
search_space_id = Column(
|
||||||
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True
|
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True
|
||||||
)
|
)
|
||||||
type = Column(String(50), nullable=False) # 'connector_indexing', 'document_processing', etc.
|
type = Column(
|
||||||
|
String(50), nullable=False
|
||||||
|
) # 'connector_indexing', 'document_processing', etc.
|
||||||
title = Column(String(200), nullable=False)
|
title = Column(String(200), nullable=False)
|
||||||
message = Column(Text, nullable=False)
|
message = Column(Text, nullable=False)
|
||||||
read = Column(Boolean, nullable=False, default=False, server_default=text("false"), index=True)
|
read = Column(
|
||||||
|
Boolean, nullable=False, default=False, server_default=text("false"), index=True
|
||||||
|
)
|
||||||
notification_metadata = Column("metadata", JSONB, nullable=True, default={})
|
notification_metadata = Column("metadata", JSONB, nullable=True, default={})
|
||||||
|
|
||||||
user = relationship("User", back_populates="notifications")
|
user = relationship("User", back_populates="notifications")
|
||||||
|
|
@ -990,7 +997,9 @@ async def setup_electric_replication():
|
||||||
# Set REPLICA IDENTITY FULL (required by Electric SQL for replication)
|
# Set REPLICA IDENTITY FULL (required by Electric SQL for replication)
|
||||||
# This logs full row data for UPDATE/DELETE operations in the WAL
|
# This logs full row data for UPDATE/DELETE operations in the WAL
|
||||||
await conn.execute(text("ALTER TABLE notifications REPLICA IDENTITY FULL;"))
|
await conn.execute(text("ALTER TABLE notifications REPLICA IDENTITY FULL;"))
|
||||||
await conn.execute(text("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;"))
|
await conn.execute(
|
||||||
|
text("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;")
|
||||||
|
)
|
||||||
await conn.execute(text("ALTER TABLE documents REPLICA IDENTITY FULL;"))
|
await conn.execute(text("ALTER TABLE documents REPLICA IDENTITY FULL;"))
|
||||||
|
|
||||||
# Add tables to Electric SQL publication for replication
|
# Add tables to Electric SQL publication for replication
|
||||||
|
|
|
||||||
|
|
@ -986,21 +986,25 @@ async def _run_indexing_with_notifications(
|
||||||
try:
|
try:
|
||||||
# Get connector info for notification
|
# Get connector info for notification
|
||||||
connector_result = await session.execute(
|
connector_result = await session.execute(
|
||||||
select(SearchSourceConnector).where(SearchSourceConnector.id == connector_id)
|
select(SearchSourceConnector).where(
|
||||||
|
SearchSourceConnector.id == connector_id
|
||||||
|
)
|
||||||
)
|
)
|
||||||
connector = connector_result.scalar_one_or_none()
|
connector = connector_result.scalar_one_or_none()
|
||||||
|
|
||||||
if connector:
|
if connector:
|
||||||
# Create notification when indexing starts
|
# Create notification when indexing starts
|
||||||
notification = await NotificationService.connector_indexing.notify_indexing_started(
|
notification = (
|
||||||
session=session,
|
await NotificationService.connector_indexing.notify_indexing_started(
|
||||||
user_id=UUID(user_id),
|
session=session,
|
||||||
connector_id=connector_id,
|
user_id=UUID(user_id),
|
||||||
connector_name=connector.name,
|
connector_id=connector_id,
|
||||||
connector_type=connector.connector_type.value,
|
connector_name=connector.name,
|
||||||
search_space_id=search_space_id,
|
connector_type=connector.connector_type.value,
|
||||||
start_date=start_date,
|
search_space_id=search_space_id,
|
||||||
end_date=end_date,
|
start_date=start_date,
|
||||||
|
end_date=end_date,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Update notification to fetching stage
|
# Update notification to fetching stage
|
||||||
|
|
@ -1640,6 +1644,7 @@ async def run_google_gmail_indexing(
|
||||||
start_date: Start date for indexing
|
start_date: Start date for indexing
|
||||||
end_date: End date for indexing
|
end_date: End date for indexing
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Create a wrapper function that calls index_google_gmail_messages with max_messages
|
# Create a wrapper function that calls index_google_gmail_messages with max_messages
|
||||||
async def gmail_indexing_wrapper(
|
async def gmail_indexing_wrapper(
|
||||||
session: AsyncSession,
|
session: AsyncSession,
|
||||||
|
|
@ -1701,7 +1706,9 @@ async def run_google_drive_indexing(
|
||||||
|
|
||||||
# Get connector info for notification
|
# Get connector info for notification
|
||||||
connector_result = await session.execute(
|
connector_result = await session.execute(
|
||||||
select(SearchSourceConnector).where(SearchSourceConnector.id == connector_id)
|
select(SearchSourceConnector).where(
|
||||||
|
SearchSourceConnector.id == connector_id
|
||||||
|
)
|
||||||
)
|
)
|
||||||
connector = connector_result.scalar_one_or_none()
|
connector = connector_result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
@ -1813,7 +1820,7 @@ async def run_google_drive_indexing(
|
||||||
f"Critical error in run_google_drive_indexing for connector {connector_id}: {e}",
|
f"Critical error in run_google_drive_indexing for connector {connector_id}: {e}",
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Update notification on exception
|
# Update notification on exception
|
||||||
if notification:
|
if notification:
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -99,7 +99,9 @@ class BaseNotificationHandler:
|
||||||
flag_modified(notification, "notification_metadata")
|
flag_modified(notification, "notification_metadata")
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(notification)
|
await session.refresh(notification)
|
||||||
logger.info(f"Updated notification {notification.id} for operation {operation_id}")
|
logger.info(
|
||||||
|
f"Updated notification {notification.id} for operation {operation_id}"
|
||||||
|
)
|
||||||
return notification
|
return notification
|
||||||
|
|
||||||
# Create new notification
|
# Create new notification
|
||||||
|
|
@ -119,7 +121,9 @@ class BaseNotificationHandler:
|
||||||
session.add(notification)
|
session.add(notification)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(notification)
|
await session.refresh(notification)
|
||||||
logger.info(f"Created notification {notification.id} for operation {operation_id}")
|
logger.info(
|
||||||
|
f"Created notification {notification.id} for operation {operation_id}"
|
||||||
|
)
|
||||||
return notification
|
return notification
|
||||||
|
|
||||||
async def update_notification(
|
async def update_notification(
|
||||||
|
|
@ -153,9 +157,9 @@ class BaseNotificationHandler:
|
||||||
if status is not None:
|
if status is not None:
|
||||||
notification.notification_metadata["status"] = status
|
notification.notification_metadata["status"] = status
|
||||||
if status in ("completed", "failed"):
|
if status in ("completed", "failed"):
|
||||||
notification.notification_metadata["completed_at"] = (
|
notification.notification_metadata["completed_at"] = datetime.now(
|
||||||
datetime.now(UTC).isoformat()
|
UTC
|
||||||
)
|
).isoformat()
|
||||||
# Mark JSONB column as modified so SQLAlchemy detects the change
|
# Mark JSONB column as modified so SQLAlchemy detects the change
|
||||||
flag_modified(notification, "notification_metadata")
|
flag_modified(notification, "notification_metadata")
|
||||||
|
|
||||||
|
|
@ -180,7 +184,10 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
|
||||||
super().__init__("connector_indexing")
|
super().__init__("connector_indexing")
|
||||||
|
|
||||||
def _generate_operation_id(
|
def _generate_operation_id(
|
||||||
self, connector_id: int, start_date: str | None = None, end_date: str | None = None
|
self,
|
||||||
|
connector_id: int,
|
||||||
|
start_date: str | None = None,
|
||||||
|
end_date: str | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Generate a unique operation ID for a connector indexing operation.
|
Generate a unique operation ID for a connector indexing operation.
|
||||||
|
|
@ -298,7 +305,7 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
|
||||||
"processing": "Preparing for search",
|
"processing": "Preparing for search",
|
||||||
"storing": "Almost done",
|
"storing": "Almost done",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Use stage-based message if stage provided, otherwise fallback
|
# Use stage-based message if stage provided, otherwise fallback
|
||||||
if stage or stage_message:
|
if stage or stage_message:
|
||||||
progress_msg = stage_message or stage_messages.get(stage, "Processing")
|
progress_msg = stage_message or stage_messages.get(stage, "Processing")
|
||||||
|
|
@ -341,7 +348,9 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
|
||||||
Returns:
|
Returns:
|
||||||
Updated notification
|
Updated notification
|
||||||
"""
|
"""
|
||||||
connector_name = notification.notification_metadata.get("connector_name", "Connector")
|
connector_name = notification.notification_metadata.get(
|
||||||
|
"connector_name", "Connector"
|
||||||
|
)
|
||||||
|
|
||||||
if error_message:
|
if error_message:
|
||||||
title = f"Failed: {connector_name}"
|
title = f"Failed: {connector_name}"
|
||||||
|
|
@ -414,7 +423,7 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
|
||||||
"indexed_count": 0,
|
"indexed_count": 0,
|
||||||
"sync_stage": "connecting",
|
"sync_stage": "connecting",
|
||||||
}
|
}
|
||||||
|
|
||||||
if folder_names:
|
if folder_names:
|
||||||
metadata["folder_names"] = folder_names
|
metadata["folder_names"] = folder_names
|
||||||
if file_names:
|
if file_names:
|
||||||
|
|
@ -454,6 +463,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
|
||||||
timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f")
|
timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f")
|
||||||
# Create a short hash of filename to ensure uniqueness
|
# Create a short hash of filename to ensure uniqueness
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
||||||
filename_hash = hashlib.md5(filename.encode()).hexdigest()[:8]
|
filename_hash = hashlib.md5(filename.encode()).hexdigest()[:8]
|
||||||
return f"doc_{document_type}_{search_space_id}_{timestamp}_{filename_hash}"
|
return f"doc_{document_type}_{search_space_id}_{timestamp}_{filename_hash}"
|
||||||
|
|
||||||
|
|
@ -480,7 +490,9 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
|
||||||
Returns:
|
Returns:
|
||||||
Notification: The created notification
|
Notification: The created notification
|
||||||
"""
|
"""
|
||||||
operation_id = self._generate_operation_id(document_type, document_name, search_space_id)
|
operation_id = self._generate_operation_id(
|
||||||
|
document_type, document_name, search_space_id
|
||||||
|
)
|
||||||
title = f"Processing: {document_name}"
|
title = f"Processing: {document_name}"
|
||||||
message = "Waiting in queue"
|
message = "Waiting in queue"
|
||||||
|
|
||||||
|
|
@ -489,7 +501,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
|
||||||
"document_name": document_name,
|
"document_name": document_name,
|
||||||
"processing_stage": "queued",
|
"processing_stage": "queued",
|
||||||
}
|
}
|
||||||
|
|
||||||
if file_size is not None:
|
if file_size is not None:
|
||||||
metadata["file_size"] = file_size
|
metadata["file_size"] = file_size
|
||||||
|
|
||||||
|
|
@ -531,7 +543,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
|
||||||
"embedding": "Preparing for search",
|
"embedding": "Preparing for search",
|
||||||
"storing": "Finalizing",
|
"storing": "Finalizing",
|
||||||
}
|
}
|
||||||
|
|
||||||
message = stage_message or stage_messages.get(stage, "Processing")
|
message = stage_message or stage_messages.get(stage, "Processing")
|
||||||
|
|
||||||
metadata_updates = {"processing_stage": stage}
|
metadata_updates = {"processing_stage": stage}
|
||||||
|
|
@ -568,7 +580,9 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
|
||||||
Returns:
|
Returns:
|
||||||
Updated notification
|
Updated notification
|
||||||
"""
|
"""
|
||||||
document_name = notification.notification_metadata.get("document_name", "Document")
|
document_name = notification.notification_metadata.get(
|
||||||
|
"document_name", "Document"
|
||||||
|
)
|
||||||
|
|
||||||
if error_message:
|
if error_message:
|
||||||
title = f"Failed: {document_name}"
|
title = f"Failed: {document_name}"
|
||||||
|
|
@ -583,7 +597,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
|
||||||
"processing_stage": "completed" if not error_message else "failed",
|
"processing_stage": "completed" if not error_message else "failed",
|
||||||
"error_message": error_message,
|
"error_message": error_message,
|
||||||
}
|
}
|
||||||
|
|
||||||
if document_id is not None:
|
if document_id is not None:
|
||||||
metadata_updates["document_id"] = document_id
|
metadata_updates["document_id"] = document_id
|
||||||
# Store chunks_count in metadata for debugging, but don't show to user
|
# Store chunks_count in metadata for debugging, but don't show to user
|
||||||
|
|
@ -645,4 +659,3 @@ class NotificationService:
|
||||||
await session.refresh(notification)
|
await session.refresh(notification)
|
||||||
logger.info(f"Created notification {notification.id} for user {user_id}")
|
logger.info(f"Created notification {notification.id} for user {user_id}")
|
||||||
return notification
|
return notification
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -92,12 +92,14 @@ async def _process_extension_document(
|
||||||
page_title += "..."
|
page_title += "..."
|
||||||
|
|
||||||
# Create notification for document processing
|
# Create notification for document processing
|
||||||
notification = await NotificationService.document_processing.notify_processing_started(
|
notification = (
|
||||||
session=session,
|
await NotificationService.document_processing.notify_processing_started(
|
||||||
user_id=UUID(user_id),
|
session=session,
|
||||||
document_type="EXTENSION",
|
user_id=UUID(user_id),
|
||||||
document_name=page_title,
|
document_type="EXTENSION",
|
||||||
search_space_id=search_space_id,
|
document_name=page_title,
|
||||||
|
search_space_id=search_space_id,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
log_entry = await task_logger.log_task_start(
|
log_entry = await task_logger.log_task_start(
|
||||||
|
|
@ -115,7 +117,10 @@ async def _process_extension_document(
|
||||||
try:
|
try:
|
||||||
# Update notification: parsing stage
|
# Update notification: parsing stage
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await NotificationService.document_processing.notify_processing_progress(
|
||||||
session, notification, stage="parsing", stage_message="Reading page content"
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Reading page content",
|
||||||
)
|
)
|
||||||
|
|
||||||
result = await add_extension_received_document(
|
result = await add_extension_received_document(
|
||||||
|
|
@ -130,11 +135,13 @@ async def _process_extension_document(
|
||||||
)
|
)
|
||||||
|
|
||||||
# Update notification on success
|
# Update notification on success
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
document_id=result.id,
|
notification=notification,
|
||||||
chunks_count=None,
|
document_id=result.id,
|
||||||
|
chunks_count=None,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
await task_logger.log_task_success(
|
await task_logger.log_task_success(
|
||||||
|
|
@ -144,10 +151,12 @@ async def _process_extension_document(
|
||||||
)
|
)
|
||||||
|
|
||||||
# Update notification for duplicate
|
# Update notification for duplicate
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
error_message="Page already saved (duplicate)",
|
notification=notification,
|
||||||
|
error_message="Page already saved (duplicate)",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
await task_logger.log_task_failure(
|
await task_logger.log_task_failure(
|
||||||
|
|
@ -198,12 +207,14 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
|
||||||
video_name = url.split("v=")[-1][:11] if "v=" in url else url
|
video_name = url.split("v=")[-1][:11] if "v=" in url else url
|
||||||
|
|
||||||
# Create notification for document processing
|
# Create notification for document processing
|
||||||
notification = await NotificationService.document_processing.notify_processing_started(
|
notification = (
|
||||||
session=session,
|
await NotificationService.document_processing.notify_processing_started(
|
||||||
user_id=UUID(user_id),
|
session=session,
|
||||||
document_type="YOUTUBE_VIDEO",
|
user_id=UUID(user_id),
|
||||||
document_name=f"YouTube: {video_name}",
|
document_type="YOUTUBE_VIDEO",
|
||||||
search_space_id=search_space_id,
|
document_name=f"YouTube: {video_name}",
|
||||||
|
search_space_id=search_space_id,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
log_entry = await task_logger.log_task_start(
|
log_entry = await task_logger.log_task_start(
|
||||||
|
|
@ -216,7 +227,10 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
|
||||||
try:
|
try:
|
||||||
# Update notification: parsing (fetching transcript)
|
# Update notification: parsing (fetching transcript)
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await NotificationService.document_processing.notify_processing_progress(
|
||||||
session, notification, stage="parsing", stage_message="Fetching video transcript"
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Fetching video transcript",
|
||||||
)
|
)
|
||||||
|
|
||||||
result = await add_youtube_video_document(
|
result = await add_youtube_video_document(
|
||||||
|
|
@ -235,11 +249,13 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
|
||||||
)
|
)
|
||||||
|
|
||||||
# Update notification on success
|
# Update notification on success
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
document_id=result.id,
|
notification=notification,
|
||||||
chunks_count=None,
|
document_id=result.id,
|
||||||
|
chunks_count=None,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
await task_logger.log_task_success(
|
await task_logger.log_task_success(
|
||||||
|
|
@ -249,10 +265,12 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
|
||||||
)
|
)
|
||||||
|
|
||||||
# Update notification for duplicate
|
# Update notification for duplicate
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
error_message="Video already exists (duplicate)",
|
notification=notification,
|
||||||
|
error_message="Video already exists (duplicate)",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
await task_logger.log_task_failure(
|
await task_logger.log_task_failure(
|
||||||
|
|
@ -317,13 +335,15 @@ async def _process_file_upload(
|
||||||
file_size = None
|
file_size = None
|
||||||
|
|
||||||
# Create notification for document processing
|
# Create notification for document processing
|
||||||
notification = await NotificationService.document_processing.notify_processing_started(
|
notification = (
|
||||||
session=session,
|
await NotificationService.document_processing.notify_processing_started(
|
||||||
user_id=UUID(user_id),
|
session=session,
|
||||||
document_type="FILE",
|
user_id=UUID(user_id),
|
||||||
document_name=filename,
|
document_type="FILE",
|
||||||
search_space_id=search_space_id,
|
document_name=filename,
|
||||||
file_size=file_size,
|
search_space_id=search_space_id,
|
||||||
|
file_size=file_size,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
log_entry = await task_logger.log_task_start(
|
log_entry = await task_logger.log_task_start(
|
||||||
|
|
@ -352,18 +372,22 @@ async def _process_file_upload(
|
||||||
|
|
||||||
# Update notification on success
|
# Update notification on success
|
||||||
if result:
|
if result:
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
document_id=result.id,
|
notification=notification,
|
||||||
chunks_count=None,
|
document_id=result.id,
|
||||||
|
chunks_count=None,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# Duplicate detected
|
# Duplicate detected
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
error_message="Document already exists (duplicate)",
|
notification=notification,
|
||||||
|
error_message="Document already exists (duplicate)",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
@ -456,12 +480,14 @@ async def _process_circleback_meeting(
|
||||||
# Create notification if user_id is available
|
# Create notification if user_id is available
|
||||||
notification = None
|
notification = None
|
||||||
if user_id:
|
if user_id:
|
||||||
notification = await NotificationService.document_processing.notify_processing_started(
|
notification = (
|
||||||
session=session,
|
await NotificationService.document_processing.notify_processing_started(
|
||||||
user_id=UUID(user_id),
|
session=session,
|
||||||
document_type="CIRCLEBACK",
|
user_id=UUID(user_id),
|
||||||
document_name=f"Meeting: {meeting_name[:40]}",
|
document_type="CIRCLEBACK",
|
||||||
search_space_id=search_space_id,
|
document_name=f"Meeting: {meeting_name[:40]}",
|
||||||
|
search_space_id=search_space_id,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
log_entry = await task_logger.log_task_start(
|
log_entry = await task_logger.log_task_start(
|
||||||
|
|
@ -479,8 +505,13 @@ async def _process_circleback_meeting(
|
||||||
try:
|
try:
|
||||||
# Update notification: parsing stage
|
# Update notification: parsing stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await (
|
||||||
session, notification, stage="parsing", stage_message="Reading meeting notes"
|
NotificationService.document_processing.notify_processing_progress(
|
||||||
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Reading meeting notes",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
result = await add_circleback_meeting_document(
|
result = await add_circleback_meeting_document(
|
||||||
|
|
@ -535,10 +566,12 @@ async def _process_circleback_meeting(
|
||||||
|
|
||||||
# Update notification on failure
|
# Update notification on failure
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_completed(
|
await (
|
||||||
session=session,
|
NotificationService.document_processing.notify_processing_completed(
|
||||||
notification=notification,
|
session=session,
|
||||||
error_message=str(e)[:100],
|
notification=notification,
|
||||||
|
error_message=str(e)[:100],
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.error(f"Error processing Circleback meeting: {e!s}")
|
logger.error(f"Error processing Circleback meeting: {e!s}")
|
||||||
|
|
|
||||||
|
|
@ -476,15 +476,21 @@ async def process_file_in_background(
|
||||||
log_entry: Log,
|
log_entry: Log,
|
||||||
connector: dict
|
connector: dict
|
||||||
| None = None, # Optional: {"type": "GOOGLE_DRIVE_FILE", "metadata": {...}}
|
| None = None, # Optional: {"type": "GOOGLE_DRIVE_FILE", "metadata": {...}}
|
||||||
notification: Notification | None = None, # Optional notification for progress updates
|
notification: Notification
|
||||||
|
| None = None, # Optional notification for progress updates
|
||||||
) -> Document | None:
|
) -> Document | None:
|
||||||
try:
|
try:
|
||||||
# Check if the file is a markdown or text file
|
# Check if the file is a markdown or text file
|
||||||
if filename.lower().endswith((".md", ".markdown", ".txt")):
|
if filename.lower().endswith((".md", ".markdown", ".txt")):
|
||||||
# Update notification: parsing stage
|
# Update notification: parsing stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await (
|
||||||
session, notification, stage="parsing", stage_message="Reading file"
|
NotificationService.document_processing.notify_processing_progress(
|
||||||
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Reading file",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
@ -508,8 +514,10 @@ async def process_file_in_background(
|
||||||
|
|
||||||
# Update notification: chunking stage
|
# Update notification: chunking stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await (
|
||||||
session, notification, stage="chunking"
|
NotificationService.document_processing.notify_processing_progress(
|
||||||
|
session, notification, stage="chunking"
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
@ -554,8 +562,13 @@ async def process_file_in_background(
|
||||||
):
|
):
|
||||||
# Update notification: parsing stage (transcription)
|
# Update notification: parsing stage (transcription)
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await (
|
||||||
session, notification, stage="parsing", stage_message="Transcribing audio"
|
NotificationService.document_processing.notify_processing_progress(
|
||||||
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Transcribing audio",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
@ -643,8 +656,10 @@ async def process_file_in_background(
|
||||||
|
|
||||||
# Update notification: chunking stage
|
# Update notification: chunking stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await (
|
||||||
session, notification, stage="chunking"
|
NotificationService.document_processing.notify_processing_progress(
|
||||||
|
session, notification, stage="chunking"
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Clean up the temp file
|
# Clean up the temp file
|
||||||
|
|
@ -749,7 +764,10 @@ async def process_file_in_background(
|
||||||
# Update notification: parsing stage
|
# Update notification: parsing stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await NotificationService.document_processing.notify_processing_progress(
|
||||||
session, notification, stage="parsing", stage_message="Extracting content"
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Extracting content",
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
@ -859,7 +877,10 @@ async def process_file_in_background(
|
||||||
# Update notification: parsing stage
|
# Update notification: parsing stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await NotificationService.document_processing.notify_processing_progress(
|
||||||
session, notification, stage="parsing", stage_message="Extracting content"
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Extracting content",
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
@ -904,7 +925,10 @@ async def process_file_in_background(
|
||||||
# Update notification: chunking stage
|
# Update notification: chunking stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await NotificationService.document_processing.notify_processing_progress(
|
||||||
session, notification, stage="chunking", chunks_count=len(markdown_documents)
|
session,
|
||||||
|
notification,
|
||||||
|
stage="chunking",
|
||||||
|
chunks_count=len(markdown_documents),
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
@ -1018,7 +1042,10 @@ async def process_file_in_background(
|
||||||
# Update notification: parsing stage
|
# Update notification: parsing stage
|
||||||
if notification:
|
if notification:
|
||||||
await NotificationService.document_processing.notify_processing_progress(
|
await NotificationService.document_processing.notify_processing_progress(
|
||||||
session, notification, stage="parsing", stage_message="Extracting content"
|
session,
|
||||||
|
notification,
|
||||||
|
stage="parsing",
|
||||||
|
stage_message="Extracting content",
|
||||||
)
|
)
|
||||||
|
|
||||||
await task_logger.log_task_progress(
|
await task_logger.log_task_progress(
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view
|
||||||
export const ConnectorIndicator: FC = () => {
|
export const ConnectorIndicator: FC = () => {
|
||||||
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
|
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
|
||||||
const searchParams = useSearchParams();
|
const searchParams = useSearchParams();
|
||||||
|
|
||||||
// Fetch document type counts using Electric SQL + PGlite for real-time updates
|
// Fetch document type counts using Electric SQL + PGlite for real-time updates
|
||||||
const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId);
|
const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId);
|
||||||
|
|
||||||
|
|
@ -96,9 +96,10 @@ export const ConnectorIndicator: FC = () => {
|
||||||
} = useConnectorsElectric(searchSpaceId);
|
} = useConnectorsElectric(searchSpaceId);
|
||||||
|
|
||||||
// Fallback to API if Electric fails or is not available
|
// Fallback to API if Electric fails or is not available
|
||||||
const connectors = connectorsFromElectric.length > 0 || !connectorsError
|
const connectors =
|
||||||
? connectorsFromElectric
|
connectorsFromElectric.length > 0 || !connectorsError
|
||||||
: allConnectors || [];
|
? connectorsFromElectric
|
||||||
|
: allConnectors || [];
|
||||||
|
|
||||||
// Manual refresh function that works with both Electric and API
|
// Manual refresh function that works with both Electric and API
|
||||||
const refreshConnectors = async () => {
|
const refreshConnectors = async () => {
|
||||||
|
|
|
||||||
|
|
@ -5,17 +5,17 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Hook to track which connectors are currently indexing using local state.
|
* Hook to track which connectors are currently indexing using local state.
|
||||||
*
|
*
|
||||||
* This provides a better UX than polling by:
|
* This provides a better UX than polling by:
|
||||||
* 1. Setting indexing state immediately when user triggers indexing (optimistic)
|
* 1. Setting indexing state immediately when user triggers indexing (optimistic)
|
||||||
* 2. Clearing indexing state when Electric SQL detects last_indexed_at changed
|
* 2. Clearing indexing state when Electric SQL detects last_indexed_at changed
|
||||||
*
|
*
|
||||||
* The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state.
|
* The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state.
|
||||||
*/
|
*/
|
||||||
export function useIndexingConnectors(connectors: SearchSourceConnector[]) {
|
export function useIndexingConnectors(connectors: SearchSourceConnector[]) {
|
||||||
// Set of connector IDs that are currently indexing
|
// Set of connector IDs that are currently indexing
|
||||||
const [indexingConnectorIds, setIndexingConnectorIds] = useState<Set<number>>(new Set());
|
const [indexingConnectorIds, setIndexingConnectorIds] = useState<Set<number>>(new Set());
|
||||||
|
|
||||||
// Track previous last_indexed_at values to detect changes
|
// Track previous last_indexed_at values to detect changes
|
||||||
const previousLastIndexedAtRef = useRef<Map<number, string | null>>(new Map());
|
const previousLastIndexedAtRef = useRef<Map<number, string | null>>(new Map());
|
||||||
|
|
||||||
|
|
@ -79,4 +79,3 @@ export function useIndexingConnectors(connectors: SearchSourceConnector[]) {
|
||||||
isIndexing,
|
isIndexing,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
||||||
return `${m.replace(/\.0$/, "")}M docs`;
|
return `${m.replace(/\.0$/, "")}M docs`;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
// Document types that should be shown as standalone cards (not from connectors)
|
// Document types that should be shown as standalone cards (not from connectors)
|
||||||
const standaloneDocumentTypes = ["EXTENSION", "FILE", "NOTE", "YOUTUBE_VIDEO", "CRAWLED_URL"];
|
const standaloneDocumentTypes = ["EXTENSION", "FILE", "NOTE", "YOUTUBE_VIDEO", "CRAWLED_URL"];
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
||||||
onManage,
|
onManage,
|
||||||
onViewAccountsList,
|
onViewAccountsList,
|
||||||
}) => {
|
}) => {
|
||||||
|
|
||||||
// Filter connectors based on search
|
// Filter connectors based on search
|
||||||
const filteredOAuth = OAUTH_CONNECTORS.filter(
|
const filteredOAuth = OAUTH_CONNECTORS.filter(
|
||||||
(c) =>
|
(c) =>
|
||||||
|
|
|
||||||
|
|
@ -119,10 +119,7 @@ const DocumentUploadPopupContent: FC<{
|
||||||
<div className="flex-1 min-h-0 relative overflow-hidden">
|
<div className="flex-1 min-h-0 relative overflow-hidden">
|
||||||
<div className="h-full overflow-y-auto">
|
<div className="h-full overflow-y-auto">
|
||||||
<div className="px-6 sm:px-12 pb-5 sm:pb-16">
|
<div className="px-6 sm:px-12 pb-5 sm:pb-16">
|
||||||
<DocumentUploadTab
|
<DocumentUploadTab searchSpaceId={searchSpaceId} onSuccess={handleSuccess} />
|
||||||
searchSpaceId={searchSpaceId}
|
|
||||||
onSuccess={handleSuccess}
|
|
||||||
/>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{/* Bottom fade shadow */}
|
{/* Bottom fade shadow */}
|
||||||
|
|
|
||||||
|
|
@ -27,11 +27,7 @@ export const TooltipIconButton = forwardRef<HTMLButtonElement, TooltipIconButton
|
||||||
<span className="aui-sr-only sr-only">{tooltip}</span>
|
<span className="aui-sr-only sr-only">{tooltip}</span>
|
||||||
</Button>
|
</Button>
|
||||||
</TooltipTrigger>
|
</TooltipTrigger>
|
||||||
<TooltipContent
|
<TooltipContent side={side}>{tooltip}</TooltipContent>
|
||||||
side={side}
|
|
||||||
>
|
|
||||||
{tooltip}
|
|
||||||
</TooltipContent>
|
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,8 @@ import { cn } from "@/lib/utils";
|
||||||
export function NotificationButton() {
|
export function NotificationButton() {
|
||||||
const { data: user } = useAtomValue(currentUserAtom);
|
const { data: user } = useAtomValue(currentUserAtom);
|
||||||
const userId = user?.id ? String(user.id) : null;
|
const userId = user?.id ? String(user.id) : null;
|
||||||
const { notifications, unreadCount, loading, markAsRead, markAllAsRead } = useNotifications(userId);
|
const { notifications, unreadCount, loading, markAsRead, markAllAsRead } =
|
||||||
|
useNotifications(userId);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Popover>
|
<Popover>
|
||||||
|
|
@ -50,4 +51,3 @@ export function NotificationButton() {
|
||||||
</Popover>
|
</Popover>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ export function NotificationPopup({
|
||||||
|
|
||||||
const getStatusIcon = (notification: Notification) => {
|
const getStatusIcon = (notification: Notification) => {
|
||||||
const status = notification.metadata?.status as string | undefined;
|
const status = notification.metadata?.status as string | undefined;
|
||||||
|
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case "in_progress":
|
case "in_progress":
|
||||||
return <Loader2 className="h-4 w-4 text-blue-500 animate-spin" />;
|
return <Loader2 className="h-4 w-4 text-blue-500 animate-spin" />;
|
||||||
|
|
@ -62,12 +62,7 @@ export function NotificationPopup({
|
||||||
<h3 className="font-semibold text-sm">Notifications</h3>
|
<h3 className="font-semibold text-sm">Notifications</h3>
|
||||||
</div>
|
</div>
|
||||||
{unreadCount > 0 && (
|
{unreadCount > 0 && (
|
||||||
<Button
|
<Button variant="ghost" size="sm" onClick={handleMarkAllAsRead} className="h-7 text-xs">
|
||||||
variant="ghost"
|
|
||||||
size="sm"
|
|
||||||
onClick={handleMarkAllAsRead}
|
|
||||||
className="h-7 text-xs"
|
|
||||||
>
|
|
||||||
<CheckCheck className="h-3.5 w-3.5 mr-0" />
|
<CheckCheck className="h-3.5 w-3.5 mr-0" />
|
||||||
Mark all read
|
Mark all read
|
||||||
</Button>
|
</Button>
|
||||||
|
|
@ -98,9 +93,7 @@ export function NotificationPopup({
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
<div className="flex items-start gap-3">
|
<div className="flex items-start gap-3">
|
||||||
<div className="flex-shrink-0 mt-0.5">
|
<div className="flex-shrink-0 mt-0.5">{getStatusIcon(notification)}</div>
|
||||||
{getStatusIcon(notification)}
|
|
||||||
</div>
|
|
||||||
<div className="flex-1 min-w-0">
|
<div className="flex-1 min-w-0">
|
||||||
<div className="flex items-start justify-between gap-2 mb-1">
|
<div className="flex items-start justify-between gap-2 mb-1">
|
||||||
<p
|
<p
|
||||||
|
|
|
||||||
|
|
@ -1,54 +1,54 @@
|
||||||
"use client"
|
"use client";
|
||||||
|
|
||||||
import { useEffect, useState } from 'react'
|
import { useEffect, useState } from "react";
|
||||||
import { initElectric, isElectricInitialized } from '@/lib/electric/client'
|
import { initElectric, isElectricInitialized } from "@/lib/electric/client";
|
||||||
|
|
||||||
interface ElectricProviderProps {
|
interface ElectricProviderProps {
|
||||||
children: React.ReactNode
|
children: React.ReactNode;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ElectricProvider initializes the Electric SQL client with PGlite
|
* ElectricProvider initializes the Electric SQL client with PGlite
|
||||||
*
|
*
|
||||||
* This provider ensures Electric is initialized before rendering children,
|
* This provider ensures Electric is initialized before rendering children,
|
||||||
* but doesn't block if initialization fails (app can still work without real-time sync)
|
* but doesn't block if initialization fails (app can still work without real-time sync)
|
||||||
*/
|
*/
|
||||||
export function ElectricProvider({ children }: ElectricProviderProps) {
|
export function ElectricProvider({ children }: ElectricProviderProps) {
|
||||||
const [initialized, setInitialized] = useState(false)
|
const [initialized, setInitialized] = useState(false);
|
||||||
const [error, setError] = useState<Error | null>(null)
|
const [error, setError] = useState<Error | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
// Skip if already initialized
|
// Skip if already initialized
|
||||||
if (isElectricInitialized()) {
|
if (isElectricInitialized()) {
|
||||||
setInitialized(true)
|
setInitialized(true);
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mounted = true
|
let mounted = true;
|
||||||
|
|
||||||
async function init() {
|
async function init() {
|
||||||
try {
|
try {
|
||||||
await initElectric()
|
await initElectric();
|
||||||
if (mounted) {
|
if (mounted) {
|
||||||
setInitialized(true)
|
setInitialized(true);
|
||||||
setError(null)
|
setError(null);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to initialize Electric SQL:', err)
|
console.error("Failed to initialize Electric SQL:", err);
|
||||||
if (mounted) {
|
if (mounted) {
|
||||||
setError(err instanceof Error ? err : new Error('Failed to initialize Electric SQL'))
|
setError(err instanceof Error ? err : new Error("Failed to initialize Electric SQL"));
|
||||||
// Don't block rendering if Electric SQL fails - app can still work
|
// Don't block rendering if Electric SQL fails - app can still work
|
||||||
setInitialized(true)
|
setInitialized(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
init()
|
init();
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
mounted = false
|
mounted = false;
|
||||||
}
|
};
|
||||||
}, [])
|
}, []);
|
||||||
|
|
||||||
// Show loading state only briefly, then render children
|
// Show loading state only briefly, then render children
|
||||||
// Electric SQL will sync in the background
|
// Electric SQL will sync in the background
|
||||||
|
|
@ -57,13 +57,13 @@ export function ElectricProvider({ children }: ElectricProviderProps) {
|
||||||
<div className="flex items-center justify-center min-h-screen">
|
<div className="flex items-center justify-center min-h-screen">
|
||||||
<div className="text-muted-foreground">Initializing...</div>
|
<div className="text-muted-foreground">Initializing...</div>
|
||||||
</div>
|
</div>
|
||||||
)
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// If there's an error, still render children but log the error
|
// If there's an error, still render children but log the error
|
||||||
if (error) {
|
if (error) {
|
||||||
console.warn('Electric SQL initialization failed, notifications may not sync:', error.message)
|
console.warn("Electric SQL initialization failed, notifications may not sync:", error.message);
|
||||||
}
|
}
|
||||||
|
|
||||||
return <>{children}</>
|
return <>{children}</>;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,19 +5,12 @@ import { documentTypeEnum } from "./document.types";
|
||||||
/**
|
/**
|
||||||
* Notification type enum - matches backend notification types
|
* Notification type enum - matches backend notification types
|
||||||
*/
|
*/
|
||||||
export const notificationTypeEnum = z.enum([
|
export const notificationTypeEnum = z.enum(["connector_indexing", "document_processing"]);
|
||||||
"connector_indexing",
|
|
||||||
"document_processing",
|
|
||||||
]);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Notification status enum - used in metadata
|
* Notification status enum - used in metadata
|
||||||
*/
|
*/
|
||||||
export const notificationStatusEnum = z.enum([
|
export const notificationStatusEnum = z.enum(["in_progress", "completed", "failed"]);
|
||||||
"in_progress",
|
|
||||||
"completed",
|
|
||||||
"failed",
|
|
||||||
]);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Document processing stage enum
|
* Document processing stage enum
|
||||||
|
|
@ -125,4 +118,3 @@ export type NotificationMetadata = z.infer<typeof notificationMetadata>;
|
||||||
export type Notification = z.infer<typeof notification>;
|
export type Notification = z.infer<typeof notification>;
|
||||||
export type ConnectorIndexingNotification = z.infer<typeof connectorIndexingNotification>;
|
export type ConnectorIndexingNotification = z.infer<typeof connectorIndexingNotification>;
|
||||||
export type DocumentProcessingNotification = z.infer<typeof documentProcessingNotification>;
|
export type DocumentProcessingNotification = z.infer<typeof documentProcessingNotification>;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,21 @@
|
||||||
"use client"
|
"use client";
|
||||||
|
|
||||||
import { useEffect, useState, useCallback, useRef } from 'react'
|
import { useEffect, useState, useCallback, useRef } from "react";
|
||||||
import { initElectric, isElectricInitialized, type ElectricClient, type SyncHandle } from '@/lib/electric/client'
|
import {
|
||||||
import type { SearchSourceConnector } from '@/contracts/types/connector.types'
|
initElectric,
|
||||||
|
isElectricInitialized,
|
||||||
|
type ElectricClient,
|
||||||
|
type SyncHandle,
|
||||||
|
} from "@/lib/electric/client";
|
||||||
|
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||||
|
|
||||||
export function useConnectorsElectric(searchSpaceId: number | string | null) {
|
export function useConnectorsElectric(searchSpaceId: number | string | null) {
|
||||||
const [electric, setElectric] = useState<ElectricClient | null>(null)
|
const [electric, setElectric] = useState<ElectricClient | null>(null);
|
||||||
const [connectors, setConnectors] = useState<SearchSourceConnector[]>([])
|
const [connectors, setConnectors] = useState<SearchSourceConnector[]>([]);
|
||||||
const [loading, setLoading] = useState(true)
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<Error | null>(null)
|
const [error, setError] = useState<Error | null>(null);
|
||||||
const syncHandleRef = useRef<SyncHandle | null>(null)
|
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null)
|
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
|
||||||
|
|
||||||
// Transform connector data from Electric SQL/PGlite to match expected format
|
// Transform connector data from Electric SQL/PGlite to match expected format
|
||||||
// Converts Date objects to ISO strings as expected by Zod schema
|
// Converts Date objects to ISO strings as expected by Zod schema
|
||||||
|
|
@ -18,184 +23,193 @@ export function useConnectorsElectric(searchSpaceId: number | string | null) {
|
||||||
return {
|
return {
|
||||||
...connector,
|
...connector,
|
||||||
last_indexed_at: connector.last_indexed_at
|
last_indexed_at: connector.last_indexed_at
|
||||||
? typeof connector.last_indexed_at === 'string'
|
? typeof connector.last_indexed_at === "string"
|
||||||
? connector.last_indexed_at
|
? connector.last_indexed_at
|
||||||
: new Date(connector.last_indexed_at).toISOString()
|
: new Date(connector.last_indexed_at).toISOString()
|
||||||
: null,
|
: null,
|
||||||
next_scheduled_at: connector.next_scheduled_at
|
next_scheduled_at: connector.next_scheduled_at
|
||||||
? typeof connector.next_scheduled_at === 'string'
|
? typeof connector.next_scheduled_at === "string"
|
||||||
? connector.next_scheduled_at
|
? connector.next_scheduled_at
|
||||||
: new Date(connector.next_scheduled_at).toISOString()
|
: new Date(connector.next_scheduled_at).toISOString()
|
||||||
: null,
|
: null,
|
||||||
created_at: connector.created_at
|
created_at: connector.created_at
|
||||||
? typeof connector.created_at === 'string'
|
? typeof connector.created_at === "string"
|
||||||
? connector.created_at
|
? connector.created_at
|
||||||
: new Date(connector.created_at).toISOString()
|
: new Date(connector.created_at).toISOString()
|
||||||
: new Date().toISOString(), // fallback
|
: new Date().toISOString(), // fallback
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize Electric SQL and start syncing with real-time updates
|
// Initialize Electric SQL and start syncing with real-time updates
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!searchSpaceId) {
|
if (!searchSpaceId) {
|
||||||
setLoading(false)
|
setLoading(false);
|
||||||
setConnectors([])
|
setConnectors([]);
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mounted = true
|
let mounted = true;
|
||||||
|
|
||||||
async function init() {
|
async function init() {
|
||||||
try {
|
try {
|
||||||
const electricClient = await initElectric()
|
const electricClient = await initElectric();
|
||||||
if (!mounted) return
|
if (!mounted) return;
|
||||||
|
|
||||||
setElectric(electricClient)
|
setElectric(electricClient);
|
||||||
|
|
||||||
// Start syncing connectors for this search space via Electric SQL
|
// Start syncing connectors for this search space via Electric SQL
|
||||||
console.log('Starting Electric SQL sync for connectors, search_space_id:', searchSpaceId)
|
console.log("Starting Electric SQL sync for connectors, search_space_id:", searchSpaceId);
|
||||||
|
|
||||||
// Use numeric format for WHERE clause (PGlite sync plugin expects this format)
|
// Use numeric format for WHERE clause (PGlite sync plugin expects this format)
|
||||||
const handle = await electricClient.syncShape({
|
const handle = await electricClient.syncShape({
|
||||||
table: 'search_source_connectors',
|
table: "search_source_connectors",
|
||||||
where: `search_space_id = ${searchSpaceId}`,
|
where: `search_space_id = ${searchSpaceId}`,
|
||||||
primaryKey: ['id'],
|
primaryKey: ["id"],
|
||||||
})
|
});
|
||||||
|
|
||||||
console.log('Electric SQL sync started for connectors:', {
|
console.log("Electric SQL sync started for connectors:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
hasStream: !!handle.stream,
|
hasStream: !!handle.stream,
|
||||||
hasInitialSyncPromise: !!handle.initialSyncPromise,
|
hasInitialSyncPromise: !!handle.initialSyncPromise,
|
||||||
})
|
});
|
||||||
|
|
||||||
// Optimized: Check if already up-to-date before waiting
|
// Optimized: Check if already up-to-date before waiting
|
||||||
if (handle.isUpToDate) {
|
if (handle.isUpToDate) {
|
||||||
console.log('Connectors sync already up-to-date, skipping wait')
|
console.log("Connectors sync already up-to-date, skipping wait");
|
||||||
} else if (handle.initialSyncPromise) {
|
} else if (handle.initialSyncPromise) {
|
||||||
// Only wait if not already up-to-date
|
// Only wait if not already up-to-date
|
||||||
console.log('Waiting for initial connectors sync to complete...')
|
console.log("Waiting for initial connectors sync to complete...");
|
||||||
try {
|
try {
|
||||||
// Use Promise.race with a shorter timeout to avoid long waits
|
// Use Promise.race with a shorter timeout to avoid long waits
|
||||||
await Promise.race([
|
await Promise.race([
|
||||||
handle.initialSyncPromise,
|
handle.initialSyncPromise,
|
||||||
new Promise(resolve => setTimeout(resolve, 2000)), // Max 2s wait
|
new Promise((resolve) => setTimeout(resolve, 2000)), // Max 2s wait
|
||||||
])
|
]);
|
||||||
console.log('Initial connectors sync promise resolved or timed out, checking status:', {
|
console.log("Initial connectors sync promise resolved or timed out, checking status:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
})
|
});
|
||||||
} catch (syncErr) {
|
} catch (syncErr) {
|
||||||
console.error('Initial connectors sync failed:', syncErr)
|
console.error("Initial connectors sync failed:", syncErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check status after waiting
|
// Check status after waiting
|
||||||
console.log('Connectors sync status after waiting:', {
|
console.log("Connectors sync status after waiting:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
hasStream: !!handle.stream,
|
hasStream: !!handle.stream,
|
||||||
})
|
});
|
||||||
|
|
||||||
if (!mounted) {
|
if (!mounted) {
|
||||||
handle.unsubscribe()
|
handle.unsubscribe();
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
syncHandleRef.current = handle
|
syncHandleRef.current = handle;
|
||||||
setLoading(false)
|
setLoading(false);
|
||||||
setError(null)
|
setError(null);
|
||||||
|
|
||||||
// Fetch connectors after sync is complete (we already waited above)
|
// Fetch connectors after sync is complete (we already waited above)
|
||||||
await fetchConnectors(electricClient.db)
|
await fetchConnectors(electricClient.db);
|
||||||
|
|
||||||
// Set up real-time updates using PGlite live queries
|
// Set up real-time updates using PGlite live queries
|
||||||
// Electric SQL syncs data to PGlite in real-time via HTTP streaming
|
// Electric SQL syncs data to PGlite in real-time via HTTP streaming
|
||||||
// PGlite live queries detect when the synced data changes and trigger callbacks
|
// PGlite live queries detect when the synced data changes and trigger callbacks
|
||||||
try {
|
try {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const db = electricClient.db as any
|
const db = electricClient.db as any;
|
||||||
|
|
||||||
// Use PGlite's live query API for real-time updates
|
// Use PGlite's live query API for real-time updates
|
||||||
// CORRECT API: await db.live.query() then use .subscribe()
|
// CORRECT API: await db.live.query() then use .subscribe()
|
||||||
if (db.live?.query && typeof db.live.query === 'function') {
|
if (db.live?.query && typeof db.live.query === "function") {
|
||||||
// IMPORTANT: db.live.query() returns a Promise - must await it!
|
// IMPORTANT: db.live.query() returns a Promise - must await it!
|
||||||
const liveQuery = await db.live.query(
|
const liveQuery = await db.live.query(
|
||||||
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
||||||
[searchSpaceId]
|
[searchSpaceId]
|
||||||
)
|
);
|
||||||
|
|
||||||
if (!mounted) {
|
if (!mounted) {
|
||||||
liveQuery.unsubscribe?.()
|
liveQuery.unsubscribe?.();
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set initial results immediately from the resolved query
|
// Set initial results immediately from the resolved query
|
||||||
if (liveQuery.initialResults?.rows) {
|
if (liveQuery.initialResults?.rows) {
|
||||||
console.log('📋 Initial live query results for connectors:', liveQuery.initialResults.rows.length)
|
console.log(
|
||||||
setConnectors(liveQuery.initialResults.rows.map(transformConnector))
|
"📋 Initial live query results for connectors:",
|
||||||
|
liveQuery.initialResults.rows.length
|
||||||
|
);
|
||||||
|
setConnectors(liveQuery.initialResults.rows.map(transformConnector));
|
||||||
} else if (liveQuery.rows) {
|
} else if (liveQuery.rows) {
|
||||||
// Some versions have rows directly on the result
|
// Some versions have rows directly on the result
|
||||||
console.log('📋 Initial live query results for connectors (direct):', liveQuery.rows.length)
|
console.log(
|
||||||
setConnectors(liveQuery.rows.map(transformConnector))
|
"📋 Initial live query results for connectors (direct):",
|
||||||
|
liveQuery.rows.length
|
||||||
|
);
|
||||||
|
setConnectors(liveQuery.rows.map(transformConnector));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subscribe to changes - this is the correct API!
|
// Subscribe to changes - this is the correct API!
|
||||||
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
||||||
if (typeof liveQuery.subscribe === 'function') {
|
if (typeof liveQuery.subscribe === "function") {
|
||||||
liveQuery.subscribe((result: { rows: any[] }) => {
|
liveQuery.subscribe((result: { rows: any[] }) => {
|
||||||
if (mounted && result.rows) {
|
if (mounted && result.rows) {
|
||||||
console.log('🔄 Connectors updated via live query:', result.rows.length)
|
console.log("🔄 Connectors updated via live query:", result.rows.length);
|
||||||
setConnectors(result.rows.map(transformConnector))
|
setConnectors(result.rows.map(transformConnector));
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
|
|
||||||
// Store unsubscribe function for cleanup
|
// Store unsubscribe function for cleanup
|
||||||
liveQueryRef.current = liveQuery
|
liveQueryRef.current = liveQuery;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
console.warn('PGlite live query API not available, falling back to polling')
|
console.warn("PGlite live query API not available, falling back to polling");
|
||||||
}
|
}
|
||||||
} catch (liveQueryErr) {
|
} catch (liveQueryErr) {
|
||||||
console.error('Failed to set up live query for connectors:', liveQueryErr)
|
console.error("Failed to set up live query for connectors:", liveQueryErr);
|
||||||
// Don't fail completely - we still have the initial fetch
|
// Don't fail completely - we still have the initial fetch
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to initialize Electric SQL for connectors:', err)
|
console.error("Failed to initialize Electric SQL for connectors:", err);
|
||||||
if (mounted) {
|
if (mounted) {
|
||||||
setError(err instanceof Error ? err : new Error('Failed to initialize Electric SQL for connectors'))
|
setError(
|
||||||
setLoading(false)
|
err instanceof Error
|
||||||
|
? err
|
||||||
|
: new Error("Failed to initialize Electric SQL for connectors")
|
||||||
|
);
|
||||||
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
init()
|
init();
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
mounted = false
|
mounted = false;
|
||||||
syncHandleRef.current?.unsubscribe?.()
|
syncHandleRef.current?.unsubscribe?.();
|
||||||
liveQueryRef.current?.unsubscribe?.()
|
liveQueryRef.current?.unsubscribe?.();
|
||||||
syncHandleRef.current = null
|
syncHandleRef.current = null;
|
||||||
liveQueryRef.current = null
|
liveQueryRef.current = null;
|
||||||
}
|
};
|
||||||
}, [searchSpaceId])
|
}, [searchSpaceId]);
|
||||||
|
|
||||||
async function fetchConnectors(db: any) {
|
async function fetchConnectors(db: any) {
|
||||||
try {
|
try {
|
||||||
const result = await db.query(
|
const result = await db.query(
|
||||||
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
||||||
[searchSpaceId]
|
[searchSpaceId]
|
||||||
)
|
);
|
||||||
console.log('📋 Fetched connectors from PGlite:', result.rows?.length || 0)
|
console.log("📋 Fetched connectors from PGlite:", result.rows?.length || 0);
|
||||||
setConnectors((result.rows || []).map(transformConnector))
|
setConnectors((result.rows || []).map(transformConnector));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to fetch connectors from PGlite:', err)
|
console.error("Failed to fetch connectors from PGlite:", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Manual refresh function (optional, for fallback)
|
// Manual refresh function (optional, for fallback)
|
||||||
const refreshConnectors = useCallback(async () => {
|
const refreshConnectors = useCallback(async () => {
|
||||||
if (!electric) return
|
if (!electric) return;
|
||||||
await fetchConnectors(electric.db)
|
await fetchConnectors(electric.db);
|
||||||
}, [electric])
|
}, [electric]);
|
||||||
|
|
||||||
return { connectors, loading, error, refreshConnectors }
|
return { connectors, loading, error, refreshConnectors };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,190 +1,201 @@
|
||||||
"use client"
|
"use client";
|
||||||
|
|
||||||
import { useEffect, useState, useRef, useMemo } from 'react'
|
import { useEffect, useState, useRef, useMemo } from "react";
|
||||||
import { initElectric, type ElectricClient, type SyncHandle } from '@/lib/electric/client'
|
import { initElectric, type ElectricClient, type SyncHandle } from "@/lib/electric/client";
|
||||||
|
|
||||||
interface Document {
|
interface Document {
|
||||||
id: number
|
id: number;
|
||||||
search_space_id: number
|
search_space_id: number;
|
||||||
document_type: string
|
document_type: string;
|
||||||
created_at: string
|
created_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useDocumentsElectric(searchSpaceId: number | string | null) {
|
export function useDocumentsElectric(searchSpaceId: number | string | null) {
|
||||||
const [electric, setElectric] = useState<ElectricClient | null>(null)
|
const [electric, setElectric] = useState<ElectricClient | null>(null);
|
||||||
const [documents, setDocuments] = useState<Document[]>([])
|
const [documents, setDocuments] = useState<Document[]>([]);
|
||||||
const [loading, setLoading] = useState(true)
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<Error | null>(null)
|
const [error, setError] = useState<Error | null>(null);
|
||||||
const syncHandleRef = useRef<SyncHandle | null>(null)
|
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null)
|
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
|
||||||
|
|
||||||
// Calculate document type counts from synced documents
|
// Calculate document type counts from synced documents
|
||||||
const documentTypeCounts = useMemo(() => {
|
const documentTypeCounts = useMemo(() => {
|
||||||
if (!documents.length) return {}
|
if (!documents.length) return {};
|
||||||
|
|
||||||
const counts: Record<string, number> = {}
|
const counts: Record<string, number> = {};
|
||||||
for (const doc of documents) {
|
for (const doc of documents) {
|
||||||
counts[doc.document_type] = (counts[doc.document_type] || 0) + 1
|
counts[doc.document_type] = (counts[doc.document_type] || 0) + 1;
|
||||||
}
|
}
|
||||||
return counts
|
return counts;
|
||||||
}, [documents])
|
}, [documents]);
|
||||||
|
|
||||||
// Initialize Electric SQL and start syncing with real-time updates
|
// Initialize Electric SQL and start syncing with real-time updates
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!searchSpaceId) {
|
if (!searchSpaceId) {
|
||||||
setLoading(false)
|
setLoading(false);
|
||||||
setDocuments([])
|
setDocuments([]);
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mounted = true
|
let mounted = true;
|
||||||
|
|
||||||
async function init() {
|
async function init() {
|
||||||
try {
|
try {
|
||||||
const electricClient = await initElectric()
|
const electricClient = await initElectric();
|
||||||
if (!mounted) return
|
if (!mounted) return;
|
||||||
|
|
||||||
setElectric(electricClient)
|
setElectric(electricClient);
|
||||||
|
|
||||||
// Start syncing documents for this search space via Electric SQL
|
// Start syncing documents for this search space via Electric SQL
|
||||||
// Only sync id, document_type, search_space_id columns for efficiency
|
// Only sync id, document_type, search_space_id columns for efficiency
|
||||||
console.log('Starting Electric SQL sync for documents, search_space_id:', searchSpaceId)
|
console.log("Starting Electric SQL sync for documents, search_space_id:", searchSpaceId);
|
||||||
|
|
||||||
const handle = await electricClient.syncShape({
|
const handle = await electricClient.syncShape({
|
||||||
table: 'documents',
|
table: "documents",
|
||||||
where: `search_space_id = ${searchSpaceId}`,
|
where: `search_space_id = ${searchSpaceId}`,
|
||||||
columns: ['id', 'document_type', 'search_space_id', 'created_at'],
|
columns: ["id", "document_type", "search_space_id", "created_at"],
|
||||||
primaryKey: ['id'],
|
primaryKey: ["id"],
|
||||||
})
|
});
|
||||||
|
|
||||||
console.log('Electric SQL sync started for documents:', {
|
console.log("Electric SQL sync started for documents:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
hasStream: !!handle.stream,
|
hasStream: !!handle.stream,
|
||||||
hasInitialSyncPromise: !!handle.initialSyncPromise,
|
hasInitialSyncPromise: !!handle.initialSyncPromise,
|
||||||
})
|
});
|
||||||
|
|
||||||
// Optimized: Check if already up-to-date before waiting
|
// Optimized: Check if already up-to-date before waiting
|
||||||
if (handle.isUpToDate) {
|
if (handle.isUpToDate) {
|
||||||
console.log('Documents sync already up-to-date, skipping wait')
|
console.log("Documents sync already up-to-date, skipping wait");
|
||||||
} else if (handle.initialSyncPromise) {
|
} else if (handle.initialSyncPromise) {
|
||||||
// Only wait if not already up-to-date
|
// Only wait if not already up-to-date
|
||||||
console.log('Waiting for initial documents sync to complete...')
|
console.log("Waiting for initial documents sync to complete...");
|
||||||
try {
|
try {
|
||||||
// Use Promise.race with a shorter timeout to avoid long waits
|
// Use Promise.race with a shorter timeout to avoid long waits
|
||||||
await Promise.race([
|
await Promise.race([
|
||||||
handle.initialSyncPromise,
|
handle.initialSyncPromise,
|
||||||
new Promise(resolve => setTimeout(resolve, 2000)), // Max 2s wait
|
new Promise((resolve) => setTimeout(resolve, 2000)), // Max 2s wait
|
||||||
])
|
]);
|
||||||
console.log('Initial documents sync promise resolved or timed out, checking status:', {
|
console.log("Initial documents sync promise resolved or timed out, checking status:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
})
|
});
|
||||||
} catch (syncErr) {
|
} catch (syncErr) {
|
||||||
console.error('Initial documents sync failed:', syncErr)
|
console.error("Initial documents sync failed:", syncErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check status after waiting
|
// Check status after waiting
|
||||||
console.log('Documents sync status after waiting:', {
|
console.log("Documents sync status after waiting:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
hasStream: !!handle.stream,
|
hasStream: !!handle.stream,
|
||||||
})
|
});
|
||||||
|
|
||||||
if (!mounted) {
|
if (!mounted) {
|
||||||
handle.unsubscribe()
|
handle.unsubscribe();
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
syncHandleRef.current = handle
|
syncHandleRef.current = handle;
|
||||||
setLoading(false)
|
setLoading(false);
|
||||||
setError(null)
|
setError(null);
|
||||||
|
|
||||||
// Fetch documents after sync is complete (we already waited above)
|
// Fetch documents after sync is complete (we already waited above)
|
||||||
await fetchDocuments(electricClient.db)
|
await fetchDocuments(electricClient.db);
|
||||||
|
|
||||||
// Set up real-time updates using PGlite live queries
|
// Set up real-time updates using PGlite live queries
|
||||||
// Electric SQL syncs data to PGlite in real-time via HTTP streaming
|
// Electric SQL syncs data to PGlite in real-time via HTTP streaming
|
||||||
// PGlite live queries detect when the synced data changes and trigger callbacks
|
// PGlite live queries detect when the synced data changes and trigger callbacks
|
||||||
try {
|
try {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const db = electricClient.db as any
|
const db = electricClient.db as any;
|
||||||
|
|
||||||
// Use PGlite's live query API for real-time updates
|
// Use PGlite's live query API for real-time updates
|
||||||
// CORRECT API: await db.live.query() then use .subscribe()
|
// CORRECT API: await db.live.query() then use .subscribe()
|
||||||
if (db.live?.query && typeof db.live.query === 'function') {
|
if (db.live?.query && typeof db.live.query === "function") {
|
||||||
// IMPORTANT: db.live.query() returns a Promise - must await it!
|
// IMPORTANT: db.live.query() returns a Promise - must await it!
|
||||||
const liveQuery = await db.live.query(
|
const liveQuery = await db.live.query(
|
||||||
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
||||||
[searchSpaceId]
|
[searchSpaceId]
|
||||||
)
|
);
|
||||||
|
|
||||||
if (!mounted) {
|
if (!mounted) {
|
||||||
liveQuery.unsubscribe?.()
|
liveQuery.unsubscribe?.();
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set initial results immediately from the resolved query
|
// Set initial results immediately from the resolved query
|
||||||
if (liveQuery.initialResults?.rows) {
|
if (liveQuery.initialResults?.rows) {
|
||||||
console.log('📋 Initial live query results for documents:', liveQuery.initialResults.rows.length)
|
console.log(
|
||||||
setDocuments(liveQuery.initialResults.rows)
|
"📋 Initial live query results for documents:",
|
||||||
|
liveQuery.initialResults.rows.length
|
||||||
|
);
|
||||||
|
setDocuments(liveQuery.initialResults.rows);
|
||||||
} else if (liveQuery.rows) {
|
} else if (liveQuery.rows) {
|
||||||
// Some versions have rows directly on the result
|
// Some versions have rows directly on the result
|
||||||
console.log('📋 Initial live query results for documents (direct):', liveQuery.rows.length)
|
console.log(
|
||||||
setDocuments(liveQuery.rows)
|
"📋 Initial live query results for documents (direct):",
|
||||||
|
liveQuery.rows.length
|
||||||
|
);
|
||||||
|
setDocuments(liveQuery.rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subscribe to changes - this is the correct API!
|
// Subscribe to changes - this is the correct API!
|
||||||
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
||||||
if (typeof liveQuery.subscribe === 'function') {
|
if (typeof liveQuery.subscribe === "function") {
|
||||||
liveQuery.subscribe((result: { rows: Document[] }) => {
|
liveQuery.subscribe((result: { rows: Document[] }) => {
|
||||||
if (mounted && result.rows) {
|
if (mounted && result.rows) {
|
||||||
console.log('🔄 Documents updated via live query:', result.rows.length)
|
console.log("🔄 Documents updated via live query:", result.rows.length);
|
||||||
setDocuments(result.rows)
|
setDocuments(result.rows);
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
|
|
||||||
// Store unsubscribe function for cleanup
|
// Store unsubscribe function for cleanup
|
||||||
liveQueryRef.current = liveQuery
|
liveQueryRef.current = liveQuery;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
console.warn('PGlite live query API not available for documents, falling back to polling')
|
console.warn(
|
||||||
|
"PGlite live query API not available for documents, falling back to polling"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} catch (liveQueryErr) {
|
} catch (liveQueryErr) {
|
||||||
console.error('Failed to set up live query for documents:', liveQueryErr)
|
console.error("Failed to set up live query for documents:", liveQueryErr);
|
||||||
// Don't fail completely - we still have the initial fetch
|
// Don't fail completely - we still have the initial fetch
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to initialize Electric SQL for documents:', err)
|
console.error("Failed to initialize Electric SQL for documents:", err);
|
||||||
if (mounted) {
|
if (mounted) {
|
||||||
setError(err instanceof Error ? err : new Error('Failed to initialize Electric SQL for documents'))
|
setError(
|
||||||
setLoading(false)
|
err instanceof Error
|
||||||
|
? err
|
||||||
|
: new Error("Failed to initialize Electric SQL for documents")
|
||||||
|
);
|
||||||
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
init()
|
init();
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
mounted = false
|
mounted = false;
|
||||||
syncHandleRef.current?.unsubscribe?.()
|
syncHandleRef.current?.unsubscribe?.();
|
||||||
liveQueryRef.current?.unsubscribe?.()
|
liveQueryRef.current?.unsubscribe?.();
|
||||||
syncHandleRef.current = null
|
syncHandleRef.current = null;
|
||||||
liveQueryRef.current = null
|
liveQueryRef.current = null;
|
||||||
}
|
};
|
||||||
}, [searchSpaceId])
|
}, [searchSpaceId]);
|
||||||
|
|
||||||
async function fetchDocuments(db: any) {
|
async function fetchDocuments(db: any) {
|
||||||
try {
|
try {
|
||||||
const result = await db.query(
|
const result = await db.query(
|
||||||
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
|
||||||
[searchSpaceId]
|
[searchSpaceId]
|
||||||
)
|
);
|
||||||
console.log('📋 Fetched documents from PGlite:', result.rows?.length || 0)
|
console.log("📋 Fetched documents from PGlite:", result.rows?.length || 0);
|
||||||
setDocuments(result.rows || [])
|
setDocuments(result.rows || []);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to fetch documents from PGlite:', err)
|
console.error("Failed to fetch documents from PGlite:", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { documentTypeCounts, loading, error }
|
return { documentTypeCounts, loading, error };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,211 +1,226 @@
|
||||||
"use client"
|
"use client";
|
||||||
|
|
||||||
import { useEffect, useState, useCallback, useRef } from 'react'
|
import { useEffect, useState, useCallback, useRef } from "react";
|
||||||
import { initElectric, isElectricInitialized, type ElectricClient, type SyncHandle } from '@/lib/electric/client'
|
import {
|
||||||
import type { Notification } from '@/contracts/types/notification.types'
|
initElectric,
|
||||||
|
isElectricInitialized,
|
||||||
|
type ElectricClient,
|
||||||
|
type SyncHandle,
|
||||||
|
} from "@/lib/electric/client";
|
||||||
|
import type { Notification } from "@/contracts/types/notification.types";
|
||||||
|
|
||||||
export type { Notification } from '@/contracts/types/notification.types'
|
export type { Notification } from "@/contracts/types/notification.types";
|
||||||
|
|
||||||
export function useNotifications(userId: string | null) {
|
export function useNotifications(userId: string | null) {
|
||||||
const [electric, setElectric] = useState<ElectricClient | null>(null)
|
const [electric, setElectric] = useState<ElectricClient | null>(null);
|
||||||
const [notifications, setNotifications] = useState<Notification[]>([])
|
const [notifications, setNotifications] = useState<Notification[]>([]);
|
||||||
const [loading, setLoading] = useState(true)
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<Error | null>(null)
|
const [error, setError] = useState<Error | null>(null);
|
||||||
const syncHandleRef = useRef<SyncHandle | null>(null)
|
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null)
|
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
|
||||||
// Use ref instead of state to track initialization - prevents cleanup from running when set
|
// Use ref instead of state to track initialization - prevents cleanup from running when set
|
||||||
const initializedRef = useRef(false)
|
const initializedRef = useRef(false);
|
||||||
|
|
||||||
// Initialize Electric SQL and start syncing with real-time updates
|
// Initialize Electric SQL and start syncing with real-time updates
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
// Use ref to prevent re-initialization without triggering cleanup
|
// Use ref to prevent re-initialization without triggering cleanup
|
||||||
if (!userId || initializedRef.current) return
|
if (!userId || initializedRef.current) return;
|
||||||
initializedRef.current = true
|
initializedRef.current = true;
|
||||||
|
|
||||||
let mounted = true
|
let mounted = true;
|
||||||
|
|
||||||
async function init() {
|
async function init() {
|
||||||
try {
|
try {
|
||||||
const electricClient = await initElectric()
|
const electricClient = await initElectric();
|
||||||
if (!mounted) return
|
if (!mounted) return;
|
||||||
|
|
||||||
setElectric(electricClient)
|
setElectric(electricClient);
|
||||||
|
|
||||||
// Start syncing notifications for this user via Electric SQL
|
// Start syncing notifications for this user via Electric SQL
|
||||||
// Note: user_id is stored as TEXT in PGlite (UUID from backend is converted)
|
// Note: user_id is stored as TEXT in PGlite (UUID from backend is converted)
|
||||||
console.log('Starting Electric SQL sync for user:', userId)
|
console.log("Starting Electric SQL sync for user:", userId);
|
||||||
|
|
||||||
// Use string format for WHERE clause (PGlite sync plugin expects this format)
|
// Use string format for WHERE clause (PGlite sync plugin expects this format)
|
||||||
// The user_id is a UUID string, so we need to quote it properly
|
// The user_id is a UUID string, so we need to quote it properly
|
||||||
const handle = await electricClient.syncShape({
|
const handle = await electricClient.syncShape({
|
||||||
table: 'notifications',
|
table: "notifications",
|
||||||
where: `user_id = '${userId}'`,
|
where: `user_id = '${userId}'`,
|
||||||
primaryKey: ['id'],
|
primaryKey: ["id"],
|
||||||
})
|
});
|
||||||
|
|
||||||
console.log('Electric SQL sync started:', {
|
console.log("Electric SQL sync started:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
hasStream: !!handle.stream,
|
hasStream: !!handle.stream,
|
||||||
hasInitialSyncPromise: !!handle.initialSyncPromise,
|
hasInitialSyncPromise: !!handle.initialSyncPromise,
|
||||||
})
|
});
|
||||||
|
|
||||||
// Optimized: Check if already up-to-date before waiting
|
// Optimized: Check if already up-to-date before waiting
|
||||||
if (handle.isUpToDate) {
|
if (handle.isUpToDate) {
|
||||||
console.log('Sync already up-to-date, skipping wait')
|
console.log("Sync already up-to-date, skipping wait");
|
||||||
} else if (handle.initialSyncPromise) {
|
} else if (handle.initialSyncPromise) {
|
||||||
// Only wait if not already up-to-date
|
// Only wait if not already up-to-date
|
||||||
console.log('Waiting for initial sync to complete...')
|
console.log("Waiting for initial sync to complete...");
|
||||||
try {
|
try {
|
||||||
// Use Promise.race with a shorter timeout to avoid long waits
|
// Use Promise.race with a shorter timeout to avoid long waits
|
||||||
await Promise.race([
|
await Promise.race([
|
||||||
handle.initialSyncPromise,
|
handle.initialSyncPromise,
|
||||||
new Promise(resolve => setTimeout(resolve, 2000)), // Max 2s wait
|
new Promise((resolve) => setTimeout(resolve, 2000)), // Max 2s wait
|
||||||
])
|
]);
|
||||||
console.log('Initial sync promise resolved or timed out, checking status:', {
|
console.log("Initial sync promise resolved or timed out, checking status:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
})
|
});
|
||||||
} catch (syncErr) {
|
} catch (syncErr) {
|
||||||
console.error('Initial sync failed:', syncErr)
|
console.error("Initial sync failed:", syncErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check status after waiting
|
// Check status after waiting
|
||||||
console.log('Sync status after waiting:', {
|
console.log("Sync status after waiting:", {
|
||||||
isUpToDate: handle.isUpToDate,
|
isUpToDate: handle.isUpToDate,
|
||||||
hasStream: !!handle.stream,
|
hasStream: !!handle.stream,
|
||||||
})
|
});
|
||||||
|
|
||||||
if (!mounted) {
|
if (!mounted) {
|
||||||
handle.unsubscribe()
|
handle.unsubscribe();
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
syncHandleRef.current = handle
|
syncHandleRef.current = handle;
|
||||||
setLoading(false)
|
setLoading(false);
|
||||||
setError(null)
|
setError(null);
|
||||||
|
|
||||||
// Fetch notifications after sync is complete (we already waited above)
|
// Fetch notifications after sync is complete (we already waited above)
|
||||||
await fetchNotifications(electricClient.db)
|
await fetchNotifications(electricClient.db);
|
||||||
|
|
||||||
// Set up real-time updates using PGlite live queries
|
// Set up real-time updates using PGlite live queries
|
||||||
// Electric SQL syncs data to PGlite in real-time via HTTP streaming
|
// Electric SQL syncs data to PGlite in real-time via HTTP streaming
|
||||||
// PGlite live queries detect when the synced data changes and trigger callbacks
|
// PGlite live queries detect when the synced data changes and trigger callbacks
|
||||||
try {
|
try {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const db = electricClient.db as any
|
const db = electricClient.db as any;
|
||||||
|
|
||||||
// Use PGlite's live query API for real-time updates
|
// Use PGlite's live query API for real-time updates
|
||||||
// CORRECT API: await db.live.query() then use .subscribe()
|
// CORRECT API: await db.live.query() then use .subscribe()
|
||||||
if (db.live?.query && typeof db.live.query === 'function') {
|
if (db.live?.query && typeof db.live.query === "function") {
|
||||||
// IMPORTANT: db.live.query() returns a Promise - must await it!
|
// IMPORTANT: db.live.query() returns a Promise - must await it!
|
||||||
const liveQuery = await db.live.query(
|
const liveQuery = await db.live.query(
|
||||||
`SELECT * FROM notifications WHERE user_id = $1 ORDER BY created_at DESC`,
|
`SELECT * FROM notifications WHERE user_id = $1 ORDER BY created_at DESC`,
|
||||||
[userId]
|
[userId]
|
||||||
)
|
);
|
||||||
|
|
||||||
if (!mounted) {
|
if (!mounted) {
|
||||||
liveQuery.unsubscribe?.()
|
liveQuery.unsubscribe?.();
|
||||||
return
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set initial results immediately from the resolved query
|
// Set initial results immediately from the resolved query
|
||||||
if (liveQuery.initialResults?.rows) {
|
if (liveQuery.initialResults?.rows) {
|
||||||
console.log('📋 Initial live query results:', liveQuery.initialResults.rows.length)
|
console.log("📋 Initial live query results:", liveQuery.initialResults.rows.length);
|
||||||
setNotifications(liveQuery.initialResults.rows)
|
setNotifications(liveQuery.initialResults.rows);
|
||||||
} else if (liveQuery.rows) {
|
} else if (liveQuery.rows) {
|
||||||
// Some versions have rows directly on the result
|
// Some versions have rows directly on the result
|
||||||
console.log('📋 Initial live query results (direct):', liveQuery.rows.length)
|
console.log("📋 Initial live query results (direct):", liveQuery.rows.length);
|
||||||
setNotifications(liveQuery.rows)
|
setNotifications(liveQuery.rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subscribe to changes - this is the correct API!
|
// Subscribe to changes - this is the correct API!
|
||||||
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
// The callback fires automatically when Electric SQL syncs new data to PGlite
|
||||||
if (typeof liveQuery.subscribe === 'function') {
|
if (typeof liveQuery.subscribe === "function") {
|
||||||
liveQuery.subscribe((result: { rows: Notification[] }) => {
|
liveQuery.subscribe((result: { rows: Notification[] }) => {
|
||||||
console.log('🔔 Live query update received:', result.rows?.length || 0, 'notifications')
|
console.log(
|
||||||
|
"🔔 Live query update received:",
|
||||||
|
result.rows?.length || 0,
|
||||||
|
"notifications"
|
||||||
|
);
|
||||||
if (mounted && result.rows) {
|
if (mounted && result.rows) {
|
||||||
setNotifications(result.rows)
|
setNotifications(result.rows);
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
console.log('✅ Real-time notifications enabled via PGlite live queries')
|
console.log("✅ Real-time notifications enabled via PGlite live queries");
|
||||||
} else {
|
} else {
|
||||||
console.warn('⚠️ Live query subscribe method not available')
|
console.warn("⚠️ Live query subscribe method not available");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store for cleanup
|
// Store for cleanup
|
||||||
if (typeof liveQuery.unsubscribe === 'function') {
|
if (typeof liveQuery.unsubscribe === "function") {
|
||||||
liveQueryRef.current = liveQuery
|
liveQueryRef.current = liveQuery;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
console.error('❌ PGlite live queries not available - db.live.query is not a function')
|
console.error("❌ PGlite live queries not available - db.live.query is not a function");
|
||||||
console.log('db.live:', db.live)
|
console.log("db.live:", db.live);
|
||||||
}
|
}
|
||||||
} catch (liveErr) {
|
} catch (liveErr) {
|
||||||
console.error('❌ Failed to set up real-time updates:', liveErr)
|
console.error("❌ Failed to set up real-time updates:", liveErr);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (!mounted) return
|
if (!mounted) return;
|
||||||
console.error('Failed to initialize Electric SQL:', err)
|
console.error("Failed to initialize Electric SQL:", err);
|
||||||
setError(err instanceof Error ? err : new Error('Failed to initialize Electric SQL'))
|
setError(err instanceof Error ? err : new Error("Failed to initialize Electric SQL"));
|
||||||
// Still mark as loaded so the UI doesn't block
|
// Still mark as loaded so the UI doesn't block
|
||||||
setLoading(false)
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchNotifications(db: InstanceType<typeof import('@electric-sql/pglite').PGlite>) {
|
async function fetchNotifications(
|
||||||
|
db: InstanceType<typeof import("@electric-sql/pglite").PGlite>
|
||||||
|
) {
|
||||||
try {
|
try {
|
||||||
// Debug: Check all notifications first
|
// Debug: Check all notifications first
|
||||||
const allNotifications = await db.query<Notification>(
|
const allNotifications = await db.query<Notification>(
|
||||||
`SELECT * FROM notifications ORDER BY created_at DESC`
|
`SELECT * FROM notifications ORDER BY created_at DESC`
|
||||||
)
|
);
|
||||||
console.log('All notifications in PGlite:', allNotifications.rows?.length || 0, allNotifications.rows)
|
console.log(
|
||||||
|
"All notifications in PGlite:",
|
||||||
|
allNotifications.rows?.length || 0,
|
||||||
|
allNotifications.rows
|
||||||
|
);
|
||||||
|
|
||||||
// Use PGlite's query method (not exec for SELECT queries)
|
// Use PGlite's query method (not exec for SELECT queries)
|
||||||
const result = await db.query<Notification>(
|
const result = await db.query<Notification>(
|
||||||
`SELECT * FROM notifications
|
`SELECT * FROM notifications
|
||||||
WHERE user_id = $1
|
WHERE user_id = $1
|
||||||
ORDER BY created_at DESC`,
|
ORDER BY created_at DESC`,
|
||||||
[userId]
|
[userId]
|
||||||
)
|
);
|
||||||
console.log(`Notifications for user ${userId}:`, result.rows?.length || 0, result.rows)
|
console.log(`Notifications for user ${userId}:`, result.rows?.length || 0, result.rows);
|
||||||
|
|
||||||
if (mounted) {
|
if (mounted) {
|
||||||
// PGlite query returns { rows: [] } format
|
// PGlite query returns { rows: [] } format
|
||||||
setNotifications(result.rows || [])
|
setNotifications(result.rows || []);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to fetch notifications:', err)
|
console.error("Failed to fetch notifications:", err);
|
||||||
// Log more details for debugging
|
// Log more details for debugging
|
||||||
console.error('Error details:', err)
|
console.error("Error details:", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
init()
|
init();
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
mounted = false
|
mounted = false;
|
||||||
// Reset initialization state so we can reinitialize with a new userId
|
// Reset initialization state so we can reinitialize with a new userId
|
||||||
initializedRef.current = false
|
initializedRef.current = false;
|
||||||
setLoading(true)
|
setLoading(true);
|
||||||
if (syncHandleRef.current) {
|
if (syncHandleRef.current) {
|
||||||
syncHandleRef.current.unsubscribe()
|
syncHandleRef.current.unsubscribe();
|
||||||
syncHandleRef.current = null
|
syncHandleRef.current = null;
|
||||||
}
|
}
|
||||||
if (liveQueryRef.current) {
|
if (liveQueryRef.current) {
|
||||||
liveQueryRef.current.unsubscribe()
|
liveQueryRef.current.unsubscribe();
|
||||||
liveQueryRef.current = null
|
liveQueryRef.current = null;
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
// Only depend on userId - using ref for initialization tracking to prevent cleanup issues
|
// Only depend on userId - using ref for initialization tracking to prevent cleanup issues
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [userId])
|
}, [userId]);
|
||||||
|
|
||||||
// Mark notification as read (local only - needs backend sync)
|
// Mark notification as read (local only - needs backend sync)
|
||||||
const markAsRead = useCallback(
|
const markAsRead = useCallback(
|
||||||
async (notificationId: number) => {
|
async (notificationId: number) => {
|
||||||
if (!electric || !isElectricInitialized()) {
|
if (!electric || !isElectricInitialized()) {
|
||||||
console.warn('Electric SQL not initialized')
|
console.warn("Electric SQL not initialized");
|
||||||
return false
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
@ -213,46 +228,46 @@ export function useNotifications(userId: string | null) {
|
||||||
await electric.db.query(
|
await electric.db.query(
|
||||||
`UPDATE notifications SET read = true, updated_at = NOW() WHERE id = $1`,
|
`UPDATE notifications SET read = true, updated_at = NOW() WHERE id = $1`,
|
||||||
[notificationId]
|
[notificationId]
|
||||||
)
|
);
|
||||||
|
|
||||||
// Update local state
|
// Update local state
|
||||||
setNotifications(prev =>
|
setNotifications((prev) =>
|
||||||
prev.map(n => n.id === notificationId ? { ...n, read: true } : n)
|
prev.map((n) => (n.id === notificationId ? { ...n, read: true } : n))
|
||||||
)
|
);
|
||||||
|
|
||||||
// TODO: Also send to backend to persist the change
|
// TODO: Also send to backend to persist the change
|
||||||
// This could be done via a REST API call
|
// This could be done via a REST API call
|
||||||
|
|
||||||
return true
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to mark notification as read:', err)
|
console.error("Failed to mark notification as read:", err);
|
||||||
return false
|
return false;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[electric]
|
[electric]
|
||||||
)
|
);
|
||||||
|
|
||||||
// Mark all notifications as read
|
// Mark all notifications as read
|
||||||
const markAllAsRead = useCallback(async () => {
|
const markAllAsRead = useCallback(async () => {
|
||||||
if (!electric || !isElectricInitialized()) {
|
if (!electric || !isElectricInitialized()) {
|
||||||
console.warn('Electric SQL not initialized')
|
console.warn("Electric SQL not initialized");
|
||||||
return false
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const unread = notifications.filter(n => !n.read)
|
const unread = notifications.filter((n) => !n.read);
|
||||||
for (const notification of unread) {
|
for (const notification of unread) {
|
||||||
await markAsRead(notification.id)
|
await markAsRead(notification.id);
|
||||||
}
|
}
|
||||||
return true
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to mark all notifications as read:', err)
|
console.error("Failed to mark all notifications as read:", err);
|
||||||
return false
|
return false;
|
||||||
}
|
}
|
||||||
}, [electric, notifications, markAsRead])
|
}, [electric, notifications, markAsRead]);
|
||||||
|
|
||||||
// Get unread count
|
// Get unread count
|
||||||
const unreadCount = notifications.filter(n => !n.read).length
|
const unreadCount = notifications.filter((n) => !n.read).length;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
notifications,
|
notifications,
|
||||||
|
|
@ -261,5 +276,5 @@ export function useNotifications(userId: string | null) {
|
||||||
markAllAsRead,
|
markAllAsRead,
|
||||||
loading,
|
loading,
|
||||||
error,
|
error,
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,17 +5,16 @@
|
||||||
|
|
||||||
export async function getElectricAuthToken(): Promise<string> {
|
export async function getElectricAuthToken(): Promise<string> {
|
||||||
// For insecure mode (development), return empty string
|
// For insecure mode (development), return empty string
|
||||||
if (process.env.NEXT_PUBLIC_ELECTRIC_AUTH_MODE === 'insecure') {
|
if (process.env.NEXT_PUBLIC_ELECTRIC_AUTH_MODE === "insecure") {
|
||||||
return ''
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
// In production, get token from your auth system
|
// In production, get token from your auth system
|
||||||
// This should match your backend auth token
|
// This should match your backend auth token
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== "undefined") {
|
||||||
const token = localStorage.getItem('surfsense_bearer_token')
|
const token = localStorage.getItem("surfsense_bearer_token");
|
||||||
return token || ''
|
return token || "";
|
||||||
}
|
}
|
||||||
|
|
||||||
return ''
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,53 +1,53 @@
|
||||||
/**
|
/**
|
||||||
* Electric SQL client setup for ElectricSQL 1.x with PGlite
|
* Electric SQL client setup for ElectricSQL 1.x with PGlite
|
||||||
*
|
*
|
||||||
* This uses the new ElectricSQL 1.x architecture:
|
* This uses the new ElectricSQL 1.x architecture:
|
||||||
* - PGlite: In-browser PostgreSQL database (local storage)
|
* - PGlite: In-browser PostgreSQL database (local storage)
|
||||||
* - @electric-sql/pglite-sync: Sync plugin to sync Electric shapes into PGlite
|
* - @electric-sql/pglite-sync: Sync plugin to sync Electric shapes into PGlite
|
||||||
* - @electric-sql/client: HTTP client for subscribing to shapes
|
* - @electric-sql/client: HTTP client for subscribing to shapes
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { PGlite } from '@electric-sql/pglite'
|
import { PGlite } from "@electric-sql/pglite";
|
||||||
import { electricSync } from '@electric-sql/pglite-sync'
|
import { electricSync } from "@electric-sql/pglite-sync";
|
||||||
import { live } from '@electric-sql/pglite/live'
|
import { live } from "@electric-sql/pglite/live";
|
||||||
|
|
||||||
// Types
|
// Types
|
||||||
export interface ElectricClient {
|
export interface ElectricClient {
|
||||||
db: PGlite
|
db: PGlite;
|
||||||
syncShape: (options: SyncShapeOptions) => Promise<SyncHandle>
|
syncShape: (options: SyncShapeOptions) => Promise<SyncHandle>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SyncShapeOptions {
|
export interface SyncShapeOptions {
|
||||||
table: string
|
table: string;
|
||||||
where?: string
|
where?: string;
|
||||||
columns?: string[]
|
columns?: string[];
|
||||||
primaryKey?: string[]
|
primaryKey?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SyncHandle {
|
export interface SyncHandle {
|
||||||
unsubscribe: () => void
|
unsubscribe: () => void;
|
||||||
readonly isUpToDate: boolean
|
readonly isUpToDate: boolean;
|
||||||
// The stream property contains the ShapeStreamInterface from pglite-sync
|
// The stream property contains the ShapeStreamInterface from pglite-sync
|
||||||
stream?: unknown
|
stream?: unknown;
|
||||||
// Promise that resolves when initial sync is complete
|
// Promise that resolves when initial sync is complete
|
||||||
initialSyncPromise?: Promise<void>
|
initialSyncPromise?: Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Singleton instance
|
// Singleton instance
|
||||||
let electricClient: ElectricClient | null = null
|
let electricClient: ElectricClient | null = null;
|
||||||
let isInitializing = false
|
let isInitializing = false;
|
||||||
let initPromise: Promise<ElectricClient> | null = null
|
let initPromise: Promise<ElectricClient> | null = null;
|
||||||
|
|
||||||
// Version for sync state - increment this to force fresh sync when Electric config changes
|
// Version for sync state - increment this to force fresh sync when Electric config changes
|
||||||
// Incremented to v4 to fix sync completion issues
|
// Incremented to v4 to fix sync completion issues
|
||||||
const SYNC_VERSION = 4
|
const SYNC_VERSION = 4;
|
||||||
|
|
||||||
// Get Electric URL from environment
|
// Get Electric URL from environment
|
||||||
function getElectricUrl(): string {
|
function getElectricUrl(): string {
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== "undefined") {
|
||||||
return process.env.NEXT_PUBLIC_ELECTRIC_URL || 'http://localhost:5133'
|
return process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133";
|
||||||
}
|
}
|
||||||
return 'http://localhost:5133'
|
return "http://localhost:5133";
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -55,14 +55,14 @@ function getElectricUrl(): string {
|
||||||
*/
|
*/
|
||||||
export async function initElectric(): Promise<ElectricClient> {
|
export async function initElectric(): Promise<ElectricClient> {
|
||||||
if (electricClient) {
|
if (electricClient) {
|
||||||
return electricClient
|
return electricClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isInitializing && initPromise) {
|
if (isInitializing && initPromise) {
|
||||||
return initPromise
|
return initPromise;
|
||||||
}
|
}
|
||||||
|
|
||||||
isInitializing = true
|
isInitializing = true;
|
||||||
initPromise = (async () => {
|
initPromise = (async () => {
|
||||||
try {
|
try {
|
||||||
// Create PGlite instance with Electric sync plugin and live queries
|
// Create PGlite instance with Electric sync plugin and live queries
|
||||||
|
|
@ -75,7 +75,7 @@ export async function initElectric(): Promise<ElectricClient> {
|
||||||
electric: electricSync({ debug: true }),
|
electric: electricSync({ debug: true }),
|
||||||
live, // Enable live queries for real-time updates
|
live, // Enable live queries for real-time updates
|
||||||
},
|
},
|
||||||
})
|
});
|
||||||
|
|
||||||
// Create the notifications table schema in PGlite
|
// Create the notifications table schema in PGlite
|
||||||
// This matches the backend schema
|
// This matches the backend schema
|
||||||
|
|
@ -95,7 +95,7 @@ export async function initElectric(): Promise<ElectricClient> {
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_notifications_user_id ON notifications(user_id);
|
CREATE INDEX IF NOT EXISTS idx_notifications_user_id ON notifications(user_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_notifications_read ON notifications(read);
|
CREATE INDEX IF NOT EXISTS idx_notifications_read ON notifications(read);
|
||||||
`)
|
`);
|
||||||
|
|
||||||
// Create the search_source_connectors table schema in PGlite
|
// Create the search_source_connectors table schema in PGlite
|
||||||
// This matches the backend schema
|
// This matches the backend schema
|
||||||
|
|
@ -118,7 +118,7 @@ export async function initElectric(): Promise<ElectricClient> {
|
||||||
CREATE INDEX IF NOT EXISTS idx_connectors_search_space_id ON search_source_connectors(search_space_id);
|
CREATE INDEX IF NOT EXISTS idx_connectors_search_space_id ON search_source_connectors(search_space_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_connectors_type ON search_source_connectors(connector_type);
|
CREATE INDEX IF NOT EXISTS idx_connectors_type ON search_source_connectors(connector_type);
|
||||||
CREATE INDEX IF NOT EXISTS idx_connectors_user_id ON search_source_connectors(user_id);
|
CREATE INDEX IF NOT EXISTS idx_connectors_user_id ON search_source_connectors(user_id);
|
||||||
`)
|
`);
|
||||||
|
|
||||||
// Create the documents table schema in PGlite
|
// Create the documents table schema in PGlite
|
||||||
// Only sync minimal fields needed for type counts: id, document_type, search_space_id
|
// Only sync minimal fields needed for type counts: id, document_type, search_space_id
|
||||||
|
|
@ -133,292 +133,309 @@ export async function initElectric(): Promise<ElectricClient> {
|
||||||
CREATE INDEX IF NOT EXISTS idx_documents_search_space_id ON documents(search_space_id);
|
CREATE INDEX IF NOT EXISTS idx_documents_search_space_id ON documents(search_space_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_documents_type ON documents(document_type);
|
CREATE INDEX IF NOT EXISTS idx_documents_type ON documents(document_type);
|
||||||
CREATE INDEX IF NOT EXISTS idx_documents_search_space_type ON documents(search_space_id, document_type);
|
CREATE INDEX IF NOT EXISTS idx_documents_search_space_type ON documents(search_space_id, document_type);
|
||||||
`)
|
`);
|
||||||
|
|
||||||
const electricUrl = getElectricUrl()
|
const electricUrl = getElectricUrl();
|
||||||
|
|
||||||
// Create the client wrapper
|
// Create the client wrapper
|
||||||
electricClient = {
|
electricClient = {
|
||||||
db,
|
db,
|
||||||
syncShape: async (options: SyncShapeOptions): Promise<SyncHandle> => {
|
syncShape: async (options: SyncShapeOptions): Promise<SyncHandle> => {
|
||||||
const { table, where, columns, primaryKey = ['id'] } = options
|
const { table, where, columns, primaryKey = ["id"] } = options;
|
||||||
|
|
||||||
// Build params for the shape request
|
// Build params for the shape request
|
||||||
// Electric SQL expects params as URL query parameters
|
// Electric SQL expects params as URL query parameters
|
||||||
const params: Record<string, string> = { table }
|
const params: Record<string, string> = { table };
|
||||||
|
|
||||||
// Validate and fix WHERE clause to ensure string literals are properly quoted
|
// Validate and fix WHERE clause to ensure string literals are properly quoted
|
||||||
let validatedWhere = where
|
let validatedWhere = where;
|
||||||
if (where) {
|
if (where) {
|
||||||
// Check if where uses positional parameters
|
// Check if where uses positional parameters
|
||||||
if (where.includes('$1')) {
|
if (where.includes("$1")) {
|
||||||
// Extract the value from the where clause if it's embedded
|
// Extract the value from the where clause if it's embedded
|
||||||
// For now, we'll use the where clause as-is and let Electric handle it
|
// For now, we'll use the where clause as-is and let Electric handle it
|
||||||
params.where = where
|
params.where = where;
|
||||||
validatedWhere = where
|
validatedWhere = where;
|
||||||
} else {
|
|
||||||
// Validate that string literals are properly quoted
|
|
||||||
// Count single quotes - should be even (pairs) for properly quoted strings
|
|
||||||
const singleQuoteCount = (where.match(/'/g) || []).length
|
|
||||||
|
|
||||||
if (singleQuoteCount % 2 !== 0) {
|
|
||||||
// Odd number of quotes means unterminated string literal
|
|
||||||
console.warn('Where clause has unmatched quotes, fixing:', where)
|
|
||||||
// Add closing quote at the end
|
|
||||||
validatedWhere = `${where}'`
|
|
||||||
params.where = validatedWhere
|
|
||||||
} else {
|
} else {
|
||||||
// Use the where clause directly (already formatted)
|
// Validate that string literals are properly quoted
|
||||||
params.where = where
|
// Count single quotes - should be even (pairs) for properly quoted strings
|
||||||
validatedWhere = where
|
const singleQuoteCount = (where.match(/'/g) || []).length;
|
||||||
|
|
||||||
|
if (singleQuoteCount % 2 !== 0) {
|
||||||
|
// Odd number of quotes means unterminated string literal
|
||||||
|
console.warn("Where clause has unmatched quotes, fixing:", where);
|
||||||
|
// Add closing quote at the end
|
||||||
|
validatedWhere = `${where}'`;
|
||||||
|
params.where = validatedWhere;
|
||||||
|
} else {
|
||||||
|
// Use the where clause directly (already formatted)
|
||||||
|
params.where = where;
|
||||||
|
validatedWhere = where;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (columns) params.columns = columns.join(',')
|
|
||||||
|
|
||||||
console.log('Syncing shape with params:', params)
|
if (columns) params.columns = columns.join(",");
|
||||||
console.log('Electric URL:', `${electricUrl}/v1/shape`)
|
|
||||||
console.log('Where clause:', where, 'Validated:', validatedWhere)
|
console.log("Syncing shape with params:", params);
|
||||||
|
console.log("Electric URL:", `${electricUrl}/v1/shape`);
|
||||||
|
console.log("Where clause:", where, "Validated:", validatedWhere);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Debug: Test Electric SQL connection directly first
|
// Debug: Test Electric SQL connection directly first
|
||||||
// Use validatedWhere to ensure proper URL encoding
|
// Use validatedWhere to ensure proper URL encoding
|
||||||
const testUrl = `${electricUrl}/v1/shape?table=${table}&offset=-1${validatedWhere ? `&where=${encodeURIComponent(validatedWhere)}` : ''}`
|
const testUrl = `${electricUrl}/v1/shape?table=${table}&offset=-1${validatedWhere ? `&where=${encodeURIComponent(validatedWhere)}` : ""}`;
|
||||||
console.log('Testing Electric SQL directly:', testUrl)
|
console.log("Testing Electric SQL directly:", testUrl);
|
||||||
try {
|
try {
|
||||||
const testResponse = await fetch(testUrl)
|
const testResponse = await fetch(testUrl);
|
||||||
const testHeaders = {
|
const testHeaders = {
|
||||||
handle: testResponse.headers.get('electric-handle'),
|
handle: testResponse.headers.get("electric-handle"),
|
||||||
offset: testResponse.headers.get('electric-offset'),
|
offset: testResponse.headers.get("electric-offset"),
|
||||||
upToDate: testResponse.headers.get('electric-up-to-date'),
|
upToDate: testResponse.headers.get("electric-up-to-date"),
|
||||||
}
|
};
|
||||||
console.log('Direct Electric SQL response headers:', testHeaders)
|
console.log("Direct Electric SQL response headers:", testHeaders);
|
||||||
const testData = await testResponse.json()
|
const testData = await testResponse.json();
|
||||||
console.log('Direct Electric SQL data count:', Array.isArray(testData) ? testData.length : 'not array', testData)
|
console.log(
|
||||||
|
"Direct Electric SQL data count:",
|
||||||
|
Array.isArray(testData) ? testData.length : "not array",
|
||||||
|
testData
|
||||||
|
);
|
||||||
} catch (testErr) {
|
} catch (testErr) {
|
||||||
console.error('Direct Electric SQL test failed:', testErr)
|
console.error("Direct Electric SQL test failed:", testErr);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use PGlite's electric sync plugin to sync the shape
|
// Use PGlite's electric sync plugin to sync the shape
|
||||||
// According to Electric SQL docs, the shape config uses params for table, where, columns
|
// According to Electric SQL docs, the shape config uses params for table, where, columns
|
||||||
// Note: mapColumns is OPTIONAL per pglite-sync types.ts
|
// Note: mapColumns is OPTIONAL per pglite-sync types.ts
|
||||||
|
|
||||||
// Create a promise that resolves when initial sync is complete
|
// Create a promise that resolves when initial sync is complete
|
||||||
// Using recommended approach: check isUpToDate immediately, watch stream, shorter timeout
|
// Using recommended approach: check isUpToDate immediately, watch stream, shorter timeout
|
||||||
// IMPORTANT: We don't unsubscribe from the stream - it must stay active for real-time updates
|
// IMPORTANT: We don't unsubscribe from the stream - it must stay active for real-time updates
|
||||||
let resolveInitialSync: () => void
|
let resolveInitialSync: () => void;
|
||||||
let rejectInitialSync: (error: Error) => void
|
let rejectInitialSync: (error: Error) => void;
|
||||||
let syncResolved = false
|
let syncResolved = false;
|
||||||
|
|
||||||
const initialSyncPromise = new Promise<void>((resolve, reject) => {
|
const initialSyncPromise = new Promise<void>((resolve, reject) => {
|
||||||
resolveInitialSync = () => {
|
resolveInitialSync = () => {
|
||||||
if (!syncResolved) {
|
if (!syncResolved) {
|
||||||
syncResolved = true
|
syncResolved = true;
|
||||||
// DON'T unsubscribe from stream - it needs to stay active for real-time updates
|
// DON'T unsubscribe from stream - it needs to stay active for real-time updates
|
||||||
resolve()
|
resolve();
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
rejectInitialSync = (error: Error) => {
|
rejectInitialSync = (error: Error) => {
|
||||||
if (!syncResolved) {
|
if (!syncResolved) {
|
||||||
syncResolved = true
|
syncResolved = true;
|
||||||
// DON'T unsubscribe from stream even on error - let Electric handle it
|
// DON'T unsubscribe from stream even on error - let Electric handle it
|
||||||
reject(error)
|
reject(error);
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
// Shorter timeout (5 seconds) as fallback
|
// Shorter timeout (5 seconds) as fallback
|
||||||
const timeoutId = setTimeout(() => {
|
const timeoutId = setTimeout(() => {
|
||||||
if (!syncResolved) {
|
if (!syncResolved) {
|
||||||
console.warn(`⚠️ Sync timeout for ${table} - checking isUpToDate one more time...`)
|
console.warn(
|
||||||
// Check isUpToDate one more time before resolving
|
`⚠️ Sync timeout for ${table} - checking isUpToDate one more time...`
|
||||||
// This will be checked after shape is created
|
);
|
||||||
setTimeout(() => {
|
// Check isUpToDate one more time before resolving
|
||||||
if (!syncResolved) {
|
// This will be checked after shape is created
|
||||||
console.warn(`⚠️ Sync timeout for ${table} - resolving anyway after 5s`)
|
setTimeout(() => {
|
||||||
resolveInitialSync()
|
if (!syncResolved) {
|
||||||
}
|
console.warn(`⚠️ Sync timeout for ${table} - resolving anyway after 5s`);
|
||||||
}, 100)
|
resolveInitialSync();
|
||||||
}
|
}
|
||||||
}, 5000)
|
}, 100);
|
||||||
|
}
|
||||||
// Store timeout ID for cleanup if needed
|
}, 5000);
|
||||||
// Note: timeout will be cleared if sync completes early
|
|
||||||
})
|
// Store timeout ID for cleanup if needed
|
||||||
|
// Note: timeout will be cleared if sync completes early
|
||||||
const shapeConfig = {
|
});
|
||||||
shape: {
|
|
||||||
url: `${electricUrl}/v1/shape`,
|
const shapeConfig = {
|
||||||
params: {
|
shape: {
|
||||||
table,
|
url: `${electricUrl}/v1/shape`,
|
||||||
...(validatedWhere ? { where: validatedWhere } : {}),
|
params: {
|
||||||
...(columns ? { columns: columns.join(',') } : {}),
|
table,
|
||||||
|
...(validatedWhere ? { where: validatedWhere } : {}),
|
||||||
|
...(columns ? { columns: columns.join(",") } : {}),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
table,
|
||||||
table,
|
primaryKey,
|
||||||
primaryKey,
|
shapeKey: `v${SYNC_VERSION}_${table}_${where?.replace(/[^a-zA-Z0-9]/g, "_") || "all"}`, // Versioned key to force fresh sync when needed
|
||||||
shapeKey: `v${SYNC_VERSION}_${table}_${where?.replace(/[^a-zA-Z0-9]/g, '_') || 'all'}`, // Versioned key to force fresh sync when needed
|
onInitialSync: () => {
|
||||||
onInitialSync: () => {
|
console.log(`✅ Initial sync complete for ${table} - data should now be in PGlite`);
|
||||||
console.log(`✅ Initial sync complete for ${table} - data should now be in PGlite`)
|
resolveInitialSync();
|
||||||
resolveInitialSync()
|
},
|
||||||
},
|
onError: (error: Error) => {
|
||||||
onError: (error: Error) => {
|
console.error(`❌ Shape sync error for ${table}:`, error);
|
||||||
console.error(`❌ Shape sync error for ${table}:`, error)
|
console.error(
|
||||||
console.error('Error details:', JSON.stringify(error, Object.getOwnPropertyNames(error)))
|
"Error details:",
|
||||||
rejectInitialSync(error)
|
JSON.stringify(error, Object.getOwnPropertyNames(error))
|
||||||
},
|
);
|
||||||
}
|
rejectInitialSync(error);
|
||||||
|
},
|
||||||
console.log('syncShapeToTable config:', JSON.stringify(shapeConfig, null, 2))
|
};
|
||||||
|
|
||||||
// Type assertion to PGlite with electric extension
|
|
||||||
const pgWithElectric = db as PGlite & { electric: { syncShapeToTable: (config: typeof shapeConfig) => Promise<{ unsubscribe: () => void; isUpToDate: boolean; stream: unknown }> } }
|
|
||||||
const shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig)
|
|
||||||
|
|
||||||
if (!shape) {
|
console.log("syncShapeToTable config:", JSON.stringify(shapeConfig, null, 2));
|
||||||
throw new Error('syncShapeToTable returned undefined')
|
|
||||||
}
|
|
||||||
|
|
||||||
// Log the actual shape result structure
|
// Type assertion to PGlite with electric extension
|
||||||
console.log('Shape sync result (initial):', {
|
const pgWithElectric = db as PGlite & {
|
||||||
hasUnsubscribe: typeof shape?.unsubscribe === 'function',
|
electric: {
|
||||||
isUpToDate: shape?.isUpToDate,
|
syncShapeToTable: (
|
||||||
hasStream: !!shape?.stream,
|
config: typeof shapeConfig
|
||||||
streamType: typeof shape?.stream,
|
) => Promise<{ unsubscribe: () => void; isUpToDate: boolean; stream: unknown }>;
|
||||||
})
|
};
|
||||||
|
};
|
||||||
// Recommended Approach Step 1: Check isUpToDate immediately
|
const shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig);
|
||||||
if (shape.isUpToDate) {
|
|
||||||
console.log(`✅ Sync already up-to-date for ${table} (resuming from previous state)`)
|
if (!shape) {
|
||||||
resolveInitialSync()
|
throw new Error("syncShapeToTable returned undefined");
|
||||||
} else {
|
}
|
||||||
// Recommended Approach Step 2: Subscribe to stream and watch for "up-to-date" message
|
|
||||||
if (shape?.stream) {
|
// Log the actual shape result structure
|
||||||
const stream = shape.stream as any
|
console.log("Shape sync result (initial):", {
|
||||||
console.log('Shape stream details:', {
|
hasUnsubscribe: typeof shape?.unsubscribe === "function",
|
||||||
shapeHandle: stream?.shapeHandle,
|
isUpToDate: shape?.isUpToDate,
|
||||||
lastOffset: stream?.lastOffset,
|
hasStream: !!shape?.stream,
|
||||||
isUpToDate: stream?.isUpToDate,
|
streamType: typeof shape?.stream,
|
||||||
error: stream?.error,
|
});
|
||||||
hasSubscribe: typeof stream?.subscribe === 'function',
|
|
||||||
hasUnsubscribe: typeof stream?.unsubscribe === 'function',
|
// Recommended Approach Step 1: Check isUpToDate immediately
|
||||||
})
|
if (shape.isUpToDate) {
|
||||||
|
console.log(`✅ Sync already up-to-date for ${table} (resuming from previous state)`);
|
||||||
// Subscribe to the stream to watch for "up-to-date" control message
|
resolveInitialSync();
|
||||||
// NOTE: We keep this subscription active - don't unsubscribe!
|
} else {
|
||||||
// The stream is what Electric SQL uses for real-time updates
|
// Recommended Approach Step 2: Subscribe to stream and watch for "up-to-date" message
|
||||||
if (typeof stream?.subscribe === 'function') {
|
if (shape?.stream) {
|
||||||
console.log('Subscribing to shape stream to watch for up-to-date message...')
|
const stream = shape.stream as any;
|
||||||
// Subscribe but don't store unsubscribe - we want it to stay active
|
console.log("Shape stream details:", {
|
||||||
stream.subscribe((messages: unknown[]) => {
|
shapeHandle: stream?.shapeHandle,
|
||||||
// Continue receiving updates even after sync is resolved
|
lastOffset: stream?.lastOffset,
|
||||||
if (!syncResolved) {
|
isUpToDate: stream?.isUpToDate,
|
||||||
console.log('🔵 Shape stream received messages:', messages?.length || 0)
|
error: stream?.error,
|
||||||
}
|
hasSubscribe: typeof stream?.subscribe === "function",
|
||||||
|
hasUnsubscribe: typeof stream?.unsubscribe === "function",
|
||||||
// Check if any message indicates sync is complete
|
});
|
||||||
if (messages && messages.length > 0) {
|
|
||||||
for (const message of messages) {
|
// Subscribe to the stream to watch for "up-to-date" control message
|
||||||
const msg = message as any
|
// NOTE: We keep this subscription active - don't unsubscribe!
|
||||||
// Check for "up-to-date" control message
|
// The stream is what Electric SQL uses for real-time updates
|
||||||
if (msg?.headers?.control === 'up-to-date' ||
|
if (typeof stream?.subscribe === "function") {
|
||||||
msg?.headers?.electric_up_to_date === 'true' ||
|
console.log("Subscribing to shape stream to watch for up-to-date message...");
|
||||||
(typeof msg === 'object' && 'up-to-date' in msg)) {
|
// Subscribe but don't store unsubscribe - we want it to stay active
|
||||||
if (!syncResolved) {
|
stream.subscribe((messages: unknown[]) => {
|
||||||
console.log(`✅ Received up-to-date message for ${table}`)
|
// Continue receiving updates even after sync is resolved
|
||||||
resolveInitialSync()
|
if (!syncResolved) {
|
||||||
|
console.log("🔵 Shape stream received messages:", messages?.length || 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if any message indicates sync is complete
|
||||||
|
if (messages && messages.length > 0) {
|
||||||
|
for (const message of messages) {
|
||||||
|
const msg = message as any;
|
||||||
|
// Check for "up-to-date" control message
|
||||||
|
if (
|
||||||
|
msg?.headers?.control === "up-to-date" ||
|
||||||
|
msg?.headers?.electric_up_to_date === "true" ||
|
||||||
|
(typeof msg === "object" && "up-to-date" in msg)
|
||||||
|
) {
|
||||||
|
if (!syncResolved) {
|
||||||
|
console.log(`✅ Received up-to-date message for ${table}`);
|
||||||
|
resolveInitialSync();
|
||||||
|
}
|
||||||
|
// Continue listening for real-time updates - don't return!
|
||||||
}
|
}
|
||||||
// Continue listening for real-time updates - don't return!
|
}
|
||||||
|
if (!syncResolved && messages.length > 0) {
|
||||||
|
console.log("First message:", JSON.stringify(messages[0], null, 2));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!syncResolved && messages.length > 0) {
|
|
||||||
console.log('First message:', JSON.stringify(messages[0], null, 2))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also check stream's isUpToDate property after receiving messages
|
|
||||||
if (!syncResolved && stream?.isUpToDate) {
|
|
||||||
console.log(`✅ Stream isUpToDate is true for ${table}`)
|
|
||||||
resolveInitialSync()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Also check stream's isUpToDate property immediately
|
|
||||||
if (stream?.isUpToDate) {
|
|
||||||
console.log(`✅ Stream isUpToDate is true immediately for ${table}`)
|
|
||||||
resolveInitialSync()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also poll isUpToDate periodically as a backup (every 200ms)
|
|
||||||
const pollInterval = setInterval(() => {
|
|
||||||
if (syncResolved) {
|
|
||||||
clearInterval(pollInterval)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (shape.isUpToDate || stream?.isUpToDate) {
|
|
||||||
console.log(`✅ Sync completed (detected via polling) for ${table}`)
|
|
||||||
clearInterval(pollInterval)
|
|
||||||
resolveInitialSync()
|
|
||||||
}
|
|
||||||
}, 200)
|
|
||||||
|
|
||||||
// Clean up polling when promise resolves
|
|
||||||
initialSyncPromise.finally(() => {
|
|
||||||
clearInterval(pollInterval)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
console.warn(`⚠️ No stream available for ${table}, relying on callback and timeout`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return the shape handle - isUpToDate is a getter that reflects current state
|
// Also check stream's isUpToDate property after receiving messages
|
||||||
return {
|
if (!syncResolved && stream?.isUpToDate) {
|
||||||
unsubscribe: () => {
|
console.log(`✅ Stream isUpToDate is true for ${table}`);
|
||||||
console.log('unsubscribing')
|
resolveInitialSync();
|
||||||
if (shape && typeof shape.unsubscribe === 'function') {
|
}
|
||||||
shape.unsubscribe()
|
});
|
||||||
|
|
||||||
|
// Also check stream's isUpToDate property immediately
|
||||||
|
if (stream?.isUpToDate) {
|
||||||
|
console.log(`✅ Stream isUpToDate is true immediately for ${table}`);
|
||||||
|
resolveInitialSync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also poll isUpToDate periodically as a backup (every 200ms)
|
||||||
|
const pollInterval = setInterval(() => {
|
||||||
|
if (syncResolved) {
|
||||||
|
clearInterval(pollInterval);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shape.isUpToDate || stream?.isUpToDate) {
|
||||||
|
console.log(`✅ Sync completed (detected via polling) for ${table}`);
|
||||||
|
clearInterval(pollInterval);
|
||||||
|
resolveInitialSync();
|
||||||
|
}
|
||||||
|
}, 200);
|
||||||
|
|
||||||
|
// Clean up polling when promise resolves
|
||||||
|
initialSyncPromise.finally(() => {
|
||||||
|
clearInterval(pollInterval);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
console.warn(`⚠️ No stream available for ${table}, relying on callback and timeout`);
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
// Use getter to always return current state
|
|
||||||
get isUpToDate() {
|
// Return the shape handle - isUpToDate is a getter that reflects current state
|
||||||
return shape?.isUpToDate ?? false
|
return {
|
||||||
},
|
unsubscribe: () => {
|
||||||
stream: shape?.stream,
|
console.log("unsubscribing");
|
||||||
initialSyncPromise, // Expose promise so callers can wait for sync
|
if (shape && typeof shape.unsubscribe === "function") {
|
||||||
}
|
shape.unsubscribe();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// Use getter to always return current state
|
||||||
|
get isUpToDate() {
|
||||||
|
return shape?.isUpToDate ?? false;
|
||||||
|
},
|
||||||
|
stream: shape?.stream,
|
||||||
|
initialSyncPromise, // Expose promise so callers can wait for sync
|
||||||
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to sync shape:', error)
|
console.error("Failed to sync shape:", error);
|
||||||
// Check if Electric SQL server is reachable
|
// Check if Electric SQL server is reachable
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${electricUrl}/v1/shape?table=${table}&offset=-1`, {
|
const response = await fetch(`${electricUrl}/v1/shape?table=${table}&offset=-1`, {
|
||||||
method: 'GET',
|
method: "GET",
|
||||||
})
|
});
|
||||||
console.log('Electric SQL server response:', response.status, response.statusText)
|
console.log("Electric SQL server response:", response.status, response.statusText);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
console.error('Electric SQL server error:', await response.text())
|
console.error("Electric SQL server error:", await response.text());
|
||||||
}
|
}
|
||||||
} catch (fetchError) {
|
} catch (fetchError) {
|
||||||
console.error('Cannot reach Electric SQL server:', fetchError)
|
console.error("Cannot reach Electric SQL server:", fetchError);
|
||||||
console.error('Make sure Electric SQL is running at:', electricUrl)
|
console.error("Make sure Electric SQL is running at:", electricUrl);
|
||||||
}
|
}
|
||||||
throw error
|
throw error;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
|
|
||||||
console.log('Electric SQL initialized successfully with PGlite')
|
console.log("Electric SQL initialized successfully with PGlite");
|
||||||
return electricClient
|
return electricClient;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to initialize Electric SQL:', error)
|
console.error("Failed to initialize Electric SQL:", error);
|
||||||
throw error
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
isInitializing = false
|
isInitializing = false;
|
||||||
}
|
}
|
||||||
})()
|
})();
|
||||||
|
|
||||||
return initPromise
|
return initPromise;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -426,21 +443,21 @@ export async function initElectric(): Promise<ElectricClient> {
|
||||||
*/
|
*/
|
||||||
export function getElectric(): ElectricClient {
|
export function getElectric(): ElectricClient {
|
||||||
if (!electricClient) {
|
if (!electricClient) {
|
||||||
throw new Error('Electric not initialized. Call initElectric() first.')
|
throw new Error("Electric not initialized. Call initElectric() first.");
|
||||||
}
|
}
|
||||||
return electricClient
|
return electricClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if Electric is initialized
|
* Check if Electric is initialized
|
||||||
*/
|
*/
|
||||||
export function isElectricInitialized(): boolean {
|
export function isElectricInitialized(): boolean {
|
||||||
return electricClient !== null
|
return electricClient !== null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the PGlite database instance
|
* Get the PGlite database instance
|
||||||
*/
|
*/
|
||||||
export function getDb(): PGlite | null {
|
export function getDb(): PGlite | null {
|
||||||
return electricClient?.db ?? null
|
return electricClient?.db ?? null;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue