feat: Add notifications table and integrate Electric SQL for real-time updates

- Introduced a new notifications table in the database schema to manage user notifications.
- Implemented Electric SQL replication setup for the notifications table, ensuring real-time synchronization.
- Updated existing database functions to support real-time updates for connectors and documents using Electric SQL.
- Refactored UI components to utilize new hooks for fetching connectors and documents, enhancing performance and user experience.

surfsense_backend/alembic/versions/61_add_notifications_table.py

@@ -1,7 +1,7 @@
 """Add notifications table
 
-Revision ID: 60
-Revises: 59
+Revision ID: 61
+Revises: 60
 
 Note: Electric SQL replication setup (REPLICA IDENTITY FULL and publication)
 is handled in app/db.py setup_electric_replication() which runs on app startup.
@@ -11,8 +11,8 @@ from collections.abc import Sequence
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision: str = "60"
-down_revision: str | None = "59"
+revision: str = "61"
+down_revision: str | None = "60"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

surfsense_backend/app/db.py

@@ -939,13 +939,15 @@ async def create_db_and_tables():
 
 
 async def setup_electric_replication():
-    """Set up Electric SQL replication for the notifications table."""
+    """Set up Electric SQL replication for real-time sync tables."""
     async with engine.begin() as conn:
         # Set REPLICA IDENTITY FULL (required by Electric SQL for replication)
         # This logs full row data for UPDATE/DELETE operations in the WAL
         await conn.execute(text("ALTER TABLE notifications REPLICA IDENTITY FULL;"))
+        await conn.execute(text("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;"))
+        await conn.execute(text("ALTER TABLE documents REPLICA IDENTITY FULL;"))
 
-        # Add notifications table to Electric SQL publication for replication
+        # Add tables to Electric SQL publication for replication
         # Only add if publication exists and table not already in it
         await conn.execute(
             text(
@@ -953,6 +955,7 @@ async def setup_electric_replication():
                 DO $$
                 BEGIN
                     IF EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
+                        -- Add notifications if not already added
                         IF NOT EXISTS (
                             SELECT 1 FROM pg_publication_tables 
                             WHERE pubname = 'electric_publication_default' 
@@ -960,6 +963,24 @@ async def setup_electric_replication():
                         ) THEN
                             ALTER PUBLICATION electric_publication_default ADD TABLE notifications;
                         END IF;
+                        
+                        -- Add search_source_connectors if not already added
+                        IF NOT EXISTS (
+                            SELECT 1 FROM pg_publication_tables 
+                            WHERE pubname = 'electric_publication_default' 
+                            AND tablename = 'search_source_connectors'
+                        ) THEN
+                            ALTER PUBLICATION electric_publication_default ADD TABLE search_source_connectors;
+                        END IF;
+                        
+                        -- Add documents if not already added
+                        IF NOT EXISTS (
+                            SELECT 1 FROM pg_publication_tables 
+                            WHERE pubname = 'electric_publication_default' 
+                            AND tablename = 'documents'
+                        ) THEN
+                            ALTER PUBLICATION electric_publication_default ADD TABLE documents;
+                        END IF;
                     END IF;
                 END
                 $$;