diff --git a/docker-compose.yml b/docker-compose.yml index 6ffb1c1dd..608b975d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - ./scripts/docker/postgresql.conf:/etc/postgresql/postgresql.conf:ro - - ./scripts/docker/init-electric-user.sql:/docker-entrypoint-initdb.d/init-electric-user.sql:ro environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} diff --git a/scripts/docker/electrify-tables.sql b/scripts/docker/electrify-tables.sql deleted file mode 100644 index 98822ebbb..000000000 --- a/scripts/docker/electrify-tables.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Electrify tables for Electric SQL sync --- This tells Electric SQL which tables to sync --- Run this after running migrations - --- Electrify notifications table -ALTER TABLE notifications ENABLE ELECTRIC; - --- You can electrify other tables as needed: --- ALTER TABLE documents ENABLE ELECTRIC; --- ALTER TABLE logs ENABLE ELECTRIC; - diff --git a/scripts/docker/init-electric-user.sql b/scripts/docker/init-electric-user.sql deleted file mode 100644 index 4317164ad..000000000 --- a/scripts/docker/init-electric-user.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Create Electric SQL replication user --- This script is run during PostgreSQL initialization - -DO $$ -BEGIN - IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'electric') THEN - CREATE USER electric WITH REPLICATION PASSWORD 'electric_password'; - END IF; -END -$$; - --- Grant necessary permissions -GRANT CONNECT ON DATABASE surfsense TO electric; -GRANT USAGE ON SCHEMA public TO electric; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric; -GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric; - --- Grant permissions on future tables -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric; - --- Create the publication that Electric SQL expects --- Electric will add tables to this publication when shapes are subscribed -DO $$ -BEGIN - IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN - CREATE PUBLICATION electric_publication_default; - END IF; -END -$$; diff --git a/surfsense_backend/alembic/versions/62_add_notifications_table.py b/surfsense_backend/alembic/versions/62_add_notifications_table.py index cf3407415..36dbd7382 100644 --- a/surfsense_backend/alembic/versions/62_add_notifications_table.py +++ b/surfsense_backend/alembic/versions/62_add_notifications_table.py @@ -4,7 +4,7 @@ Revision ID: 62 Revises: 61 Creates notifications table and sets up Electric SQL replication -(REPLICA IDENTITY FULL and publication) for notifications, +(user, publication, REPLICA IDENTITY FULL) for notifications, search_source_connectors, and documents tables. """ @@ -45,45 +45,93 @@ def upgrade() -> None: op.create_index("ix_notifications_created_at", "notifications", ["created_at"]) op.create_index("ix_notifications_user_read", "notifications", ["user_id", "read"]) - # Set up Electric SQL replication for real-time sync tables - # Set REPLICA IDENTITY FULL (required by Electric SQL for replication) - # This logs full row data for UPDATE/DELETE operations in the WAL - op.execute("ALTER TABLE notifications REPLICA IDENTITY FULL;") - op.execute("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;") - op.execute("ALTER TABLE documents REPLICA IDENTITY FULL;") - - # Add tables to Electric SQL publication for replication if publication exists + # ===================================================== + # Electric SQL Setup - User and Publication + # ===================================================== + + # Create Electric SQL replication user if not exists op.execute( """ DO $$ BEGIN - IF EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'electric_publication_default') THEN - -- Add notifications if not already added - IF NOT EXISTS ( - SELECT 1 FROM pg_publication_tables - WHERE pubname = 'electric_publication_default' - AND tablename = 'notifications' - ) THEN - ALTER PUBLICATION electric_publication_default ADD TABLE notifications; - END IF; - - -- Add search_source_connectors if not already added - IF NOT EXISTS ( - SELECT 1 FROM pg_publication_tables - WHERE pubname = 'electric_publication_default' - AND tablename = 'search_source_connectors' - ) THEN - ALTER PUBLICATION electric_publication_default ADD TABLE search_source_connectors; - END IF; - - -- Add documents if not already added - IF NOT EXISTS ( - SELECT 1 FROM pg_publication_tables - WHERE pubname = 'electric_publication_default' - AND tablename = 'documents' - ) THEN - ALTER PUBLICATION electric_publication_default ADD TABLE documents; - END IF; + IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'electric') THEN + CREATE USER electric WITH REPLICATION PASSWORD 'electric_password'; + END IF; + END + $$; + """ + ) + + # Grant necessary permissions to electric user + op.execute( + """ + DO $$ + DECLARE + db_name TEXT := current_database(); + BEGIN + EXECUTE format('GRANT CONNECT ON DATABASE %I TO electric', db_name); + END + $$; + """ + ) + op.execute("GRANT USAGE ON SCHEMA public TO electric;") + op.execute("GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric;") + op.execute("GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric;") + op.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric;") + op.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric;") + + # Create the publication if not exists + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN + CREATE PUBLICATION electric_publication_default; + END IF; + END + $$; + """ + ) + + # ===================================================== + # Electric SQL Setup - Table Configuration + # ===================================================== + + # Set REPLICA IDENTITY FULL (required by Electric SQL for replication) + op.execute("ALTER TABLE notifications REPLICA IDENTITY FULL;") + op.execute("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;") + op.execute("ALTER TABLE documents REPLICA IDENTITY FULL;") + + # Add tables to Electric SQL publication for replication + op.execute( + """ + DO $$ + BEGIN + -- Add notifications if not already added + IF NOT EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'notifications' + ) THEN + ALTER PUBLICATION electric_publication_default ADD TABLE notifications; + END IF; + + -- Add search_source_connectors if not already added + IF NOT EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'search_source_connectors' + ) THEN + ALTER PUBLICATION electric_publication_default ADD TABLE search_source_connectors; + END IF; + + -- Add documents if not already added + IF NOT EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'documents' + ) THEN + ALTER PUBLICATION electric_publication_default ADD TABLE documents; END IF; END $$;