diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 7119fcb6d..491df0992 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -57,7 +57,7 @@ jobs: working-directory: surfsense_web env: NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_URL }} - NEXT_PUBLIC_ELECTRIC_URL: ${{ vars.NEXT_PUBLIC_ELECTRIC_URL }} + NEXT_PUBLIC_ZERO_CACHE_URL: ${{ vars.NEXT_PUBLIC_ZERO_CACHE_URL }} NEXT_PUBLIC_DEPLOYMENT_MODE: ${{ vars.NEXT_PUBLIC_DEPLOYMENT_MODE }} NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE }} diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a53a4b414..2e5de8cc6 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -164,8 +164,7 @@ jobs: ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }} ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }} ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }} - ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__' || '' }} - ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__' || '' }} + ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ZERO_CACHE_URL=__NEXT_PUBLIC_ZERO_CACHE_URL__' || '' }} ${{ matrix.image == 'web' && 'NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__' || '' }} - name: Export digest diff --git a/docker/.env.example b/docker/.env.example index a226c2624..766f92dcc 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -35,7 +35,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # BACKEND_PORT=8929 # FRONTEND_PORT=3929 -# ELECTRIC_PORT=5929 +# ZERO_CACHE_PORT=5929 # SEARXNG_PORT=8888 # FLOWER_PORT=5555 @@ -58,7 +58,6 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL # NEXT_PUBLIC_ETL_SERVICE=DOCLING # NEXT_PUBLIC_DEPLOYMENT_MODE=self-hosted -# NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure # ------------------------------------------------------------------------------ # Custom Domain / Reverse Proxy @@ -71,8 +70,35 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # NEXT_FRONTEND_URL=https://app.yourdomain.com # BACKEND_URL=https://api.yourdomain.com # NEXT_PUBLIC_FASTAPI_BACKEND_URL=https://api.yourdomain.com -# NEXT_PUBLIC_ELECTRIC_URL=https://electric.yourdomain.com +# NEXT_PUBLIC_ZERO_CACHE_URL=https://zero.yourdomain.com +# ------------------------------------------------------------------------------ +# Zero-cache (real-time sync) +# ------------------------------------------------------------------------------ +# Defaults work out of the box for Docker deployments. +# Change ZERO_ADMIN_PASSWORD for security in production. + +# ZERO_ADMIN_PASSWORD=surfsense-zero-admin +# Full override for the Zero → Postgres connection URLs. +# Leave commented out to use the Docker-managed `db` container (default). +# ZERO_UPSTREAM_DB=postgresql://surfsense:surfsense@db:5432/surfsense +# ZERO_CVR_DB=postgresql://surfsense:surfsense@db:5432/surfsense +# ZERO_CHANGE_DB=postgresql://surfsense:surfsense@db:5432/surfsense + +# ZERO_QUERY_URL: where zero-cache forwards query requests for resolution. +# ZERO_MUTATE_URL: required by zero-cache when auth tokens are used, even though +# SurfSense does not use Zero mutators. Setting both URLs tells zero-cache to +# skip its own JWT verification and let the app endpoints handle auth instead. +# The mutate endpoint is a no-op that returns an empty response. +# Default: Docker service networking (http://frontend:3000/api/zero/...). +# Override when running the frontend outside Docker: +# ZERO_QUERY_URL=http://host.docker.internal:3000/api/zero/query +# ZERO_MUTATE_URL=http://host.docker.internal:3000/api/zero/mutate +# Override for custom domain: +# ZERO_QUERY_URL=https://app.yourdomain.com/api/zero/query +# ZERO_MUTATE_URL=https://app.yourdomain.com/api/zero/mutate +# ZERO_QUERY_URL=http://frontend:3000/api/zero/query +# ZERO_MUTATE_URL=http://frontend:3000/api/zero/mutate # ------------------------------------------------------------------------------ # Database (defaults work out of the box, change for security) @@ -101,19 +127,6 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # Supports TLS: rediss://:password@host:6380/0 # REDIS_URL=redis://redis:6379/0 -# ------------------------------------------------------------------------------ -# Electric SQL (real-time sync credentials) -# ------------------------------------------------------------------------------ -# These must match on the db, backend, and electric services. -# Change for security; defaults work out of the box. - -# ELECTRIC_DB_USER=electric -# ELECTRIC_DB_PASSWORD=electric_password -# Full override for the Electric → Postgres connection URL. -# Leave commented out to use the Docker-managed `db` container (default). -# Uncomment and set `db` to `host.docker.internal` when pointing Electric at a local Postgres instance (e.g. Postgres.app on macOS): -# ELECTRIC_DATABASE_URL=postgresql://electric:electric_password@db:5432/surfsense?sslmode=disable - # ------------------------------------------------------------------------------ # TTS & STT (Text-to-Speech / Speech-to-Text) # ------------------------------------------------------------------------------ diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 15531bf55..564ecd772 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -18,13 +18,10 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro - - ./scripts/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro environment: - POSTGRES_USER=${DB_USER:-postgres} - POSTGRES_PASSWORD=${DB_PASSWORD:-postgres} - POSTGRES_DB=${DB_NAME:-surfsense} - - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} - - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} command: postgres -c config_file=/etc/postgresql/postgresql.conf healthcheck: test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-surfsense}"] @@ -91,8 +88,6 @@ services: - UNSTRUCTURED_HAS_PATCHED_LOOP=1 - LANGCHAIN_TRACING_V2=false - LANGSMITH_TRACING=false - - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} - - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} - AUTH_TYPE=${AUTH_TYPE:-LOCAL} - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000} - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080} @@ -130,8 +125,6 @@ services: - REDIS_APP_URL=${REDIS_URL:-redis://redis:6379/0} - CELERY_TASK_DEFAULT_QUEUE=surfsense - PYTHONPATH=/app - - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} - - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080} - SERVICE_ROLE=worker depends_on: @@ -176,20 +169,28 @@ services: # - redis # - celery_worker - electric: - image: electricsql/electric:1.4.10 + zero-cache: + image: rocicorp/zero:0.26.2 ports: - - "${ELECTRIC_PORT:-5133}:3000" + - "${ZERO_CACHE_PORT:-4848}:4848" + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: db: condition: service_healthy environment: - - DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} - - ELECTRIC_INSECURE=true - - ELECTRIC_WRITE_TO_PG_MODE=direct + - ZERO_UPSTREAM_DB=${ZERO_UPSTREAM_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} + - ZERO_CVR_DB=${ZERO_CVR_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} + - ZERO_CHANGE_DB=${ZERO_CHANGE_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} + - ZERO_REPLICA_FILE=/data/zero.db + - ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} + - ZERO_QUERY_URL=${ZERO_QUERY_URL:-http://frontend:3000/api/zero/query} + - ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} + volumes: + - zero_cache_data:/data restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"] + test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"] interval: 10s timeout: 5s retries: 5 @@ -201,8 +202,7 @@ services: NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000} NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL} NEXT_PUBLIC_ETL_SERVICE: ${NEXT_PUBLIC_ETL_SERVICE:-DOCLING} - NEXT_PUBLIC_ELECTRIC_URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133} - NEXT_PUBLIC_ELECTRIC_AUTH_MODE: ${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure} + NEXT_PUBLIC_ZERO_CACHE_URL: ${NEXT_PUBLIC_ZERO_CACHE_URL:-http://localhost:${ZERO_CACHE_PORT:-4848}} NEXT_PUBLIC_DEPLOYMENT_MODE: ${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted} ports: - "${FRONTEND_PORT:-3000}:3000" @@ -211,7 +211,7 @@ services: depends_on: backend: condition: service_healthy - electric: + zero-cache: condition: service_healthy volumes: @@ -223,3 +223,5 @@ volumes: name: surfsense-dev-redis shared_temp: name: surfsense-dev-shared-temp + zero_cache_data: + name: surfsense-dev-zero-cache diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8c85248d2..b03efdd2f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,13 +15,10 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro - - ./scripts/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro environment: POSTGRES_USER: ${DB_USER:-surfsense} POSTGRES_PASSWORD: ${DB_PASSWORD:-surfsense} POSTGRES_DB: ${DB_NAME:-surfsense} - ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric} - ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password} command: postgres -c config_file=/etc/postgresql/postgresql.conf restart: unless-stopped healthcheck: @@ -72,8 +69,6 @@ services: PYTHONPATH: /app UVICORN_LOOP: asyncio UNSTRUCTURED_HAS_PATCHED_LOOP: "1" - ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric} - ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password} NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-http://localhost:${FRONTEND_PORT:-3929}} SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution @@ -112,8 +107,6 @@ services: REDIS_APP_URL: ${REDIS_URL:-redis://redis:6379/0} CELERY_TASK_DEFAULT_QUEUE: surfsense PYTHONPATH: /app - ELECTRIC_DB_USER: ${ELECTRIC_DB_USER:-electric} - ELECTRIC_DB_PASSWORD: ${ELECTRIC_DB_PASSWORD:-electric_password} SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} SERVICE_ROLE: worker depends_on: @@ -165,20 +158,28 @@ services: # - celery_worker # restart: unless-stopped - electric: - image: electricsql/electric:1.4.10 + zero-cache: + image: rocicorp/zero:0.26.2 ports: - - "${ELECTRIC_PORT:-5929}:3000" + - "${ZERO_CACHE_PORT:-5929}:4848" + extra_hosts: + - "host.docker.internal:host-gateway" environment: - DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} - ELECTRIC_INSECURE: "true" - ELECTRIC_WRITE_TO_PG_MODE: direct + ZERO_UPSTREAM_DB: ${ZERO_UPSTREAM_DB:-postgresql://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} + ZERO_CVR_DB: ${ZERO_CVR_DB:-postgresql://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} + ZERO_CHANGE_DB: ${ZERO_CHANGE_DB:-postgresql://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} + ZERO_REPLICA_FILE: /data/zero.db + ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} + ZERO_QUERY_URL: ${ZERO_QUERY_URL:-http://frontend:3000/api/zero/query} + ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} + volumes: + - zero_cache_data:/data restart: unless-stopped depends_on: db: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"] + test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"] interval: 10s timeout: 5s retries: 5 @@ -189,17 +190,16 @@ services: - "${FRONTEND_PORT:-3929}:3000" environment: NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:${BACKEND_PORT:-8929}} - NEXT_PUBLIC_ELECTRIC_URL: ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:${ELECTRIC_PORT:-5929}} + NEXT_PUBLIC_ZERO_CACHE_URL: ${NEXT_PUBLIC_ZERO_CACHE_URL:-http://localhost:${ZERO_CACHE_PORT:-5929}} NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${AUTH_TYPE:-LOCAL} NEXT_PUBLIC_ETL_SERVICE: ${ETL_SERVICE:-DOCLING} NEXT_PUBLIC_DEPLOYMENT_MODE: ${DEPLOYMENT_MODE:-self-hosted} - NEXT_PUBLIC_ELECTRIC_AUTH_MODE: ${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure} labels: - "com.centurylinklabs.watchtower.enable=true" depends_on: backend: condition: service_healthy - electric: + zero-cache: condition: service_healthy restart: unless-stopped @@ -210,3 +210,5 @@ volumes: name: surfsense-redis shared_temp: name: surfsense-shared-temp + zero_cache_data: + name: surfsense-zero-cache diff --git a/docker/postgresql.conf b/docker/postgresql.conf index 99b29ba30..d0936dce8 100644 --- a/docker/postgresql.conf +++ b/docker/postgresql.conf @@ -1,11 +1,11 @@ -# PostgreSQL configuration for Electric SQL +# PostgreSQL configuration for SurfSense # This file is mounted into the PostgreSQL container listen_addresses = '*' max_connections = 200 shared_buffers = 256MB -# Enable logical replication (required for Electric SQL) +# Enable logical replication (required for Zero-cache real-time sync) wal_level = logical max_replication_slots = 10 max_wal_senders = 10 diff --git a/docker/scripts/init-electric-user.sh b/docker/scripts/init-electric-user.sh deleted file mode 100755 index fbd1c361a..000000000 --- a/docker/scripts/init-electric-user.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# Creates the Electric SQL replication user on first DB initialization. -# Idempotent — safe to run alongside Alembic migration 66. - -set -e - -ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}" -ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}" - -echo "Creating Electric SQL replication user: $ELECTRIC_DB_USER" - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '$ELECTRIC_DB_USER') THEN - CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD'; - END IF; - END - \$\$; - - GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER; - GRANT CREATE ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER; - GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER; - - DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN - CREATE PUBLICATION electric_publication_default; - END IF; - END - \$\$; -EOSQL - -echo "Electric SQL user '$ELECTRIC_DB_USER' and publication created successfully" diff --git a/docker/scripts/install.ps1 b/docker/scripts/install.ps1 index b7004bae2..0eb3886a2 100644 --- a/docker/scripts/install.ps1 +++ b/docker/scripts/install.ps1 @@ -109,7 +109,6 @@ $Files = @( @{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" } @{ Src = "docker/.env.example"; Dest = ".env.example" } @{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" } - @{ Src = "docker/scripts/init-electric-user.sh"; Dest = "scripts/init-electric-user.sh" } @{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" } @{ Src = "docker/searxng/settings.yml"; Dest = "searxng/settings.yml" } @{ Src = "docker/searxng/limiter.toml"; Dest = "searxng/limiter.toml" } diff --git a/docker/scripts/install.sh b/docker/scripts/install.sh index 7a68a9bd1..fcab4d55a 100644 --- a/docker/scripts/install.sh +++ b/docker/scripts/install.sh @@ -108,7 +108,6 @@ FILES=( "docker/docker-compose.yml:docker-compose.yml" "docker/.env.example:.env.example" "docker/postgresql.conf:postgresql.conf" - "docker/scripts/init-electric-user.sh:scripts/init-electric-user.sh" "docker/scripts/migrate-database.sh:scripts/migrate-database.sh" "docker/searxng/settings.yml:searxng/settings.yml" "docker/searxng/limiter.toml:searxng/limiter.toml" @@ -122,7 +121,6 @@ for entry in "${FILES[@]}"; do || error "Failed to download ${dest}. Check your internet connection and try again." done -chmod +x "${INSTALL_DIR}/scripts/init-electric-user.sh" chmod +x "${INSTALL_DIR}/scripts/migrate-database.sh" success "All files downloaded to ${INSTALL_DIR}/" diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index d3717778b..94d5c8c9b 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -17,10 +17,6 @@ REDIS_APP_URL=redis://localhost:6379/0 # Only uncomment if running the backend outside Docker (e.g. uvicorn on host). # SEARXNG_DEFAULT_HOST=http://localhost:8888 -#Electric(for migrations only) -ELECTRIC_DB_USER=electric -ELECTRIC_DB_PASSWORD=electric_password - # Periodic task interval # # Run every minute (default) # SCHEDULE_CHECKER_INTERVAL=1m diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py index fa213121c..bd8c20356 100644 --- a/surfsense_backend/alembic/env.py +++ b/surfsense_backend/alembic/env.py @@ -25,13 +25,6 @@ database_url = os.getenv("DATABASE_URL") if database_url: config.set_main_option("sqlalchemy.url", database_url) -# Electric SQL user credentials - centralized configuration for migrations -# These are used by migrations that set up Electric SQL replication -config.set_main_option("electric_db_user", os.getenv("ELECTRIC_DB_USER", "electric")) -config.set_main_option( - "electric_db_password", os.getenv("ELECTRIC_DB_PASSWORD", "electric_password") -) - # Interpret the config file for Python logging. # This line sets up loggers basically. if config.config_file_name is not None: diff --git a/surfsense_backend/alembic/versions/104_add_notification_composite_indexes.py b/surfsense_backend/alembic/versions/104_add_notification_composite_indexes.py index 69e97eb0d..c3afb58d0 100644 --- a/surfsense_backend/alembic/versions/104_add_notification_composite_indexes.py +++ b/surfsense_backend/alembic/versions/104_add_notification_composite_indexes.py @@ -30,21 +30,25 @@ def upgrade() -> None: "ix_notifications_user_read_type_created", "notifications", ["user_id", "read", "type", "created_at"], + if_not_exists=True, ) op.create_index( "ix_notifications_user_space_created", "notifications", ["user_id", "search_space_id", "created_at"], + if_not_exists=True, ) op.create_index( "ix_notifications_type", "notifications", ["type"], + if_not_exists=True, ) op.create_index( "ix_notifications_search_space_id", "notifications", ["search_space_id"], + if_not_exists=True, ) diff --git a/surfsense_backend/alembic/versions/107_add_video_presentations_table.py b/surfsense_backend/alembic/versions/107_add_video_presentations_table.py index 76cd42a23..1dbfb63de 100644 --- a/surfsense_backend/alembic/versions/107_add_video_presentations_table.py +++ b/surfsense_backend/alembic/versions/107_add_video_presentations_table.py @@ -35,52 +35,60 @@ def upgrade() -> None: END $$; """) - op.create_table( - "video_presentations", - sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), - sa.Column("title", sa.String(length=500), nullable=False), - sa.Column("slides", JSONB(), nullable=True), - sa.Column("scene_codes", JSONB(), nullable=True), - sa.Column( - "status", - video_presentation_status_enum, - server_default="ready", - nullable=False, - ), - sa.Column("search_space_id", sa.Integer(), nullable=False), - sa.Column("thread_id", sa.Integer(), nullable=True), - sa.Column( - "created_at", - sa.TIMESTAMP(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["search_space_id"], - ["searchspaces.id"], - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["thread_id"], - ["new_chat_threads.id"], - ondelete="SET NULL", - ), - sa.PrimaryKeyConstraint("id"), + conn = op.get_bind() + result = conn.execute( + sa.text("SELECT 1 FROM information_schema.tables WHERE table_name = 'video_presentations'") ) + if not result.fetchone(): + op.create_table( + "video_presentations", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("title", sa.String(length=500), nullable=False), + sa.Column("slides", JSONB(), nullable=True), + sa.Column("scene_codes", JSONB(), nullable=True), + sa.Column( + "status", + video_presentation_status_enum, + server_default="ready", + nullable=False, + ), + sa.Column("search_space_id", sa.Integer(), nullable=False), + sa.Column("thread_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["search_space_id"], + ["searchspaces.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["thread_id"], + ["new_chat_threads.id"], + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id"), + ) op.create_index( "ix_video_presentations_status", "video_presentations", ["status"], + if_not_exists=True, ) op.create_index( "ix_video_presentations_thread_id", "video_presentations", ["thread_id"], + if_not_exists=True, ) op.create_index( "ix_video_presentations_created_at", "video_presentations", ["created_at"], + if_not_exists=True, ) diff --git a/surfsense_backend/alembic/versions/108_cleanup_electric_sql_artifacts.py b/surfsense_backend/alembic/versions/108_cleanup_electric_sql_artifacts.py new file mode 100644 index 000000000..0f60a8bca --- /dev/null +++ b/surfsense_backend/alembic/versions/108_cleanup_electric_sql_artifacts.py @@ -0,0 +1,104 @@ +"""Clean up Electric SQL artifacts (user, publication, replication slots) + +Revision ID: 108 +Revises: 107 + +Removes leftover Electric SQL infrastructure that is no longer needed after +the migration to Rocicorp Zero. Fully idempotent — safe on databases that +never had Electric SQL set up (fresh installs). + +Cleaned up: +- Replication slots containing 'electric' (prevents unbounded WAL growth) +- The 'electric_publication_default' publication +- Default privileges, grants, and the 'electric' database user +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "108" +down_revision: str | None = "107" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + DO $$ + DECLARE + slot RECORD; + BEGIN + -- 1. Drop inactive Electric replication slots (prevents WAL growth) + FOR slot IN + SELECT slot_name FROM pg_replication_slots + WHERE slot_name LIKE '%electric%' AND active = false + LOOP + BEGIN + PERFORM pg_drop_replication_slot(slot.slot_name); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Could not drop replication slot %: %', slot.slot_name, SQLERRM; + END; + END LOOP; + + -- Warn about active Electric slots that cannot be safely dropped + FOR slot IN + SELECT slot_name FROM pg_replication_slots + WHERE slot_name LIKE '%electric%' AND active = true + LOOP + RAISE WARNING 'Active Electric replication slot "%" was not dropped — drop it manually to stop WAL growth', slot.slot_name; + END LOOP; + + -- 2. Drop the Electric publication + BEGIN + IF EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'electric_publication_default') THEN + DROP PUBLICATION electric_publication_default; + END IF; + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Could not drop publication electric_publication_default: %', SQLERRM; + END; + + -- 3. Revoke privileges and drop the Electric user + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'electric') THEN + BEGIN + ALTER DEFAULT PRIVILEGES IN SCHEMA public + REVOKE SELECT ON TABLES FROM electric; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + REVOKE SELECT ON SEQUENCES FROM electric; + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Could not revoke default privileges from electric: %', SQLERRM; + END; + + BEGIN + REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM electric; + REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM electric; + REVOKE USAGE ON SCHEMA public FROM electric; + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Could not revoke schema privileges from electric: %', SQLERRM; + END; + + BEGIN + EXECUTE format( + 'REVOKE CONNECT ON DATABASE %I FROM electric', + current_database() + ); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Could not revoke CONNECT from electric: %', SQLERRM; + END; + + BEGIN + REASSIGN OWNED BY electric TO CURRENT_USER; + DROP ROLE electric; + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Could not drop role electric: %', SQLERRM; + END; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + pass diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index ff8bf442d..132bd8dae 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -722,7 +722,7 @@ class ChatComment(BaseModel, TimestampMixin): nullable=False, index=True, ) - # Denormalized thread_id for efficient Electric SQL subscriptions (one per thread) + # Denormalized thread_id for efficient Zero subscriptions (one per thread) thread_id = Column( Integer, ForeignKey("new_chat_threads.id", ondelete="CASCADE"), @@ -792,7 +792,7 @@ class ChatCommentMention(BaseModel, TimestampMixin): class ChatSessionState(BaseModel): """ Tracks real-time session state for shared chat collaboration. - One record per thread, synced via Electric SQL. + One record per thread, synced via Zero. """ __tablename__ = "chat_session_state" diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 66471b0ed..f6975b69d 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -80,7 +80,7 @@ router.include_router(model_list_router) # Dynamic LLM model catalogue from Ope router.include_router(logs_router) router.include_router(circleback_webhook_router) # Circleback meeting webhooks router.include_router(surfsense_docs_router) # Surfsense documentation for citations -router.include_router(notifications_router) # Notifications with Electric SQL sync +router.include_router(notifications_router) # Notifications with Zero sync router.include_router(composio_router) # Composio OAuth and toolkit management router.include_router(public_chat_router) # Public chat sharing and cloning router.include_router(incentive_tasks_router) # Incentive tasks for earning free pages diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 2dfe1b530..503f2cf32 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -128,7 +128,7 @@ async def create_documents_file_upload( Upload files as documents with real-time status tracking. Implements 2-phase document status updates for real-time UI feedback: - - Phase 1: Create all documents with 'pending' status (visible in UI immediately via ElectricSQL) + - Phase 1: Create all documents with 'pending' status (visible in UI immediately via Zero) - Phase 2: Celery processes each file: pending → processing → ready/failed Requires DOCUMENTS_CREATE permission. diff --git a/surfsense_backend/app/routes/notifications_routes.py b/surfsense_backend/app/routes/notifications_routes.py index 82c267c9d..611227795 100644 --- a/surfsense_backend/app/routes/notifications_routes.py +++ b/surfsense_backend/app/routes/notifications_routes.py @@ -1,7 +1,7 @@ """ Notifications API routes. These endpoints allow marking notifications as read and fetching older notifications. -Electric SQL automatically syncs the changes to all connected clients for recent items. +Zero automatically syncs the changes to all connected clients for recent items. For older items (beyond the sync window), use the list endpoint. """ @@ -267,7 +267,7 @@ async def get_unread_count( This allows the frontend to calculate: - older_unread = total_unread - recent_unread (static until reconciliation) - - Display count = older_unread + live_recent_count (from Electric SQL) + - Display count = older_unread + live_recent_count (from Zero) """ # Calculate cutoff date for sync window cutoff_date = datetime.now(UTC) - timedelta(days=SYNC_WINDOW_DAYS) @@ -344,7 +344,7 @@ async def list_notifications( List notifications for the current user with pagination. This endpoint is used as a fallback for older notifications that are - outside the Electric SQL sync window (2 weeks). + outside the Zero sync window (2 weeks). Use `before_date` to paginate through older notifications efficiently. """ @@ -487,7 +487,7 @@ async def mark_notification_as_read( """ Mark a single notification as read. - Electric SQL will automatically sync this change to all connected clients. + Zero will automatically sync this change to all connected clients. """ # Verify the notification belongs to the user result = await session.execute( @@ -528,7 +528,7 @@ async def mark_all_notifications_as_read( """ Mark all notifications as read for the current user. - Electric SQL will automatically sync these changes to all connected clients. + Zero will automatically sync these changes to all connected clients. """ # Update all unread notifications for the user result = await session.execute( diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index c12621f60..1ffc6341f 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -1543,7 +1543,7 @@ async def _run_indexing_with_notifications( ) await ( session.commit() - ) # Commit to ensure Electric SQL syncs the notification update + ) # Commit to ensure Zero syncs the notification update elif documents_processed > 0: # Update notification to storing stage if notification: @@ -1570,7 +1570,7 @@ async def _run_indexing_with_notifications( ) await ( session.commit() - ) # Commit to ensure Electric SQL syncs the notification update + ) # Commit to ensure Zero syncs the notification update else: # No new documents processed - check if this is an error or just no changes if error_or_warning: @@ -1596,7 +1596,7 @@ async def _run_indexing_with_notifications( if is_duplicate_warning or is_empty_result or is_info_warning: # These are success cases - sync worked, just found nothing new logger.info(f"Indexing completed successfully: {error_or_warning}") - # Still update timestamp so ElectricSQL syncs and clears "Syncing" UI + # Still update timestamp so Zero syncs and clears "Syncing" UI if update_timestamp_func: await update_timestamp_func(session, connector_id) await session.commit() # Commit timestamp update @@ -1619,7 +1619,7 @@ async def _run_indexing_with_notifications( ) await ( session.commit() - ) # Commit to ensure Electric SQL syncs the notification update + ) # Commit to ensure Zero syncs the notification update else: # Actual failure logger.error(f"Indexing failed: {error_or_warning}") @@ -1637,13 +1637,13 @@ async def _run_indexing_with_notifications( ) await ( session.commit() - ) # Commit to ensure Electric SQL syncs the notification update + ) # Commit to ensure Zero syncs the notification update else: # Success - just no new documents to index (all skipped/unchanged) logger.info( "Indexing completed: No new documents to process (all up to date)" ) - # Still update timestamp so ElectricSQL syncs and clears "Syncing" UI + # Still update timestamp so Zero syncs and clears "Syncing" UI if update_timestamp_func: await update_timestamp_func(session, connector_id) await session.commit() # Commit timestamp update @@ -1659,7 +1659,7 @@ async def _run_indexing_with_notifications( ) await ( session.commit() - ) # Commit to ensure Electric SQL syncs the notification update + ) # Commit to ensure Zero syncs the notification update except SoftTimeLimitExceeded: # Celery soft time limit was reached - task is about to be killed # Gracefully save progress and mark as interrupted @@ -2776,7 +2776,7 @@ async def run_composio_indexing( Run Composio connector indexing with real-time notifications. This wraps the Composio indexer with the notification system so that - Electric SQL can sync indexing progress to the frontend in real-time. + Zero can sync indexing progress to the frontend in real-time. Args: session: Database session diff --git a/surfsense_backend/app/services/chat_comments_service.py b/surfsense_backend/app/services/chat_comments_service.py index c2bb65aee..54662fe5b 100644 --- a/surfsense_backend/app/services/chat_comments_service.py +++ b/surfsense_backend/app/services/chat_comments_service.py @@ -456,7 +456,7 @@ async def create_comment( thread = message.thread comment = ChatComment( message_id=message_id, - thread_id=thread.id, # Denormalized for efficient Electric subscriptions + thread_id=thread.id, # Denormalized for efficient per-thread sync author_id=user.id, content=content, ) @@ -569,7 +569,7 @@ async def create_reply( thread = parent_comment.message.thread reply = ChatComment( message_id=parent_comment.message_id, - thread_id=thread.id, # Denormalized for efficient Electric subscriptions + thread_id=thread.id, # Denormalized for efficient per-thread sync parent_id=comment_id, author_id=user.id, content=content, diff --git a/surfsense_backend/app/services/notification_service.py b/surfsense_backend/app/services/notification_service.py index a759f3536..5e40a3b42 100644 --- a/surfsense_backend/app/services/notification_service.py +++ b/surfsense_backend/app/services/notification_service.py @@ -1,4 +1,4 @@ -"""Service for creating and managing notifications with Electric SQL sync.""" +"""Service for creating and managing notifications with Zero sync.""" import logging from datetime import UTC, datetime @@ -1045,7 +1045,7 @@ class PageLimitNotificationHandler(BaseNotificationHandler): class NotificationService: - """Service for creating and managing notifications that sync via Electric SQL.""" + """Service for creating and managing notifications that sync via Zero.""" # Handler instances connector_indexing = ConnectorIndexingNotificationHandler() @@ -1065,7 +1065,7 @@ class NotificationService: notification_metadata: dict[str, Any] | None = None, ) -> Notification: """ - Create a notification - Electric SQL will automatically sync it to frontend. + Create a notification - Zero will automatically sync it to frontend. Args: session: Database session diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index b0f08636a..a7da11749 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -887,7 +887,7 @@ async def _process_file_with_document( ) try: - # Set status to PROCESSING (shows spinner in UI via ElectricSQL) + # Set status to PROCESSING (shows spinner in UI via Zero) document.status = DocumentStatus.processing() await session.commit() logger.info( @@ -951,7 +951,7 @@ async def _process_file_with_document( ): page_limit_error = e.__cause__ - # Mark document as failed (shows error in UI via ElectricSQL) + # Mark document as failed (shows error in UI via Zero) error_message = str(e)[:500] document.status = DocumentStatus.failed(error_message) document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index 6f020685a..f77a0632a 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -139,7 +139,7 @@ async def index_airtable_records( await task_logger.log_task_success( log_entry, success_msg, {"bases_count": 0} ) - # CRITICAL: Update timestamp even when no bases found so Electric SQL syncs + # CRITICAL: Update timestamp even when no bases found so Zero syncs await update_connector_last_indexed( session, connector, update_last_indexed ) @@ -460,7 +460,7 @@ async def index_airtable_records( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) total_processed = documents_indexed diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index 0660531b2..8e64e56ba 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -462,7 +462,7 @@ async def index_bookstack_pages( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs # This ensures the UI shows "Last indexed" instead of "Never indexed" await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index af796ba3c..5a6cc3485 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -470,7 +470,7 @@ async def index_clickup_tasks( total_processed = documents_indexed - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs # This ensures the UI shows "Last indexed" instead of "Never indexed" await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index 3495c59a4..3b46b6437 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -442,7 +442,7 @@ async def index_confluence_pages( documents_failed += 1 continue # Skip this page and continue with others - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs # This ensures the UI shows "Last indexed" instead of "Never indexed" await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index e8e80a646..5e784cb4f 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -718,7 +718,7 @@ async def index_discord_messages( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py index f07c6c580..3283b41eb 100644 --- a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py @@ -413,7 +413,7 @@ async def index_elasticsearch_documents( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs # This ensures the UI shows "Last indexed" instead of "Never indexed" if update_last_indexed: connector.last_indexed_at = ( diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index 61607dda3..ae24d750b 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -451,7 +451,7 @@ async def index_github_repos( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index 99fc666fa..233bc66e4 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -599,7 +599,7 @@ async def index_google_calendar_events( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 30f806c19..384ad85e2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -519,7 +519,7 @@ async def index_google_gmail_messages( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index 1765a592e..25491a8f6 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -422,7 +422,7 @@ async def index_jira_issues( documents_failed += 1 continue # Skip this issue and continue with others - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs # This ensures the UI shows "Last indexed" instead of "Never indexed" await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index bacafccc7..6e9ccaa01 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -463,7 +463,7 @@ async def index_linear_issues( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index 83cf54f4e..a698bfd46 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -520,7 +520,7 @@ async def index_luma_events( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs # This ensures the UI shows "Last indexed" instead of "Never indexed" await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index 85daff94c..619b8dcd7 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -252,7 +252,7 @@ async def index_notion_pages( {"pages_found": 0}, ) logger.info("No Notion pages found to index") - # CRITICAL: Update timestamp even when no pages found so Electric SQL syncs + # CRITICAL: Update timestamp even when no pages found so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) await session.commit() await notion_client.close() @@ -506,7 +506,7 @@ async def index_notion_pages( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) total_processed = documents_indexed diff --git a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py index d53baa3b0..5356ecfb7 100644 --- a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py @@ -599,7 +599,7 @@ async def index_obsidian_vault( failed_count += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py index 1f2693844..2c6d0e11e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py @@ -256,7 +256,7 @@ async def index_slack_messages( f"No Slack channels found for connector {connector_id}", {"channels_found": 0}, ) - # CRITICAL: Update timestamp even when no channels found so Electric SQL syncs + # CRITICAL: Update timestamp even when no channels found so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) await session.commit() return 0, None # Return None (not error) when no channels found @@ -593,7 +593,7 @@ async def index_slack_messages( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py index d04a98177..12cdf384e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py @@ -249,7 +249,7 @@ async def index_teams_messages( f"No Teams found for connector {connector_id}", {"teams_found": 0}, ) - # CRITICAL: Update timestamp even when no teams found so Electric SQL syncs + # CRITICAL: Update timestamp even when no teams found so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) await session.commit() return 0, None # Return None (not error) when no items found @@ -635,7 +635,7 @@ async def index_teams_messages( documents_failed += 1 continue - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index 4d2644420..ada54e7fc 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -444,7 +444,7 @@ async def index_crawled_urls( total_processed = documents_indexed + documents_updated - # CRITICAL: Always update timestamp (even if 0 documents indexed) so Electric SQL syncs + # CRITICAL: Always update timestamp (even if 0 documents indexed) so Zero syncs await update_connector_last_indexed(session, connector, update_last_indexed) # Final commit for any remaining documents not yet committed in batches diff --git a/surfsense_backend/tests/fixtures/sample.md b/surfsense_backend/tests/fixtures/sample.md index 7217540d8..9ba0cacdd 100644 --- a/surfsense_backend/tests/fixtures/sample.md +++ b/surfsense_backend/tests/fixtures/sample.md @@ -10,7 +10,7 @@ document upload pipeline. It includes various markdown formatting elements. - Document upload and processing - Automatic chunking of content - Embedding generation for semantic search -- Real-time status tracking via ElectricSQL +- Real-time status tracking via Zero ## Technical Architecture diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index 9cb01786e..b674d8e9b 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -1,10 +1,7 @@ NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING - -# Electric SQL -NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133 -NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure +NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 # Contact Form Vars (optional) DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres diff --git a/surfsense_web/Dockerfile b/surfsense_web/Dockerfile index 311c3c784..da6bc8b7e 100644 --- a/surfsense_web/Dockerfile +++ b/surfsense_web/Dockerfile @@ -35,15 +35,13 @@ RUN corepack enable pnpm ARG NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__ ARG NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__ ARG NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__ -ARG NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__ -ARG NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__ +ARG NEXT_PUBLIC_ZERO_CACHE_URL=__NEXT_PUBLIC_ZERO_CACHE_URL__ ARG NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__ ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=$NEXT_PUBLIC_FASTAPI_BACKEND_URL ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=$NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE ENV NEXT_PUBLIC_ETL_SERVICE=$NEXT_PUBLIC_ETL_SERVICE -ENV NEXT_PUBLIC_ELECTRIC_URL=$NEXT_PUBLIC_ELECTRIC_URL -ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=$NEXT_PUBLIC_ELECTRIC_AUTH_MODE +ENV NEXT_PUBLIC_ZERO_CACHE_URL=$NEXT_PUBLIC_ZERO_CACHE_URL ENV NEXT_PUBLIC_DEPLOYMENT_MODE=$NEXT_PUBLIC_DEPLOYMENT_MODE COPY --from=deps /app/node_modules ./node_modules diff --git a/surfsense_web/app/api/zero/mutate/route.ts b/surfsense_web/app/api/zero/mutate/route.ts new file mode 100644 index 000000000..0076e1ae8 --- /dev/null +++ b/surfsense_web/app/api/zero/mutate/route.ts @@ -0,0 +1,5 @@ +import { NextResponse } from "next/server"; + +export async function POST() { + return NextResponse.json([]); +} diff --git a/surfsense_web/app/api/zero/query/route.ts b/surfsense_web/app/api/zero/query/route.ts new file mode 100644 index 000000000..3d8ff0d33 --- /dev/null +++ b/surfsense_web/app/api/zero/query/route.ts @@ -0,0 +1,50 @@ +import { mustGetQuery } from "@rocicorp/zero"; +import { handleQueryRequest } from "@rocicorp/zero/server"; +import { NextResponse } from "next/server"; +import type { Context } from "@/types/zero"; +import { queries } from "@/zero/queries"; +import { schema } from "@/zero/schema"; + +const backendURL = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; + +async function authenticateRequest( + request: Request +): Promise<{ ctx: Context; error?: never } | { ctx?: never; error: NextResponse }> { + const authHeader = request.headers.get("Authorization"); + if (!authHeader?.startsWith("Bearer ")) { + return { ctx: undefined }; + } + + try { + const res = await fetch(`${backendURL}/users/me`, { + headers: { Authorization: authHeader }, + }); + + if (!res.ok) { + return { error: NextResponse.json({ error: "Unauthorized" }, { status: 401 }) }; + } + + const user = await res.json(); + return { ctx: { userId: String(user.id) } }; + } catch { + return { error: NextResponse.json({ error: "Auth service unavailable" }, { status: 503 }) }; + } +} + +export async function POST(request: Request) { + const auth = await authenticateRequest(request); + if (auth.error) { + return auth.error; + } + + const result = await handleQueryRequest( + (name, args) => { + const query = mustGetQuery(queries, name); + return query.fn({ args, ctx: auth.ctx }); + }, + schema, + request + ); + + return NextResponse.json(result); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index f0de33826..8578d2dcb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -40,7 +40,7 @@ import { MobileHitlEditPanel } from "@/components/hitl-edit-panel/hitl-edit-pane import { MobileReportPanel } from "@/components/report-panel/report-panel"; import { Skeleton } from "@/components/ui/skeleton"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; -import { useMessagesElectric } from "@/hooks/use-messages-electric"; +import { useMessagesSync } from "@/hooks/use-messages-sync"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getBearerToken } from "@/lib/auth-utils"; import { convertToThreadMessage } from "@/lib/chat/message-utils"; @@ -192,13 +192,13 @@ export default function NewChatPage() { // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); - // Live collaboration: sync session state and messages via Electric SQL + // Live collaboration: sync session state and messages via Zero useChatSessionStateSync(threadId); const { data: membersData } = useAtomValue(membersAtom); - const handleElectricMessagesUpdate = useCallback( + const handleSyncedMessagesUpdate = useCallback( ( - electricMessages: { + syncedMessages: { id: number; thread_id: number; role: string; @@ -212,11 +212,11 @@ export default function NewChatPage() { } setMessages((prev) => { - if (electricMessages.length < prev.length) { + if (syncedMessages.length < prev.length) { return prev; } - return electricMessages.map((msg) => { + return syncedMessages.map((msg) => { const member = msg.author_id ? membersData?.find((m) => m.user_id === msg.author_id) : null; @@ -243,7 +243,7 @@ export default function NewChatPage() { [isRunning, membersData] ); - useMessagesElectric(threadId, handleElectricMessagesUpdate); + useMessagesSync(threadId, handleSyncedMessagesUpdate); // Extract search_space_id from URL params const searchSpaceId = useMemo(() => { @@ -266,6 +266,7 @@ export default function NewChatPage() { // Initialize thread and load messages // For new chats (no urlChatId), we use lazy creation - thread is created on first message + // biome-ignore lint/correctness/useExhaustiveDependencies: searchSpaceId triggers re-init when switching spaces with the same urlChatId const initializeThread = useCallback(async () => { setIsInitializing(true); diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index b0aa58a8f..784fd3bcf 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -3,10 +3,10 @@ import "./globals.css"; import { RootProvider } from "fumadocs-ui/provider/next"; import { Roboto } from "next/font/google"; import { AnnouncementToastProvider } from "@/components/announcements/AnnouncementToastProvider"; -import { ElectricProvider } from "@/components/providers/ElectricProvider"; import { GlobalLoadingProvider } from "@/components/providers/GlobalLoadingProvider"; import { I18nProvider } from "@/components/providers/I18nProvider"; import { PostHogProvider } from "@/components/providers/PostHogProvider"; +import { ZeroProvider } from "@/components/providers/ZeroProvider"; import { ThemeProvider } from "@/components/theme/theme-provider"; import { Toaster } from "@/components/ui/sonner"; import { LocaleProvider } from "@/contexts/LocaleContext"; @@ -141,9 +141,9 @@ export default function RootLayout({ > - + {children} - + diff --git a/surfsense_web/app/sitemap.ts b/surfsense_web/app/sitemap.ts index 5a11ef3fc..f1f0bad72 100644 --- a/surfsense_web/app/sitemap.ts +++ b/surfsense_web/app/sitemap.ts @@ -213,7 +213,7 @@ export default function sitemap(): MetadataRoute.Sitemap { }, // How-to documentation { - url: "https://www.surfsense.com/docs/how-to/electric-sql", + url: "https://www.surfsense.com/docs/how-to/zero-sync", lastModified, changeFrequency: "daily", priority: 0.8, diff --git a/surfsense_web/components/UserDropdown.tsx b/surfsense_web/components/UserDropdown.tsx index 5b398ae0b..b79ab6e79 100644 --- a/surfsense_web/components/UserDropdown.tsx +++ b/surfsense_web/components/UserDropdown.tsx @@ -16,7 +16,6 @@ import { } from "@/components/ui/dropdown-menu"; import { Spinner } from "@/components/ui/spinner"; import { logout } from "@/lib/auth-utils"; -import { cleanupElectric } from "@/lib/electric/client"; import { resetUser, trackLogout } from "@/lib/posthog/events"; export function UserDropdown({ @@ -39,14 +38,6 @@ export function UserDropdown({ trackLogout(); resetUser(); - // Best-effort cleanup of Electric SQL / PGlite - // Even if this fails, login-time cleanup will handle it - try { - await cleanupElectric(); - } catch (err) { - console.warn("[Logout] Electric cleanup failed (will be handled on next login):", err); - } - // Revoke refresh token on server and clear all tokens from localStorage await logout(); diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 569b2ece4..3187d3c33 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -20,7 +20,7 @@ import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; import { Spinner } from "@/components/ui/spinner"; import { Tabs, TabsContent } from "@/components/ui/tabs"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { useConnectorsElectric } from "@/hooks/use-connectors-electric"; +import { useConnectorsSync } from "@/hooks/use-connectors-sync"; import { PICKER_CLOSE_EVENT, PICKER_OPEN_EVENT } from "@/hooks/use-google-picker"; import { cn } from "@/lib/utils"; import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header"; @@ -155,33 +155,23 @@ export const ConnectorIndicator = forwardRef 0 || (connectorsLoading && !connectorsError); - const connectors = useElectricData ? connectorsFromElectric : allConnectors || []; + const useSyncData = connectorsFromSync.length > 0 || (connectorsLoading && !connectorsError); + const connectors = useSyncData ? connectorsFromSync : allConnectors || []; - // Manual refresh function that works with both Electric and API const refreshConnectors = async () => { - if (useElectricData) { - await refreshConnectorsElectric(); - } else { - // Fallback: use allConnectors from useConnectorDialog (which uses connectorsAtom) - // The connectorsAtom will handle refetching if needed + if (useSyncData) { + await refreshConnectorsSync(); } }; - // Track indexing state locally - clears automatically when Electric SQL detects last_indexed_at changed + // Track indexing state locally - clears automatically when last_indexed_at changes via real-time sync // Also clears when failed notifications are detected const { indexingConnectorIds, startIndexing, stopIndexing } = useIndexingConnectors( connectors as SearchSourceConnector[], @@ -202,7 +192,7 @@ export const ConnectorIndicator = forwardRef( (connectors || []).map((c: SearchSourceConnector) => c.connector_type) ); @@ -291,7 +281,7 @@ export const ConnectorIndicator = forwardRef c.id === editingConnector.id) ?.last_indexed_at ?? editingConnector.last_indexed_at, diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index c710347e3..7b4aa29fb 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -1254,7 +1254,7 @@ export const useConnectorDialog = () => { queryKey: cacheKeys.logs.summary(Number(searchSpaceId)), }); // Note: Don't call stopIndexing here - let useIndexingConnectors hook - // detect when last_indexed_at changes via Electric SQL + // detect when last_indexed_at changes via real-time sync } catch (error) { console.error("Error indexing connector content:", error); toast.error(error instanceof Error ? error.message : "Failed to start indexing"); diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-indexing-connectors.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-indexing-connectors.ts index 5783540d8..fb778fc3b 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-indexing-connectors.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-indexing-connectors.ts @@ -48,13 +48,13 @@ function isTaskTimedOut(startedAt: string | null | undefined): boolean { * * This provides a better UX than polling by: * 1. Setting indexing state immediately when user triggers indexing (optimistic) - * 2. Detecting in_progress notifications from Electric SQL to restore state after remounts + * 2. Detecting in_progress notifications to restore state after remounts * 3. Clearing indexing state when notifications become completed or failed - * 4. Clearing indexing state when Electric SQL detects last_indexed_at changed + * 4. Clearing indexing state when real-time sync detects last_indexed_at changed * 5. Detecting stale/stuck tasks that haven't updated in 15+ minutes * 6. Detecting hard timeout (8h) - tasks that definitely cannot still be running * - * The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state. + * The actual `last_indexed_at` value comes from real-time sync, not local state. */ export function useIndexingConnectors( connectors: SearchSourceConnector[], @@ -66,7 +66,7 @@ export function useIndexingConnectors( // Track previous last_indexed_at values to detect changes const previousLastIndexedAtRef = useRef>(new Map()); - // Detect when last_indexed_at changes (indexing completed) via Electric SQL + // Detect when last_indexed_at changes (indexing completed) via real-time sync useEffect(() => { const previousValues = previousLastIndexedAtRef.current; diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 89869dc7e..195afc090 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -81,7 +81,7 @@ import { } from "@/contracts/enums/toolIcons"; import type { Document } from "@/contracts/types/document.types"; import { useBatchCommentsPreload } from "@/hooks/use-comments"; -import { useCommentsElectric } from "@/hooks/use-comments-electric"; +import { useCommentsSync } from "@/hooks/use-comments-sync"; import { useMediaQuery } from "@/hooks/use-media-query"; import { cn } from "@/lib/utils"; @@ -347,8 +347,8 @@ const Composer: FC = () => { const respondingToUserId = sessionState?.respondingToUserId ?? null; const isBlockedByOtherUser = isAiResponding && respondingToUserId !== currentUser?.id; - // Sync comments for the entire thread via Electric SQL (one subscription per thread) - useCommentsElectric(threadId); + // Sync comments for the entire thread via Zero (one subscription per thread) + useCommentsSync(threadId); // Batch-prefetch comments for all assistant messages so individual useComments // hooks never fire their own network requests (eliminates N+1 API calls). diff --git a/surfsense_web/components/contact/contact-form.tsx b/surfsense_web/components/contact/contact-form.tsx index 967c1c524..03e4118de 100644 --- a/surfsense_web/components/contact/contact-form.tsx +++ b/surfsense_web/components/contact/contact-form.tsx @@ -161,7 +161,7 @@ export const FeatureIconContainer = ({ ); }; -export const Grid = ({ pattern, size }: { pattern?: number[][]; size?: number }) => { +export const Grid = ({ pattern, size }: { pattern?: [number, number][]; size?: number }) => { const p = pattern ?? [ [9, 3], [8, 5], @@ -185,7 +185,7 @@ export const Grid = ({ pattern, size }: { pattern?: number[][]; size?: number }) ); }; -export function GridPattern({ width, height, x, y, squares, ...props }: any) { +export function GridPattern({ width, height, x, y, squares, ...props }: React.ComponentProps<"svg"> & { width: number; height: number; x: string | number; y: string | number; squares?: [number, number][] }) { const patternId = useId(); return ( @@ -205,7 +205,7 @@ export function GridPattern({ width, height, x, y, squares, ...props }: any) { {squares && (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ), + } +); + // Official Google "G" logo with brand colors const GoogleLogo = ({ className }: { className?: string }) => ( diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 9f302a91b..be139c586 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -54,7 +54,6 @@ import { notificationsApiService } from "@/lib/apis/notifications-api.service"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { logout } from "@/lib/auth-utils"; import { deleteThread, fetchThreads, updateThread } from "@/lib/chat/thread-persistence"; -import { cleanupElectric } from "@/lib/electric/client"; import { resetUser, trackLogout } from "@/lib/posthog/events"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import type { ChatItem, NavItem, SearchSpace } from "../types/layout.types"; @@ -155,8 +154,6 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid // Search space dialog state const [isCreateSearchSpaceDialogOpen, setIsCreateSearchSpaceDialogOpen] = useState(false); - // Per-tab inbox hooks — each has independent API loading, pagination, - // and Electric live queries. The Electric sync shape is shared (client-level cache). const userId = user?.id ? String(user.id) : null; const numericSpaceId = Number(searchSpaceId) || null; @@ -579,14 +576,6 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid trackLogout(); resetUser(); - // Best-effort cleanup of Electric SQL / PGlite - // Even if this fails, login-time cleanup will handle it - try { - await cleanupElectric(); - } catch (err) { - console.warn("[Logout] Electric cleanup failed (will be handled on next login):", err); - } - // Revoke refresh token on server and clear all tokens from localStorage await logout(); diff --git a/surfsense_web/components/providers/ElectricProvider.tsx b/surfsense_web/components/providers/ElectricProvider.tsx deleted file mode 100644 index 02005d098..000000000 --- a/surfsense_web/components/providers/ElectricProvider.tsx +++ /dev/null @@ -1,118 +0,0 @@ -"use client"; - -import { useAtomValue } from "jotai"; -import { usePathname } from "next/navigation"; -import { useEffect, useRef, useState } from "react"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import { useGlobalLoadingEffect } from "@/hooks/use-global-loading"; -import { getBearerToken } from "@/lib/auth-utils"; -import { - cleanupElectric, - type ElectricClient, - initElectric, - isElectricInitialized, -} from "@/lib/electric/client"; -import { ElectricContext } from "@/lib/electric/context"; - -const IS_DEV = process.env.NODE_ENV === "development"; - -interface ElectricProviderProps { - children: React.ReactNode; -} - -/** - * Initializes user-specific PGlite database with Electric SQL sync. - * Handles user isolation, cleanup, and re-initialization on user change. - */ -export function ElectricProvider({ children }: ElectricProviderProps) { - const [electricClient, setElectricClient] = useState(null); - const [error, setError] = useState(null); - const { - data: user, - isSuccess: isUserLoaded, - isError: isUserError, - } = useAtomValue(currentUserAtom); - const previousUserIdRef = useRef(null); - const initializingRef = useRef(false); - const pathname = usePathname(); - - useEffect(() => { - if (typeof window === "undefined") return; - - // No user logged in - cleanup if previous user existed - if (!isUserLoaded || !user?.id) { - if (previousUserIdRef.current && isElectricInitialized()) { - if (IS_DEV) console.log("[ElectricProvider] User logged out, cleaning up..."); - cleanupElectric().then(() => { - previousUserIdRef.current = null; - setElectricClient(null); - }); - } - return; - } - - const userId = String(user.id); - - // Skip if already initialized for this user or currently initializing - if ((electricClient && previousUserIdRef.current === userId) || initializingRef.current) { - return; - } - - initializingRef.current = true; - let mounted = true; - - async function init() { - try { - if (IS_DEV) console.log(`[ElectricProvider] Initializing for user: ${userId}`); - const client = await initElectric(userId); - - if (mounted) { - previousUserIdRef.current = userId; - setElectricClient(client); - setError(null); - if (IS_DEV) console.log(`[ElectricProvider] ✅ Ready for user: ${userId}`); - } - } catch (err) { - console.error("[ElectricProvider] Failed to initialize:", err); - if (mounted) { - setError(err instanceof Error ? err : new Error("Failed to initialize Electric SQL")); - setElectricClient(null); - } - } finally { - if (mounted) { - initializingRef.current = false; - } - } - } - - init(); - return () => { - mounted = false; - }; - }, [user?.id, isUserLoaded, electricClient]); - - const hasToken = typeof window !== "undefined" && !!getBearerToken(); - - // Only block UI on dashboard routes; public pages render immediately - const requiresElectricLoading = pathname?.startsWith("/dashboard"); - const shouldShowLoading = - hasToken && isUserLoaded && !!user?.id && !electricClient && !error && requiresElectricLoading; - - useGlobalLoadingEffect(shouldShowLoading); - - // Render immediately for unauthenticated users or failed user queries - if (!hasToken || !isUserLoaded || !user?.id || isUserError) { - return {children}; - } - - // Render with null context while initializing - if (!electricClient && !error) { - return {children}; - } - - if (error) { - console.warn("[ElectricProvider] Initialization failed, sync may not work:", error.message); - } - - return {children}; -} diff --git a/surfsense_web/components/providers/ZeroProvider.tsx b/surfsense_web/components/providers/ZeroProvider.tsx new file mode 100644 index 000000000..f4df921f3 --- /dev/null +++ b/surfsense_web/components/providers/ZeroProvider.tsx @@ -0,0 +1,65 @@ +"use client"; + +import { + useConnectionState, + useZero, + ZeroProvider as ZeroReactProvider, +} from "@rocicorp/zero/react"; +import { useAtomValue } from "jotai"; +import { useEffect, useRef } from "react"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import { getBearerToken, handleUnauthorized, refreshAccessToken } from "@/lib/auth-utils"; +import { queries } from "@/zero/queries"; +import { schema } from "@/zero/schema"; + +const cacheURL = process.env.NEXT_PUBLIC_ZERO_CACHE_URL || "http://localhost:4848"; + +function ZeroAuthGuard({ children }: { children: React.ReactNode }) { + const zero = useZero(); + const connectionState = useConnectionState(); + const isRefreshingRef = useRef(false); + + useEffect(() => { + if (connectionState.name !== "needs-auth" || isRefreshingRef.current) return; + + isRefreshingRef.current = true; + + refreshAccessToken() + .then((newToken) => { + if (newToken) { + zero.connection.connect({ auth: newToken }); + } else { + handleUnauthorized(); + } + }) + .finally(() => { + isRefreshingRef.current = false; + }); + }, [connectionState, zero]); + + return <>{children}; +} + +export function ZeroProvider({ children }: { children: React.ReactNode }) { + const { data: user } = useAtomValue(currentUserAtom); + + const hasUser = !!user?.id; + const userID = hasUser ? String(user.id) : "anon"; + const context = hasUser ? { userId: String(user.id) } : undefined; + const auth = hasUser ? getBearerToken() || undefined : undefined; + + const opts = { + userID, + schema, + queries, + context, + cacheURL, + auth, + }; + + return ( + + {hasUser ? {children} : children} + + ); +} diff --git a/surfsense_web/components/settings/roles-manager.tsx b/surfsense_web/components/settings/roles-manager.tsx index 23b9aa4b6..0ccbb077d 100644 --- a/surfsense_web/components/settings/roles-manager.tsx +++ b/surfsense_web/components/settings/roles-manager.tsx @@ -510,93 +510,87 @@ function RolesContent({
{roles.map((role) => (
- -
- {role.description && ( -

- {role.description} -

- )} -
+ +
-
- -
+
+ +
- {!role.is_system_role && ( -
e.stopPropagation()} - onKeyDown={(e) => e.stopPropagation()} - > - - - - - e.preventDefault()}> - {canUpdate && ( - setEditingRoleId(role.id)}> - - Edit Role - - )} - {canDelete && ( - <> - - - - e.preventDefault()}> - - Delete Role - - - - - Delete role? - - This will permanently delete the "{role.name}" role. - Members with this role will lose their permissions. - - - - Cancel - onDeleteRole(role.id)} - className="bg-destructive text-destructive-foreground hover:bg-destructive/90" - > - Delete - - - - - - )} - - -
- )} - - + {!role.is_system_role && ( +
+ + + + + e.preventDefault()}> + {canUpdate && ( + setEditingRoleId(role.id)}> + + Edit Role + + )} + {canDelete && ( + <> + + + + e.preventDefault()}> + + Delete Role + + + + + Delete role? + + This will permanently delete the "{role.name}" role. + Members with this role will lose their permissions. + + + + Cancel + onDeleteRole(role.id)} + className="bg-destructive text-destructive-foreground hover:bg-destructive/90" + > + Delete + + + + + + )} + + +
+ )} +
))} diff --git a/surfsense_web/components/tool-ui/audio.tsx b/surfsense_web/components/tool-ui/audio.tsx index ad4adeac0..dae752034 100644 --- a/surfsense_web/components/tool-ui/audio.tsx +++ b/surfsense_web/components/tool-ui/audio.tsx @@ -24,6 +24,7 @@ function formatTime(seconds: number): string { export function Audio({ id, src, title, durationMs, className }: AudioProps) { const audioRef = useRef(null); + const downloadControllerRef = useRef(null); const [isPlaying, setIsPlaying] = useState(false); const [currentTime, setCurrentTime] = useState(0); const [duration, setDuration] = useState(durationMs ? durationMs / 1000 : 0); @@ -81,8 +82,12 @@ export function Audio({ id, src, title, durationMs, className }: AudioProps) { // Handle download const handleDownload = useCallback(async () => { + downloadControllerRef.current?.abort(); + const controller = new AbortController(); + downloadControllerRef.current = controller; + try { - const response = await fetch(src); + const response = await fetch(src, { signal: controller.signal }); const blob = await response.blob(); const url = window.URL.createObjectURL(blob); const a = document.createElement("a"); @@ -93,10 +98,16 @@ export function Audio({ id, src, title, durationMs, className }: AudioProps) { document.body.removeChild(a); window.URL.revokeObjectURL(url); } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") return; console.error("Error downloading audio:", err); } }, [src, title]); + // Abort in-flight download on unmount + useEffect(() => { + return () => downloadControllerRef.current?.abort(); + }, []); + // Set up audio event listeners useEffect(() => { const audio = audioRef.current; diff --git a/surfsense_web/components/ui/hero-carousel.tsx b/surfsense_web/components/ui/hero-carousel.tsx index e051ac518..5760aad94 100644 --- a/surfsense_web/components/ui/hero-carousel.tsx +++ b/surfsense_web/components/ui/hero-carousel.tsx @@ -80,11 +80,23 @@ function HeroCarouselCard({ useEffect(() => { const video = videoRef.current; - if (video) { - setHasLoaded(false); - video.currentTime = 0; - video.play().catch(() => {}); - } + if (!video) return; + + setHasLoaded(false); + video.currentTime = 0; + + const observer = new IntersectionObserver( + ([entry]) => { + if (entry.isIntersecting) { + video.play().catch(() => {}); + observer.disconnect(); + } + }, + { threshold: 0.1 } + ); + observer.observe(video); + + return () => observer.disconnect(); }, [src]); const handleCanPlay = useCallback(() => { @@ -94,7 +106,6 @@ function HeroCarouselCard({ return ( <>
- {" "}

@@ -108,7 +119,7 @@ function HeroCarouselCard({