diff --git a/docker/.env.example b/docker/.env.example index fc0b0de6d..3729f369a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -79,6 +79,23 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # Change ZERO_ADMIN_PASSWORD for security in production. # ZERO_ADMIN_PASSWORD=surfsense-zero-admin + +# Publication restricting which tables zero-cache replicates from Postgres. +# Created automatically by Alembic migration 116. +# Only change this if you manage publications manually. +# ZERO_APP_PUBLICATIONS=zero_publication + +# Sync worker tuning — zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number +# of CPU cores, which can exceed the connection pool limits on high-core machines. +# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR +# pools, so these constraints must hold: +# ZERO_UPSTREAM_MAX_CONNS >= ZERO_NUM_SYNC_WORKERS +# ZERO_CVR_MAX_CONNS >= ZERO_NUM_SYNC_WORKERS +# Default of 4 workers is sufficient for self-hosted / personal use. +# ZERO_NUM_SYNC_WORKERS=4 +# ZERO_UPSTREAM_MAX_CONNS=20 +# ZERO_CVR_MAX_CONNS=30 + # Full override for the Zero → Postgres connection URLs. # Leave commented out to use the Docker-managed `db` container (default). # ZERO_UPSTREAM_DB=postgresql://surfsense:surfsense@db:5432/surfsense diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 564ecd772..c7922e3ef 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -176,7 +176,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" depends_on: - db: + backend: condition: service_healthy environment: - ZERO_UPSTREAM_DB=${ZERO_UPSTREAM_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} @@ -184,6 +184,10 @@ services: - ZERO_CHANGE_DB=${ZERO_CHANGE_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} - ZERO_REPLICA_FILE=/data/zero.db - ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} + - ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication} + - ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4} + - ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20} + - ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30} - ZERO_QUERY_URL=${ZERO_QUERY_URL:-http://frontend:3000/api/zero/query} - ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b03efdd2f..549190947 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -170,13 +170,17 @@ services: ZERO_CHANGE_DB: ${ZERO_CHANGE_DB:-postgresql://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} ZERO_REPLICA_FILE: /data/zero.db ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} + ZERO_APP_PUBLICATIONS: ${ZERO_APP_PUBLICATIONS:-zero_publication} + ZERO_NUM_SYNC_WORKERS: ${ZERO_NUM_SYNC_WORKERS:-4} + ZERO_UPSTREAM_MAX_CONNS: ${ZERO_UPSTREAM_MAX_CONNS:-20} + ZERO_CVR_MAX_CONNS: ${ZERO_CVR_MAX_CONNS:-30} ZERO_QUERY_URL: ${ZERO_QUERY_URL:-http://frontend:3000/api/zero/query} ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} volumes: - zero_cache_data:/data restart: unless-stopped depends_on: - db: + backend: condition: service_healthy healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"] diff --git a/surfsense_backend/alembic/versions/116_create_zero_publication.py b/surfsense_backend/alembic/versions/116_create_zero_publication.py new file mode 100644 index 000000000..8f0d7b5d3 --- /dev/null +++ b/surfsense_backend/alembic/versions/116_create_zero_publication.py @@ -0,0 +1,52 @@ +"""create zero_publication for zero-cache replication + +Restricts zero-cache replication to only the tables the frontend +queries via Zero, instead of replicating all tables in public schema. + +See: https://zero.rocicorp.dev/docs/zero-cache-config#app-publications + +Revision ID: 116 +Revises: 115 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "116" +down_revision: str | None = "115" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +TABLES = [ + "notifications", + "documents", + "folders", + "search_source_connectors", + "new_chat_messages", + "chat_comments", + "chat_session_state", +] + + +def upgrade() -> None: + conn = op.get_bind() + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + table_list = ", ".join(TABLES) + conn.execute( + sa.text( + f"CREATE PUBLICATION {PUBLICATION_NAME} FOR TABLE {table_list}" + ) + ) + + +def downgrade() -> None: + op.execute(f"DROP PUBLICATION IF EXISTS {PUBLICATION_NAME}") diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index 3e79e58f4..bd7f579d0 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -71,6 +71,10 @@ Defaults work out of the box. Change `ZERO_ADMIN_PASSWORD` for security in produ | `ZERO_UPSTREAM_DB` | PostgreSQL connection URL for replication (must be a direct connection, not via pgbouncer) | *(built from DB_* vars)* | | `ZERO_CVR_DB` | PostgreSQL connection URL for client view records | *(built from DB_* vars)* | | `ZERO_CHANGE_DB` | PostgreSQL connection URL for replication log entries | *(built from DB_* vars)* | +| `ZERO_APP_PUBLICATIONS` | PostgreSQL publication restricting which tables are replicated (created by migration 116) | `zero_publication` | +| `ZERO_NUM_SYNC_WORKERS` | Number of view-sync worker processes. Must be ≤ connection pool sizes | `4` | +| `ZERO_UPSTREAM_MAX_CONNS` | Max connections to upstream PostgreSQL for mutations | `20` | +| `ZERO_CVR_MAX_CONNS` | Max connections to the CVR database | `30` | ### Database diff --git a/surfsense_web/content/docs/how-to/zero-sync.mdx b/surfsense_web/content/docs/how-to/zero-sync.mdx index df7ffe0bc..e764d384a 100644 --- a/surfsense_web/content/docs/how-to/zero-sync.mdx +++ b/surfsense_web/content/docs/how-to/zero-sync.mdx @@ -36,6 +36,10 @@ zero-cache is included in the Docker Compose setup. The key environment variable | `ZERO_ADMIN_PASSWORD` | Password for the zero-cache admin UI and `/statz` endpoint | `surfsense-zero-admin` | | `ZERO_UPSTREAM_DB` | PostgreSQL connection URL for replication | Built from `DB_*` vars | | `NEXT_PUBLIC_ZERO_CACHE_URL` | URL the frontend uses to connect to zero-cache | `http://localhost:` | +| `ZERO_APP_PUBLICATIONS` | PostgreSQL publication restricting which tables are replicated | `zero_publication` | +| `ZERO_NUM_SYNC_WORKERS` | Number of view-sync worker processes. Must be ≤ `ZERO_UPSTREAM_MAX_CONNS` and ≤ `ZERO_CVR_MAX_CONNS` | `4` | +| `ZERO_UPSTREAM_MAX_CONNS` | Max connections to upstream PostgreSQL for mutations | `20` | +| `ZERO_CVR_MAX_CONNS` | Max connections to the CVR database | `30` | ### Manual / Local Development @@ -83,6 +87,7 @@ Zero syncs the following tables for real-time features: ## Troubleshooting - **zero-cache not starting**: Check `docker compose logs zero-cache`. Ensure PostgreSQL has `wal_level=logical` (configured in `postgresql.conf`). +- **"Insufficient upstream connections" error**: zero-cache defaults `ZERO_NUM_SYNC_WORKERS` to the number of CPU cores, which can exceed connection pool limits on high-core machines. Lower `ZERO_NUM_SYNC_WORKERS` or raise `ZERO_UPSTREAM_MAX_CONNS` / `ZERO_CVR_MAX_CONNS` in your `.env`. - **Frontend not syncing**: Open DevTools → Console and check for WebSocket connection errors. Verify `NEXT_PUBLIC_ZERO_CACHE_URL` matches the running zero-cache address. - **Stale data after restart**: zero-cache rebuilds its SQLite replica from PostgreSQL on startup. This may take a moment for large databases.