diff --git a/docker/.env.example b/docker/.env.example index fc0b0de6d..3729f369a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -79,6 +79,23 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # Change ZERO_ADMIN_PASSWORD for security in production. # ZERO_ADMIN_PASSWORD=surfsense-zero-admin + +# Publication restricting which tables zero-cache replicates from Postgres. +# Created automatically by Alembic migration 116. +# Only change this if you manage publications manually. +# ZERO_APP_PUBLICATIONS=zero_publication + +# Sync worker tuning — zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number +# of CPU cores, which can exceed the connection pool limits on high-core machines. +# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR +# pools, so these constraints must hold: +# ZERO_UPSTREAM_MAX_CONNS >= ZERO_NUM_SYNC_WORKERS +# ZERO_CVR_MAX_CONNS >= ZERO_NUM_SYNC_WORKERS +# Default of 4 workers is sufficient for self-hosted / personal use. +# ZERO_NUM_SYNC_WORKERS=4 +# ZERO_UPSTREAM_MAX_CONNS=20 +# ZERO_CVR_MAX_CONNS=30 + # Full override for the Zero → Postgres connection URLs. # Leave commented out to use the Docker-managed `db` container (default). # ZERO_UPSTREAM_DB=postgresql://surfsense:surfsense@db:5432/surfsense diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 564ecd772..c7922e3ef 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -176,7 +176,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" depends_on: - db: + backend: condition: service_healthy environment: - ZERO_UPSTREAM_DB=${ZERO_UPSTREAM_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} @@ -184,6 +184,10 @@ services: - ZERO_CHANGE_DB=${ZERO_CHANGE_DB:-postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} - ZERO_REPLICA_FILE=/data/zero.db - ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} + - ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication} + - ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4} + - ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20} + - ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30} - ZERO_QUERY_URL=${ZERO_QUERY_URL:-http://frontend:3000/api/zero/query} - ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b03efdd2f..549190947 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -170,13 +170,17 @@ services: ZERO_CHANGE_DB: ${ZERO_CHANGE_DB:-postgresql://${DB_USER:-surfsense}:${DB_PASSWORD:-surfsense}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} ZERO_REPLICA_FILE: /data/zero.db ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} + ZERO_APP_PUBLICATIONS: ${ZERO_APP_PUBLICATIONS:-zero_publication} + ZERO_NUM_SYNC_WORKERS: ${ZERO_NUM_SYNC_WORKERS:-4} + ZERO_UPSTREAM_MAX_CONNS: ${ZERO_UPSTREAM_MAX_CONNS:-20} + ZERO_CVR_MAX_CONNS: ${ZERO_CVR_MAX_CONNS:-30} ZERO_QUERY_URL: ${ZERO_QUERY_URL:-http://frontend:3000/api/zero/query} ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} volumes: - zero_cache_data:/data restart: unless-stopped depends_on: - db: + backend: condition: service_healthy healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"] diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index 3e79e58f4..bd7f579d0 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -71,6 +71,10 @@ Defaults work out of the box. Change `ZERO_ADMIN_PASSWORD` for security in produ | `ZERO_UPSTREAM_DB` | PostgreSQL connection URL for replication (must be a direct connection, not via pgbouncer) | *(built from DB_* vars)* | | `ZERO_CVR_DB` | PostgreSQL connection URL for client view records | *(built from DB_* vars)* | | `ZERO_CHANGE_DB` | PostgreSQL connection URL for replication log entries | *(built from DB_* vars)* | +| `ZERO_APP_PUBLICATIONS` | PostgreSQL publication restricting which tables are replicated (created by migration 116) | `zero_publication` | +| `ZERO_NUM_SYNC_WORKERS` | Number of view-sync worker processes. Must be ≤ connection pool sizes | `4` | +| `ZERO_UPSTREAM_MAX_CONNS` | Max connections to upstream PostgreSQL for mutations | `20` | +| `ZERO_CVR_MAX_CONNS` | Max connections to the CVR database | `30` | ### Database diff --git a/surfsense_web/content/docs/how-to/zero-sync.mdx b/surfsense_web/content/docs/how-to/zero-sync.mdx index df7ffe0bc..e764d384a 100644 --- a/surfsense_web/content/docs/how-to/zero-sync.mdx +++ b/surfsense_web/content/docs/how-to/zero-sync.mdx @@ -36,6 +36,10 @@ zero-cache is included in the Docker Compose setup. The key environment variable | `ZERO_ADMIN_PASSWORD` | Password for the zero-cache admin UI and `/statz` endpoint | `surfsense-zero-admin` | | `ZERO_UPSTREAM_DB` | PostgreSQL connection URL for replication | Built from `DB_*` vars | | `NEXT_PUBLIC_ZERO_CACHE_URL` | URL the frontend uses to connect to zero-cache | `http://localhost:` | +| `ZERO_APP_PUBLICATIONS` | PostgreSQL publication restricting which tables are replicated | `zero_publication` | +| `ZERO_NUM_SYNC_WORKERS` | Number of view-sync worker processes. Must be ≤ `ZERO_UPSTREAM_MAX_CONNS` and ≤ `ZERO_CVR_MAX_CONNS` | `4` | +| `ZERO_UPSTREAM_MAX_CONNS` | Max connections to upstream PostgreSQL for mutations | `20` | +| `ZERO_CVR_MAX_CONNS` | Max connections to the CVR database | `30` | ### Manual / Local Development @@ -83,6 +87,7 @@ Zero syncs the following tables for real-time features: ## Troubleshooting - **zero-cache not starting**: Check `docker compose logs zero-cache`. Ensure PostgreSQL has `wal_level=logical` (configured in `postgresql.conf`). +- **"Insufficient upstream connections" error**: zero-cache defaults `ZERO_NUM_SYNC_WORKERS` to the number of CPU cores, which can exceed connection pool limits on high-core machines. Lower `ZERO_NUM_SYNC_WORKERS` or raise `ZERO_UPSTREAM_MAX_CONNS` / `ZERO_CVR_MAX_CONNS` in your `.env`. - **Frontend not syncing**: Open DevTools → Console and check for WebSocket connection errors. Verify `NEXT_PUBLIC_ZERO_CACHE_URL` matches the running zero-cache address. - **Stale data after restart**: zero-cache rebuilds its SQLite replica from PostgreSQL on startup. This may take a moment for large databases.