diff --git a/surfsense_web/app/sitemap.ts b/surfsense_web/app/sitemap.ts index 2f59c2b2d..414b41719 100644 --- a/surfsense_web/app/sitemap.ts +++ b/surfsense_web/app/sitemap.ts @@ -55,7 +55,25 @@ export default function sitemap(): MetadataRoute.Sitemap { priority: 0.9, }, { - url: "https://www.surfsense.com/docs/docker-installation", + url: "https://www.surfsense.com/docs/docker-installation/install-script", + lastModified, + changeFrequency: "daily", + priority: 0.9, + }, + { + url: "https://www.surfsense.com/docs/docker-installation/docker-compose", + lastModified, + changeFrequency: "daily", + priority: 0.9, + }, + { + url: "https://www.surfsense.com/docs/docker-installation/updating", + lastModified, + changeFrequency: "daily", + priority: 0.9, + }, + { + url: "https://www.surfsense.com/docs/docker-installation/dev-compose", lastModified, changeFrequency: "daily", priority: 0.9, diff --git a/surfsense_web/content/docs/connectors/airtable.mdx b/surfsense_web/content/docs/connectors/airtable.mdx index 71148335c..e948e9dd3 100644 --- a/surfsense_web/content/docs/connectors/airtable.mdx +++ b/surfsense_web/content/docs/connectors/airtable.mdx @@ -88,7 +88,7 @@ After saving, you'll find your OAuth credentials on the integration page: ## Running SurfSense with Airtable Connector -Add the Airtable credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Airtable credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash AIRTABLE_CLIENT_ID=your_airtable_client_id diff --git a/surfsense_web/content/docs/connectors/clickup.mdx b/surfsense_web/content/docs/connectors/clickup.mdx index 768bca859..31709dfdd 100644 --- a/surfsense_web/content/docs/connectors/clickup.mdx +++ b/surfsense_web/content/docs/connectors/clickup.mdx @@ -44,7 +44,7 @@ After creating the app, you'll see your credentials: ## Running SurfSense with ClickUp Connector -Add the ClickUp credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the ClickUp credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash CLICKUP_CLIENT_ID=your_clickup_client_id diff --git a/surfsense_web/content/docs/connectors/confluence.mdx b/surfsense_web/content/docs/connectors/confluence.mdx index 3ee3394a4..08effa104 100644 --- a/surfsense_web/content/docs/connectors/confluence.mdx +++ b/surfsense_web/content/docs/connectors/confluence.mdx @@ -97,7 +97,7 @@ Select the **"Granular scopes"** tab and enable: ## Running SurfSense with Confluence Connector -Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash ATLASSIAN_CLIENT_ID=your_atlassian_client_id diff --git a/surfsense_web/content/docs/connectors/discord.mdx b/surfsense_web/content/docs/connectors/discord.mdx index 05825e0ea..cef0c2d10 100644 --- a/surfsense_web/content/docs/connectors/discord.mdx +++ b/surfsense_web/content/docs/connectors/discord.mdx @@ -64,7 +64,7 @@ You'll also see your **Application ID** and **Public Key** on this page. ## Running SurfSense with Discord Connector -Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash DISCORD_CLIENT_ID=your_discord_client_id diff --git a/surfsense_web/content/docs/connectors/gmail.mdx b/surfsense_web/content/docs/connectors/gmail.mdx index 1b3f81efe..4b7e93c49 100644 --- a/surfsense_web/content/docs/connectors/gmail.mdx +++ b/surfsense_web/content/docs/connectors/gmail.mdx @@ -70,7 +70,7 @@ This guide walks you through setting up a Google OAuth 2.0 integration for SurfS ## Running SurfSense with Gmail Connector -Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash GOOGLE_OAUTH_CLIENT_ID=your_google_client_id diff --git a/surfsense_web/content/docs/connectors/google-calendar.mdx b/surfsense_web/content/docs/connectors/google-calendar.mdx index 481b05444..f0745c8a0 100644 --- a/surfsense_web/content/docs/connectors/google-calendar.mdx +++ b/surfsense_web/content/docs/connectors/google-calendar.mdx @@ -69,7 +69,7 @@ This guide walks you through setting up a Google OAuth 2.0 integration for SurfS ## Running SurfSense with Google Calendar Connector -Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash GOOGLE_OAUTH_CLIENT_ID=your_google_client_id diff --git a/surfsense_web/content/docs/connectors/google-drive.mdx b/surfsense_web/content/docs/connectors/google-drive.mdx index 238100860..1193b860b 100644 --- a/surfsense_web/content/docs/connectors/google-drive.mdx +++ b/surfsense_web/content/docs/connectors/google-drive.mdx @@ -70,7 +70,7 @@ This guide walks you through setting up a Google OAuth 2.0 integration for SurfS ## Running SurfSense with Google Drive Connector -Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash GOOGLE_OAUTH_CLIENT_ID=your_google_client_id diff --git a/surfsense_web/content/docs/connectors/jira.mdx b/surfsense_web/content/docs/connectors/jira.mdx index 5bddbab8d..ddbc70a8f 100644 --- a/surfsense_web/content/docs/connectors/jira.mdx +++ b/surfsense_web/content/docs/connectors/jira.mdx @@ -84,7 +84,7 @@ This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration ## Running SurfSense with Jira Connector -Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash ATLASSIAN_CLIENT_ID=your_atlassian_client_id diff --git a/surfsense_web/content/docs/connectors/linear.mdx b/surfsense_web/content/docs/connectors/linear.mdx index 3fd82aba1..1dd5af9d5 100644 --- a/surfsense_web/content/docs/connectors/linear.mdx +++ b/surfsense_web/content/docs/connectors/linear.mdx @@ -53,7 +53,7 @@ After creating the application, you'll see your OAuth credentials: ## Running SurfSense with Linear Connector -Add the Linear credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Linear credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash LINEAR_CLIENT_ID=your_linear_client_id diff --git a/surfsense_web/content/docs/connectors/microsoft-teams.mdx b/surfsense_web/content/docs/connectors/microsoft-teams.mdx index 5a05be709..aba64da20 100644 --- a/surfsense_web/content/docs/connectors/microsoft-teams.mdx +++ b/surfsense_web/content/docs/connectors/microsoft-teams.mdx @@ -90,7 +90,7 @@ After registration, you'll be taken to the app's **Overview** page. Here you'll ## Running SurfSense with Microsoft Teams Connector -Add the Microsoft Teams credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Microsoft Teams credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash TEAMS_CLIENT_ID=your_microsoft_client_id diff --git a/surfsense_web/content/docs/connectors/notion.mdx b/surfsense_web/content/docs/connectors/notion.mdx index ca5856340..99c95d8bd 100644 --- a/surfsense_web/content/docs/connectors/notion.mdx +++ b/surfsense_web/content/docs/connectors/notion.mdx @@ -91,7 +91,7 @@ For additional information: ## Running SurfSense with Notion Connector -Add the Notion credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Notion credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash NOTION_OAUTH_CLIENT_ID=your_notion_client_id diff --git a/surfsense_web/content/docs/connectors/slack.mdx b/surfsense_web/content/docs/connectors/slack.mdx index af38487cc..f68d6e875 100644 --- a/surfsense_web/content/docs/connectors/slack.mdx +++ b/surfsense_web/content/docs/connectors/slack.mdx @@ -80,7 +80,7 @@ Click **"Add an OAuth Scope"** to add each scope. ## Running SurfSense with Slack Connector -Add the Slack credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): +Add the Slack credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): ```bash SLACK_CLIENT_ID=your_slack_client_id diff --git a/surfsense_web/content/docs/docker-installation.mdx b/surfsense_web/content/docs/docker-installation.mdx deleted file mode 100644 index 043405609..000000000 --- a/surfsense_web/content/docs/docker-installation.mdx +++ /dev/null @@ -1,301 +0,0 @@ ---- -title: Docker Installation -description: Setting up SurfSense using Docker -icon: Container ---- - -This guide explains how to run SurfSense using Docker, with options ranging from a single-command install to a fully manual setup. - -## Quick Start - -### Option 1 — Install Script (recommended) - -Downloads the compose files, generates a `SECRET_KEY`, starts all services, and sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) for automatic daily updates. - -**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running. - -#### For Linux/macOS users: - -```bash -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -``` - -#### For Windows users (PowerShell): - -```powershell -irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex -``` - -This creates a `./surfsense/` directory with `docker-compose.yml` and `.env`, then runs `docker compose up -d`. - -To skip Watchtower (e.g. in production where you manage updates yourself): - -```bash -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower -``` - -To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`. - -### Option 2 — Manual Docker Compose - -```bash -git clone https://github.com/MODSetter/SurfSense.git -cd SurfSense/docker -cp .env.example .env -# Edit .env — at minimum set SECRET_KEY -docker compose up -d -``` - -After starting, access SurfSense at: - -- **Frontend**: [http://localhost:3000](http://localhost:3000) -- **Backend API**: [http://localhost:8000](http://localhost:8000) -- **API Docs**: [http://localhost:8000/docs](http://localhost:8000/docs) -- **Electric SQL**: [http://localhost:5133](http://localhost:5133) - ---- - -## Updating - -**Option 1 — Watchtower daemon (recommended, auto-updates every 24 h):** - -If you used the install script (Option 1 above), Watchtower is already running. No extra setup needed. - -For manual Docker Compose installs (Option 2), start Watchtower separately: - -```bash -docker run -d --name watchtower \ - --restart unless-stopped \ - -v /var/run/docker.sock:/var/run/docker.sock \ - nickfedor/watchtower \ - --label-enable \ - --interval 86400 -``` - -**Option 2 — Watchtower one-time update:** - -```bash -docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ - nickfedor/watchtower --run-once \ - --label-filter "com.docker.compose.project=surfsense" -``` - - -Use `nickfedor/watchtower`. The original `containrrr/watchtower` is no longer maintained and may fail with newer Docker versions. - - -**Option 3 — Manual:** - -```bash -cd surfsense # or SurfSense/docker if you cloned manually -docker compose pull && docker compose up -d -``` - -Database migrations are applied automatically on every startup. - ---- - -## Configuration - -All configuration lives in a single `docker/.env` file (or `surfsense/.env` if you used the install script). Copy `.env.example` to `.env` and edit the values you need. - -### Required - -| Variable | Description | -|----------|-------------| -| `SECRET_KEY` | JWT secret key. Generate with: `openssl rand -base64 32`. Auto-generated by the install script. | - -### Core Settings - -| Variable | Description | Default | -|----------|-------------|---------| -| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` | -| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` | -| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` | -| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` | -| `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` | -| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` | -| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` | - -### Ports - -| Variable | Description | Default | -|----------|-------------|---------| -| `FRONTEND_PORT` | Frontend service port | `3000` | -| `BACKEND_PORT` | Backend API service port | `8000` | -| `ELECTRIC_PORT` | Electric SQL service port | `5133` | - -### Custom Domain / Reverse Proxy - -Only set these if serving SurfSense on a real domain via a reverse proxy (Caddy, Nginx, Cloudflare Tunnel, etc.). Leave commented out for standard localhost deployments. - -| Variable | Description | -|----------|-------------| -| `NEXT_FRONTEND_URL` | Public frontend URL (e.g. `https://app.yourdomain.com`) | -| `BACKEND_URL` | Public backend URL for OAuth callbacks (e.g. `https://api.yourdomain.com`) | -| `NEXT_PUBLIC_FASTAPI_BACKEND_URL` | Backend URL used by the frontend (e.g. `https://api.yourdomain.com`) | -| `NEXT_PUBLIC_ELECTRIC_URL` | Electric SQL URL used by the frontend (e.g. `https://electric.yourdomain.com`) | - -### Database - -Defaults work out of the box. Change for security in production. - -| Variable | Description | Default | -|----------|-------------|---------| -| `DB_USER` | PostgreSQL username | `surfsense` | -| `DB_PASSWORD` | PostgreSQL password | `surfsense` | -| `DB_NAME` | PostgreSQL database name | `surfsense` | -| `DB_HOST` | PostgreSQL host | `db` | -| `DB_PORT` | PostgreSQL port | `5432` | -| `DB_SSLMODE` | SSL mode: `disable`, `require`, `verify-ca`, `verify-full` | `disable` | -| `DATABASE_URL` | Full connection URL override. Use for managed databases (RDS, Supabase, etc.) | *(built from above)* | - -### Electric SQL - -| Variable | Description | Default | -|----------|-------------|---------| -| `ELECTRIC_DB_USER` | Replication user for Electric SQL | `electric` | -| `ELECTRIC_DB_PASSWORD` | Replication password for Electric SQL | `electric_password` | -| `ELECTRIC_DATABASE_URL` | Full connection URL override for Electric. Set to `host.docker.internal` when pointing at a local Postgres instance | *(built from above)* | - -### Authentication - -| Variable | Description | -|----------|-------------| -| `GOOGLE_OAUTH_CLIENT_ID` | Google OAuth client ID (required if `AUTH_TYPE=GOOGLE`) | -| `GOOGLE_OAUTH_CLIENT_SECRET` | Google OAuth client secret (required if `AUTH_TYPE=GOOGLE`) | - -Create credentials at the [Google Cloud Console](https://console.cloud.google.com/apis/credentials). - -### External API Keys - -| Variable | Description | -|----------|-------------| -| `FIRECRAWL_API_KEY` | Firecrawl API key for web crawling | -| `UNSTRUCTURED_API_KEY` | Unstructured.io API key (required if `ETL_SERVICE=UNSTRUCTURED`) | -| `LLAMA_CLOUD_API_KEY` | LlamaCloud API key (required if `ETL_SERVICE=LLAMACLOUD`) | - -### Connector OAuth Keys - -Uncomment the connectors you want to use. Redirect URIs follow the pattern `http://localhost:8000/api/v1/auth//connector/callback`. - -| Connector | Variables | -|-----------|-----------| -| Google Drive / Gmail / Calendar | `GOOGLE_DRIVE_REDIRECT_URI`, `GOOGLE_GMAIL_REDIRECT_URI`, `GOOGLE_CALENDAR_REDIRECT_URI` | -| Notion | `NOTION_CLIENT_ID`, `NOTION_CLIENT_SECRET`, `NOTION_REDIRECT_URI` | -| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `SLACK_REDIRECT_URI` | -| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `DISCORD_REDIRECT_URI` | -| Jira & Confluence | `ATLASSIAN_CLIENT_ID`, `ATLASSIAN_CLIENT_SECRET`, `JIRA_REDIRECT_URI`, `CONFLUENCE_REDIRECT_URI` | -| Linear | `LINEAR_CLIENT_ID`, `LINEAR_CLIENT_SECRET`, `LINEAR_REDIRECT_URI` | -| ClickUp | `CLICKUP_CLIENT_ID`, `CLICKUP_CLIENT_SECRET`, `CLICKUP_REDIRECT_URI` | -| Airtable | `AIRTABLE_CLIENT_ID`, `AIRTABLE_CLIENT_SECRET`, `AIRTABLE_REDIRECT_URI` | -| Microsoft Teams | `TEAMS_CLIENT_ID`, `TEAMS_CLIENT_SECRET`, `TEAMS_REDIRECT_URI` | - -For Airtable, create an OAuth integration at the [Airtable Developer Hub](https://airtable.com/create/oauth). - -### Observability (optional) - -| Variable | Description | -|----------|-------------| -| `LANGSMITH_TRACING` | Enable LangSmith tracing (`true` / `false`) | -| `LANGSMITH_ENDPOINT` | LangSmith API endpoint | -| `LANGSMITH_API_KEY` | LangSmith API key | -| `LANGSMITH_PROJECT` | LangSmith project name | - -### Advanced (optional) - -| Variable | Description | Default | -|----------|-------------|---------| -| `SCHEDULE_CHECKER_INTERVAL` | How often to check for scheduled connector tasks (e.g. `5m`, `1h`) | `5m` | -| `RERANKERS_ENABLED` | Enable document reranking for improved search | `FALSE` | -| `RERANKERS_MODEL_NAME` | Reranker model name (e.g. `ms-marco-MiniLM-L-12-v2`) | | -| `RERANKERS_MODEL_TYPE` | Reranker model type (e.g. `flashrank`) | | -| `PAGES_LIMIT` | Max pages per user for ETL services | unlimited | - ---- - -## Docker Services - -| Service | Description | -|---------|-------------| -| `db` | PostgreSQL with pgvector extension | -| `redis` | Message broker for Celery | -| `backend` | FastAPI application server | -| `celery_worker` | Background task processing (document indexing, etc.) | -| `celery_beat` | Periodic task scheduler (connector sync) | -| `electric` | Electric SQL — real-time sync for the frontend | -| `frontend` | Next.js web application | - -All services start automatically with `docker compose up -d`. - -The backend includes a health check — dependent services (workers, frontend) wait until the API is fully ready before starting. You can monitor startup progress with `docker compose ps` (look for `(health: starting)` → `(healthy)`). - ---- - -## Development Compose File - -If you're contributing to SurfSense and want to build from source, use `docker-compose.dev.yml` instead: - -```bash -cd SurfSense/docker -docker compose -f docker-compose.dev.yml up --build -``` - -This file builds the backend and frontend from your local source code (instead of pulling prebuilt images) and includes pgAdmin for database inspection at [http://localhost:5050](http://localhost:5050). Use the production `docker-compose.yml` for all other cases. - -The following `.env` variables are **only used by the dev compose file** (they have no effect on the production `docker-compose.yml`): - -| Variable | Description | Default | -|----------|-------------|---------| -| `PGADMIN_PORT` | pgAdmin web UI port | `5050` | -| `PGADMIN_DEFAULT_EMAIL` | pgAdmin login email | `admin@surfsense.com` | -| `PGADMIN_DEFAULT_PASSWORD` | pgAdmin login password | `surfsense` | -| `REDIS_PORT` | Exposed Redis port (internal-only in prod) | `6379` | -| `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` | Frontend build arg for auth type | `LOCAL` | -| `NEXT_PUBLIC_ETL_SERVICE` | Frontend build arg for ETL service | `DOCLING` | -| `NEXT_PUBLIC_DEPLOYMENT_MODE` | Frontend build arg for deployment mode | `self-hosted` | -| `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | Frontend build arg for Electric auth | `insecure` | - -In the production compose file, the `NEXT_PUBLIC_*` frontend variables are automatically derived from `AUTH_TYPE`, `ETL_SERVICE`, and the port settings. In the dev compose file, they are passed as build args since the frontend is built from source. - ---- - -## Migrating from the All-in-One Container - - -If you were previously using `docker-compose.quickstart.yml` (the legacy all-in-one `surfsense` container), your data lives in a `surfsense-data` volume and requires a **one-time migration** before switching to the current setup. PostgreSQL has been upgraded from version 14 to 17, so a simple volume swap will not work. - -See the full step-by-step guide: [Migrate from the All-in-One Container](/docs/how-to/migrate-from-allinone). - - ---- - -## Useful Commands - -```bash -# View logs (all services) -docker compose logs -f - -# View logs for a specific service -docker compose logs -f backend -docker compose logs -f electric - -# Stop all services -docker compose down - -# Restart a specific service -docker compose restart backend - -# Stop and remove all containers + volumes (destructive!) -docker compose down -v -``` - ---- - -## Troubleshooting - -- **Ports already in use** — Change the relevant `*_PORT` variable in `.env` and restart. -- **Permission errors on Linux** — You may need to prefix `docker` commands with `sudo`. -- **Electric SQL not connecting** — Check `docker compose logs electric`. If it shows `domain does not exist: db`, ensure `ELECTRIC_DATABASE_URL` is not set to a stale value in `.env`. -- **Real-time updates not working in browser** — Open DevTools → Console and look for `[Electric]` errors. Check that `NEXT_PUBLIC_ELECTRIC_URL` matches the running Electric SQL address. -- **Line ending issues on Windows** — Run `git config --global core.autocrlf true` before cloning. diff --git a/surfsense_web/content/docs/docker-installation/dev-compose.mdx b/surfsense_web/content/docs/docker-installation/dev-compose.mdx new file mode 100644 index 000000000..19b76eb7d --- /dev/null +++ b/surfsense_web/content/docs/docker-installation/dev-compose.mdx @@ -0,0 +1,30 @@ +--- +title: Development Compose +description: Building SurfSense from source using docker-compose.dev.yml +--- + +If you're contributing to SurfSense and want to build from source, use `docker-compose.dev.yml` instead: + +```bash +cd SurfSense/docker +docker compose -f docker-compose.dev.yml up --build +``` + +This file builds the backend and frontend from your local source code (instead of pulling prebuilt images) and includes pgAdmin for database inspection at [http://localhost:5050](http://localhost:5050). Use the production `docker-compose.yml` for all other cases. + +## Dev-Only Environment Variables + +The following `.env` variables are **only used by the dev compose file** (they have no effect on the production `docker-compose.yml`): + +| Variable | Description | Default | +|----------|-------------|---------| +| `PGADMIN_PORT` | pgAdmin web UI port | `5050` | +| `PGADMIN_DEFAULT_EMAIL` | pgAdmin login email | `admin@surfsense.com` | +| `PGADMIN_DEFAULT_PASSWORD` | pgAdmin login password | `surfsense` | +| `REDIS_PORT` | Exposed Redis port (internal-only in prod) | `6379` | +| `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` | Frontend build arg for auth type | `LOCAL` | +| `NEXT_PUBLIC_ETL_SERVICE` | Frontend build arg for ETL service | `DOCLING` | +| `NEXT_PUBLIC_DEPLOYMENT_MODE` | Frontend build arg for deployment mode | `self-hosted` | +| `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | Frontend build arg for Electric auth | `insecure` | + +In the production compose file, the `NEXT_PUBLIC_*` frontend variables are automatically derived from `AUTH_TYPE`, `ETL_SERVICE`, and the port settings. In the dev compose file, they are passed as build args since the frontend is built from source. diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx new file mode 100644 index 000000000..c56f08106 --- /dev/null +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -0,0 +1,188 @@ +--- +title: Docker Compose +description: Manual Docker Compose setup for SurfSense +--- + +## Setup + +```bash +git clone https://github.com/MODSetter/SurfSense.git +cd SurfSense/docker +cp .env.example .env +# Edit .env, at minimum set SECRET_KEY +docker compose up -d +``` + +After starting, access SurfSense at: + +- **Frontend**: [http://localhost:3929](http://localhost:3929) +- **Backend API**: [http://localhost:8929](http://localhost:8929) +- **API Docs**: [http://localhost:8929/docs](http://localhost:8929/docs) +- **Electric SQL**: [http://localhost:5929](http://localhost:5929) + +--- + +## Configuration + +All configuration lives in a single `docker/.env` file (or `surfsense/.env` if you used the install script). Copy `.env.example` to `.env` and edit the values you need. + +### Required + +| Variable | Description | +|----------|-------------| +| `SECRET_KEY` | JWT secret key. Generate with: `openssl rand -base64 32`. Auto-generated by the install script. | + +### Core Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` | +| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` | +| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` | +| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` | +| `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` | +| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` | +| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` | + +### Ports + +| Variable | Description | Default | +|----------|-------------|---------| +| `FRONTEND_PORT` | Frontend service port | `3929` | +| `BACKEND_PORT` | Backend API service port | `8929` | +| `ELECTRIC_PORT` | Electric SQL service port | `5929` | + +### Custom Domain / Reverse Proxy + +Only set these if serving SurfSense on a real domain via a reverse proxy (Caddy, Nginx, Cloudflare Tunnel, etc.). Leave commented out for standard localhost deployments. + +| Variable | Description | +|----------|-------------| +| `NEXT_FRONTEND_URL` | Public frontend URL (e.g. `https://app.yourdomain.com`) | +| `BACKEND_URL` | Public backend URL for OAuth callbacks (e.g. `https://api.yourdomain.com`) | +| `NEXT_PUBLIC_FASTAPI_BACKEND_URL` | Backend URL used by the frontend (e.g. `https://api.yourdomain.com`) | +| `NEXT_PUBLIC_ELECTRIC_URL` | Electric SQL URL used by the frontend (e.g. `https://electric.yourdomain.com`) | + +### Database + +Defaults work out of the box. Change for security in production. + +| Variable | Description | Default | +|----------|-------------|---------| +| `DB_USER` | PostgreSQL username | `surfsense` | +| `DB_PASSWORD` | PostgreSQL password | `surfsense` | +| `DB_NAME` | PostgreSQL database name | `surfsense` | +| `DB_HOST` | PostgreSQL host | `db` | +| `DB_PORT` | PostgreSQL port | `5432` | +| `DB_SSLMODE` | SSL mode: `disable`, `require`, `verify-ca`, `verify-full` | `disable` | +| `DATABASE_URL` | Full connection URL override. Use for managed databases (RDS, Supabase, etc.) | *(built from above)* | + +### Electric SQL + +| Variable | Description | Default | +|----------|-------------|---------| +| `ELECTRIC_DB_USER` | Replication user for Electric SQL | `electric` | +| `ELECTRIC_DB_PASSWORD` | Replication password for Electric SQL | `electric_password` | +| `ELECTRIC_DATABASE_URL` | Full connection URL override for Electric. Set to `host.docker.internal` when pointing at a local Postgres instance | *(built from above)* | + +### Authentication + +| Variable | Description | +|----------|-------------| +| `GOOGLE_OAUTH_CLIENT_ID` | Google OAuth client ID (required if `AUTH_TYPE=GOOGLE`) | +| `GOOGLE_OAUTH_CLIENT_SECRET` | Google OAuth client secret (required if `AUTH_TYPE=GOOGLE`) | + +Create credentials at the [Google Cloud Console](https://console.cloud.google.com/apis/credentials). + +### External API Keys + +| Variable | Description | +|----------|-------------| +| `FIRECRAWL_API_KEY` | [Firecrawl](https://www.firecrawl.dev/) API key for web crawling | +| `UNSTRUCTURED_API_KEY` | [Unstructured.io](https://unstructured.io/) API key (required if `ETL_SERVICE=UNSTRUCTURED`) | +| `LLAMA_CLOUD_API_KEY` | [LlamaCloud](https://cloud.llamaindex.ai/) API key (required if `ETL_SERVICE=LLAMACLOUD`) | + +### Connector OAuth Keys + +Uncomment the connectors you want to use. Redirect URIs follow the pattern `http://localhost:8000/api/v1/auth//connector/callback`. + +| Connector | Variables | +|-----------|-----------| +| Google Drive / Gmail / Calendar | `GOOGLE_DRIVE_REDIRECT_URI`, `GOOGLE_GMAIL_REDIRECT_URI`, `GOOGLE_CALENDAR_REDIRECT_URI` | +| Notion | `NOTION_CLIENT_ID`, `NOTION_CLIENT_SECRET`, `NOTION_REDIRECT_URI` | +| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `SLACK_REDIRECT_URI` | +| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `DISCORD_REDIRECT_URI` | +| Jira & Confluence | `ATLASSIAN_CLIENT_ID`, `ATLASSIAN_CLIENT_SECRET`, `JIRA_REDIRECT_URI`, `CONFLUENCE_REDIRECT_URI` | +| Linear | `LINEAR_CLIENT_ID`, `LINEAR_CLIENT_SECRET`, `LINEAR_REDIRECT_URI` | +| ClickUp | `CLICKUP_CLIENT_ID`, `CLICKUP_CLIENT_SECRET`, `CLICKUP_REDIRECT_URI` | +| Airtable | `AIRTABLE_CLIENT_ID`, `AIRTABLE_CLIENT_SECRET`, `AIRTABLE_REDIRECT_URI` | +| Microsoft Teams | `TEAMS_CLIENT_ID`, `TEAMS_CLIENT_SECRET`, `TEAMS_REDIRECT_URI` | + +### Observability (optional) + +| Variable | Description | +|----------|-------------| +| `LANGSMITH_TRACING` | Enable LangSmith tracing (`true` / `false`) | +| `LANGSMITH_ENDPOINT` | LangSmith API endpoint | +| `LANGSMITH_API_KEY` | LangSmith API key | +| `LANGSMITH_PROJECT` | LangSmith project name | + +### Advanced (optional) + +| Variable | Description | Default | +|----------|-------------|---------| +| `SCHEDULE_CHECKER_INTERVAL` | How often to check for scheduled connector tasks (e.g. `5m`, `1h`) | `5m` | +| `RERANKERS_ENABLED` | Enable document reranking for improved search | `FALSE` | +| `RERANKERS_MODEL_NAME` | Reranker model name (e.g. `ms-marco-MiniLM-L-12-v2`) | | +| `RERANKERS_MODEL_TYPE` | Reranker model type (e.g. `flashrank`) | | +| `PAGES_LIMIT` | Max pages per user for ETL services | unlimited | + +--- + +## Docker Services + +| Service | Description | +|---------|-------------| +| `db` | PostgreSQL with pgvector extension | +| `redis` | Message broker for Celery | +| `backend` | FastAPI application server | +| `celery_worker` | Background task processing (document indexing, etc.) | +| `celery_beat` | Periodic task scheduler (connector sync) | +| `electric` | Electric SQL (real-time sync for the frontend) | +| `frontend` | Next.js web application | + +All services start automatically with `docker compose up -d`. + +The backend includes a health check. Dependent services (workers, frontend) wait until the API is fully ready before starting. You can monitor startup progress with `docker compose ps` (look for `(health: starting)` → `(healthy)`). + +--- + +## Useful Commands + +```bash +# View logs (all services) +docker compose logs -f + +# View logs for a specific service +docker compose logs -f backend +docker compose logs -f electric + +# Stop all services +docker compose down + +# Restart a specific service +docker compose restart backend + +# Stop and remove all containers + volumes (destructive!) +docker compose down -v +``` + +--- + +## Troubleshooting + +- **Ports already in use**: Change the relevant `*_PORT` variable in `.env` and restart. +- **Permission errors on Linux**: You may need to prefix `docker` commands with `sudo`. +- **Electric SQL not connecting**: Check `docker compose logs electric`. If it shows `domain does not exist: db`, ensure `ELECTRIC_DATABASE_URL` is not set to a stale value in `.env`. +- **Real-time updates not working in browser**: Open DevTools → Console and look for `[Electric]` errors. Check that `NEXT_PUBLIC_ELECTRIC_URL` matches the running Electric SQL address. +- **Line ending issues on Windows**: Run `git config --global core.autocrlf true` before cloning. diff --git a/surfsense_web/content/docs/docker-installation/install-script.mdx b/surfsense_web/content/docs/docker-installation/install-script.mdx new file mode 100644 index 000000000..d68938a3f --- /dev/null +++ b/surfsense_web/content/docs/docker-installation/install-script.mdx @@ -0,0 +1,41 @@ +--- +title: Install Script +description: One-command installation of SurfSense using Docker +--- + +Downloads the compose files, generates a `SECRET_KEY`, starts all services, and sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) for automatic daily updates. + +**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running. + +### For Linux/macOS users: + +```bash +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash +``` + +### For Windows users (PowerShell): + +```powershell +irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex +``` + +This creates a `./surfsense/` directory with `docker-compose.yml` and `.env`, then runs `docker compose up -d`. + +To skip Watchtower (e.g. in production where you manage updates yourself): + +```bash +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower +``` + +To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`. + +--- + +## Access SurfSense + +After starting, access SurfSense at: + +- **Frontend**: [http://localhost:3929](http://localhost:3929) +- **Backend API**: [http://localhost:8929](http://localhost:8929) +- **API Docs**: [http://localhost:8929/docs](http://localhost:8929/docs) +- **Electric SQL**: [http://localhost:5929](http://localhost:5929) diff --git a/surfsense_web/content/docs/docker-installation/meta.json b/surfsense_web/content/docs/docker-installation/meta.json new file mode 100644 index 000000000..13683547b --- /dev/null +++ b/surfsense_web/content/docs/docker-installation/meta.json @@ -0,0 +1,6 @@ +{ + "title": "Docker Installation", + "pages": ["install-script", "docker-compose", "updating", "dev-compose", "migrate-from-allinone"], + "icon": "Container", + "defaultOpen": false +} diff --git a/surfsense_web/content/docs/how-to/migrate-from-allinone.mdx b/surfsense_web/content/docs/docker-installation/migrate-from-allinone.mdx similarity index 100% rename from surfsense_web/content/docs/how-to/migrate-from-allinone.mdx rename to surfsense_web/content/docs/docker-installation/migrate-from-allinone.mdx diff --git a/surfsense_web/content/docs/docker-installation/updating.mdx b/surfsense_web/content/docs/docker-installation/updating.mdx new file mode 100644 index 000000000..6ef2fcecc --- /dev/null +++ b/surfsense_web/content/docs/docker-installation/updating.mdx @@ -0,0 +1,50 @@ +--- +title: Updating +description: How to update your SurfSense Docker deployment +--- + +## Watchtower Daemon (recommended) + +Auto-updates every 24 hours. If you used the [install script](/docs/docker-installation/install-script), Watchtower is already running. No extra setup needed. + +For [manual Docker Compose](/docs/docker-installation/docker-compose) installs, start Watchtower separately: + +```bash +docker run -d --name watchtower \ + --restart unless-stopped \ + -v /var/run/docker.sock:/var/run/docker.sock \ + nickfedor/watchtower \ + --label-enable \ + --interval 86400 +``` + +## Watchtower One-Time Update + +```bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ + nickfedor/watchtower --run-once \ + --label-filter "com.docker.compose.project=surfsense" +``` + + +Use `nickfedor/watchtower`. The original `containrrr/watchtower` is no longer maintained and may fail with newer Docker versions. + + +## Manual Update + +```bash +cd surfsense # or SurfSense/docker if you cloned manually +docker compose pull && docker compose up -d +``` + +Database migrations are applied automatically on every startup. + +--- + +## Migrating from the All-in-One Container + + +If you were previously using `docker-compose.quickstart.yml` (the legacy all-in-one `surfsense` container), your data lives in a `surfsense-data` volume and requires a **one-time migration** before switching to the current setup. PostgreSQL has been upgraded from version 14 to 17, so a simple volume swap will not work. + +See the full step-by-step guide: [Migrate from the All-in-One Container](/docs/docker-installation/migrate-from-allinone). + diff --git a/surfsense_web/content/docs/how-to/electric-sql.mdx b/surfsense_web/content/docs/how-to/electric-sql.mdx index fb2cf941a..f051a9ab5 100644 --- a/surfsense_web/content/docs/how-to/electric-sql.mdx +++ b/surfsense_web/content/docs/how-to/electric-sql.mdx @@ -5,7 +5,7 @@ description: Setting up Electric SQL for real-time data synchronization in SurfS [Electric SQL](https://electric-sql.com/) enables real-time data synchronization in SurfSense, providing instant updates for inbox items, document indexing status, and connector sync progress without manual refresh. The frontend uses [PGlite](https://pglite.dev/) (a lightweight PostgreSQL in the browser) to maintain a local database that syncs with the backend via Electric SQL. -## What Does Electric SQL Do? +## What does Electric SQL do? When you index documents or receive inbox updates, Electric SQL pushes updates to your browser in real-time. The data flows like this: @@ -23,45 +23,24 @@ This means: ## Docker Setup -The `docker-compose.yml` includes the Electric SQL service. It is pre-configured to connect to the Docker-managed `db` container out of the box. +- The `docker-compose.yml` includes the Electric SQL service, pre-configured to connect to the Docker-managed `db` container. +- No additional configuration is required. Electric SQL works with the Docker PostgreSQL instance out of the box. -```bash -docker compose up -d -``` +## Manual Setup (Development Only) -The Electric SQL service configuration in `docker-compose.yml`: - -```yaml -electric: - image: electricsql/electric:1.4.6 - ports: - - "${ELECTRIC_PORT:-5133}:3000" - environment: - DATABASE_URL: ${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}} - ELECTRIC_INSECURE: "true" - ELECTRIC_WRITE_TO_PG_MODE: direct - depends_on: - db: - condition: service_healthy -``` - -No additional configuration is required — Electric SQL is pre-configured to work with the Docker PostgreSQL instance. - -## Manual Setup - -Follow the steps below based on your PostgreSQL setup. +This section is intended for local development environments. Follow the steps below based on your PostgreSQL setup. ### Step 1: Configure Environment Variables Ensure your environment files are configured. If you haven't set up SurfSense yet, follow the [Manual Installation Guide](/docs/manual-installation) first. -For Electric SQL, verify these variables are set in `docker/.env`: +For Electric SQL, verify these variables are set: + +**Backend (`surfsense_backend/.env`):** ```bash -ELECTRIC_PORT=5133 ELECTRIC_DB_USER=electric ELECTRIC_DB_PASSWORD=electric_password -NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133 ``` **Frontend (`surfsense_web/.env`):** @@ -71,17 +50,19 @@ NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133 NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure ``` +Next, choose the option that matches your PostgreSQL setup: + --- ### Option A: Using Docker PostgreSQL -If you're using the Docker-managed PostgreSQL instance, no extra configuration is needed. Just start the services: +If you're using the Docker-managed PostgreSQL instance, no extra configuration is needed. Just start the services using the development compose file (which exposes the PostgreSQL port to your host machine): ```bash -docker compose up -d db electric +docker compose -f docker-compose.dev.yml up -d db electric ``` -Then run the database migration and start the backend: +Then run the database migration, start the backend, and launch the frontend: ```bash cd surfsense_backend @@ -89,6 +70,13 @@ uv run alembic upgrade head uv run main.py ``` +In a separate terminal, start the frontend: + +```bash +cd surfsense_web +pnpm run dev +``` + Electric SQL is now configured and connected to your Docker PostgreSQL database. --- @@ -148,7 +136,7 @@ ELECTRIC_DATABASE_URL=postgresql://electric:electric_password@host.docker.intern **4. Start Electric SQL only (skip the Docker `db` container):** ```bash -docker compose up -d --no-deps electric +docker compose -f docker-compose.dev.yml up -d --no-deps electric ``` The `--no-deps` flag starts only the `electric` service without starting the Docker-managed `db` container. @@ -161,18 +149,32 @@ uv run alembic upgrade head uv run main.py ``` +In a separate terminal, start the frontend: + +```bash +cd surfsense_web +pnpm run dev +``` + Electric SQL is now configured and connected to your local PostgreSQL database. ## Environment Variables Reference +**Required for manual setup:** + | Variable | Location | Description | Default | |----------|----------|-------------|---------| -| `ELECTRIC_PORT` | `docker/.env` | Port to expose Electric SQL | `5133` | -| `ELECTRIC_DB_USER` | `docker/.env` | Database user for Electric replication | `electric` | -| `ELECTRIC_DB_PASSWORD` | `docker/.env` | Database password for Electric replication | `electric_password` | -| `ELECTRIC_DATABASE_URL` | `docker/.env` | Full connection URL override for Electric. Set to use `host.docker.internal` when pointing at a local Postgres instance | *(built from above defaults)* | -| `NEXT_PUBLIC_ELECTRIC_URL` | Frontend `.env` | Electric SQL server URL (PGlite connects to this) | `http://localhost:5133` | -| `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | Frontend `.env` | Authentication mode (`insecure` for dev, `secure` for production) | `insecure` | +| `ELECTRIC_DB_USER` | `surfsense_backend/.env` | Database user for Electric replication | `electric` | +| `ELECTRIC_DB_PASSWORD` | `surfsense_backend/.env` | Database password for Electric replication | `electric_password` | +| `NEXT_PUBLIC_ELECTRIC_URL` | `surfsense_web/.env` | Electric SQL server URL (PGlite connects to this) | `http://localhost:5133` | +| `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | `surfsense_web/.env` | Authentication mode (`insecure` for dev, `secure` for production) | `insecure` | + +**Optional / Docker-only:** + +| Variable | Location | Description | Default | +|----------|----------|-------------|---------| +| `ELECTRIC_PORT` | `docker/.env` | Port to expose Electric SQL on the host | `5133` (dev), `5929` (production) | +| `ELECTRIC_DATABASE_URL` | `docker/.env` | Full connection URL override for Electric. Only needed for Option B (local Postgres via `host.docker.internal`) | *(built from above defaults)* | ## Verify Setup diff --git a/surfsense_web/content/docs/how-to/meta.json b/surfsense_web/content/docs/how-to/meta.json index c8ecb05d9..aeb1bc3b3 100644 --- a/surfsense_web/content/docs/how-to/meta.json +++ b/surfsense_web/content/docs/how-to/meta.json @@ -1,6 +1,6 @@ { "title": "How to", - "pages": ["electric-sql", "realtime-collaboration", "migrate-from-allinone"], + "pages": ["electric-sql", "realtime-collaboration"], "icon": "BookOpen", "defaultOpen": false } diff --git a/surfsense_web/content/docs/installation.mdx b/surfsense_web/content/docs/installation.mdx index 6aa2eeb90..aa3a2a72d 100644 --- a/surfsense_web/content/docs/installation.mdx +++ b/surfsense_web/content/docs/installation.mdx @@ -12,7 +12,7 @@ There are two ways to install SurfSense, but both require the repository to be c This method provides a containerized environment with all dependencies pre-configured. Less Customization. -[Learn more about Docker installation](/docs/docker-installation) +[Learn more about Docker installation](/docs/docker-installation/install-script) ## Manual Installation (Preferred)