diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6fef9b20e..340e5b51a 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -17,10 +17,12 @@ REDIS_APP_URL=redis://localhost:6379/0 # Telegram Gateway # TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, _ or - +# GATEWAY_TELEGRAM_INTAKE_MODE: webhook for production, longpoll for single-replica self-host fallback, disabled to skip Telegram intake TELEGRAM_SHARED_BOT_TOKEN= TELEGRAM_SHARED_BOT_USERNAME= TELEGRAM_WEBHOOK_SECRET= GATEWAY_BASE_URL=http://localhost:8000 +GATEWAY_TELEGRAM_INTAKE_MODE=webhook # Platform Web Search (SearXNG) # Set this to enable built-in web search. Docker Compose sets it automatically. diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 89bf4c925..c77b95fde 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -546,9 +546,13 @@ class Config: TELEGRAM_SHARED_BOT_USERNAME = os.getenv("TELEGRAM_SHARED_BOT_USERNAME") TELEGRAM_WEBHOOK_SECRET = os.getenv("TELEGRAM_WEBHOOK_SECRET") GATEWAY_BASE_URL = os.getenv("GATEWAY_BASE_URL", BACKEND_URL) - GATEWAY_BYO_LONGPOLL_ENABLED = ( - os.getenv("GATEWAY_BYO_LONGPOLL_ENABLED", "TRUE").upper() == "TRUE" - ) + GATEWAY_TELEGRAM_INTAKE_MODE = os.getenv( + "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook" + ).lower() + if GATEWAY_TELEGRAM_INTAKE_MODE not in {"webhook", "longpoll", "disabled"}: + raise ValueError( + "GATEWAY_TELEGRAM_INTAKE_MODE must be one of: webhook, longpoll, disabled" + ) # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") diff --git a/surfsense_backend/app/gateway/byo_long_poll.py b/surfsense_backend/app/gateway/byo_long_poll.py index d02f19f95..0be448ae3 100644 --- a/surfsense_backend/app/gateway/byo_long_poll.py +++ b/surfsense_backend/app/gateway/byo_long_poll.py @@ -46,7 +46,7 @@ async def start_byo_long_poll_supervisors() -> None: """Start one BYO long-poll supervisor per active non-system Telegram account.""" global _shutdown_event - if not config.GATEWAY_BYO_LONGPOLL_ENABLED: + if config.GATEWAY_TELEGRAM_INTAKE_MODE != "longpoll": return if _tasks: return diff --git a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py index b8212ec9a..951c2d124 100644 --- a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py +++ b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py @@ -38,8 +38,8 @@ async def cleanup_supervisors(): @pytest.mark.asyncio -async def test_start_byo_long_poll_noops_when_flag_off(monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", False) +async def test_start_byo_long_poll_noops_when_mode_is_webhook(monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook") await byo_long_poll.start_byo_long_poll_supervisors() @@ -48,7 +48,7 @@ async def test_start_byo_long_poll_noops_when_flag_off(monkeypatch): @pytest.mark.asyncio async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") session = mocker.AsyncMock() session.execute.return_value = ScalarResult([]) monkeypatch.setattr( @@ -64,7 +64,7 @@ async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatc @pytest.mark.asyncio async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") accounts = [mocker.Mock(id=1), mocker.Mock(id=2)] session = mocker.AsyncMock() session.execute.return_value = ScalarResult(accounts) @@ -108,7 +108,7 @@ async def test_supervisor_retries_after_run_returns(mocker, monkeypatch): @pytest.mark.asyncio async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") session = mocker.AsyncMock() session.execute.return_value = ScalarResult([mocker.Mock(id=1)]) monkeypatch.setattr( diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 599cb6238..f977ef11d 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -39,6 +39,39 @@ Complete all the [setup steps](/docs), including: The backend is the core of SurfSense. Follow these steps to set it up: +### Optional: Telegram External Chat Surface + +SurfSense can expose the same canonical chat agent through Telegram. The `external_chat_*` tables store adapter identity, delivery configuration, and durable inbox rows. The actual chat thread and messages remain in `new_chat_threads` and `new_chat_messages`, and all chat-message sources are eligible for Zero replication so a future SurfSense UI layer can render external chat surfaces. The web app initially shows pairing, health, revoke, and resume controls under **User Settings > Messaging Channels**. + +Add these variables to `surfsense_backend/.env` when enabling the Telegram surface: + +```bash +TELEGRAM_SHARED_BOT_TOKEN=123456:bot-token-from-botfather +TELEGRAM_SHARED_BOT_USERNAME=your_bot_username +TELEGRAM_WEBHOOK_SECRET=generate-a-long-random-secret +GATEWAY_BASE_URL=https://api.example.com +GATEWAY_TELEGRAM_INTAKE_MODE=webhook +``` + +`GATEWAY_TELEGRAM_INTAKE_MODE` must be `webhook`, `longpoll`, or `disabled`. Use `webhook` for production/SaaS deployments, `longpoll` only for single-replica self-host installs that cannot expose a public HTTPS webhook, and `disabled` to skip Telegram intake. `TELEGRAM_WEBHOOK_SECRET` must use only `A-Z`, `a-z`, `0-9`, `_`, and `-` characters. `REDIS_APP_URL` is reused for external chat rate limits and per-thread locks. The webhook URL shape is: + +```text +${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/telegram/{account_id} +``` + +After deploying the backend, register the webhook: + +```bash +cd surfsense_backend +uv run python scripts/register_webhook.py +``` + +Keep the FastAPI backend, Celery worker, and Celery beat running. Telegram webhooks write inbound updates into `external_chat_inbound_events`. The FastAPI process owns external chat inbox processing and runs the same SurfSense agent used by web UI chats, then replies back to Telegram. Celery remains maintenance-only for external chat reconciliation, health checks, and retention sweeps. There is no separate gateway service, `SERVICE_ROLE=gateway` process, or Celery agent-processing path. + +For self-hosted BYO Telegram bots without a public HTTPS URL, set `GATEWAY_TELEGRAM_INTAKE_MODE=longpoll`. The FastAPI process starts one lifespan long-poll supervisor per non-system Telegram account and writes updates into the same durable inbox. The FastAPI inbox worker then processes those rows in-process through the canonical `new_chat_*` surface. This fallback is intended for single-replica self-hosted installs. For SaaS-style multi-replica deployments, prefer public webhooks and keep `GATEWAY_TELEGRAM_INTAKE_MODE=webhook` so API replicas skip BYO polling entirely. Telegram does not allow `get_updates()` while a webhook is active, so delete any existing webhook for a BYO bot before relying on long polling. + +When upgrading from an older gateway-runner deployment, apply the rewritten migration 144 external chat schema, deploy the new backend, worker, and beat images, then stop the old `gateway` service. Wait about 30 seconds for any old Telegram `getUpdates` long-poll request to release its advisory lock before starting the new API process. Register each webhook again with the account-id URL above and the per-account `webhook_secret`. If you roll back before using the migration in production, restore the old image and downgrade the schema first. + ### 1. Environment Configuration First, create and configure your environment variables by copying the example file: @@ -350,7 +383,7 @@ redis-cli ping ### 6. Start Celery Worker -In a new terminal window, start the Celery worker to handle background tasks: +In a new terminal window, start the Celery worker to handle background tasks. For external chat surfaces, Celery only runs maintenance tasks; agent turns run inside the FastAPI process. **If using uv:** @@ -358,9 +391,9 @@ In a new terminal window, start the Celery worker to handle background tasks: # Make sure you're in the surfsense_backend directory cd surfsense_backend -# Start Celery worker (consume both default and connectors queues) +# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors" +uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` **If using pip/venv:** @@ -374,9 +407,9 @@ source .venv/bin/activate # Linux/macOS # OR .venv\Scripts\activate # Windows -# Start Celery worker (consume both default and connectors queues) +# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors" +celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` **Optional: Start Flower for monitoring Celery tasks:**