From f44cb8a493f0d36f85a53a6944c44afa1872c528 Mon Sep 17 00:00:00 2001 From: pelazas Date: Sat, 4 Jul 2026 09:53:51 +0200 Subject: [PATCH 1/2] fix: dispatch periodic BookStack indexing from the scheduler task_map BOOKSTACK_CONNECTOR was missing from check_periodic_schedules' task_map, so connectors with periodic indexing enabled fell through to the "No task found" warning branch every minute and never synced. Fixes #891 Co-Authored-By: Claude Fable 5 --- .../celery_tasks/schedule_checker_task.py | 2 + .../tests/unit/tasks/celery_tasks/__init__.py | 0 .../test_schedule_checker_task.py | 129 ++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py create mode 100644 surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py diff --git a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py index e88fb58b9..5f088e4d8 100644 --- a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py @@ -48,6 +48,7 @@ async def _check_and_trigger_schedules(): # Live connectors (Linear, Slack, Jira, ClickUp, Airtable, Discord, # Teams, Gmail, Calendar, Luma) use real-time tools instead. from app.tasks.celery_tasks.connector_tasks import ( + index_bookstack_pages_task, index_confluence_pages_task, index_crawled_urls_task, index_elasticsearch_documents_task, @@ -58,6 +59,7 @@ async def _check_and_trigger_schedules(): task_map = { SearchSourceConnectorType.NOTION_CONNECTOR: index_notion_pages_task, + SearchSourceConnectorType.BOOKSTACK_CONNECTOR: index_bookstack_pages_task, SearchSourceConnectorType.GITHUB_CONNECTOR: index_github_repos_task, SearchSourceConnectorType.CONFLUENCE_CONNECTOR: index_confluence_pages_task, SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: index_elasticsearch_documents_task, diff --git a/surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py b/surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py b/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py new file mode 100644 index 000000000..4bca74135 --- /dev/null +++ b/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py @@ -0,0 +1,129 @@ +"""Unit tests for the periodic schedule checker's connector-to-task dispatch.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from app.db import SearchSourceConnectorType +from app.tasks.celery_tasks import schedule_checker_task + +pytestmark = pytest.mark.unit + + +class _FakeScalars: + def __init__(self, connectors): + self._connectors = connectors + + def all(self): + return self._connectors + + +class _FakeDueConnectorsResult: + def __init__(self, connectors): + self._connectors = connectors + + def scalars(self): + return _FakeScalars(self._connectors) + + +class _FakeEmptyResult: + def first(self): + return None + + +class _FakeSession: + """Session stub: first execute() returns due connectors, later ones no rows.""" + + def __init__(self, connectors): + self._results = [_FakeDueConnectorsResult(connectors)] + self.commits = 0 + + async def execute(self, _query): + if self._results: + return self._results.pop(0) + return _FakeEmptyResult() + + async def commit(self): + self.commits += 1 + + async def rollback(self): + pass + + +def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace: + return SimpleNamespace( + id=42, + connector_type=connector_type, + search_space_id=7, + user_id="00000000-0000-0000-0000-000000000001", + config={}, + periodic_indexing_enabled=True, + indexing_frequency_minutes=60, + next_scheduled_at=datetime.now(UTC) - timedelta(minutes=5), + ) + + +async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespace): + session = _FakeSession([connector]) + + @asynccontextmanager + async def _session_ctx(): + yield session + + monkeypatch.setattr( + schedule_checker_task, "get_celery_session_maker", lambda: _session_ctx + ) + monkeypatch.setattr( + schedule_checker_task, "is_connector_indexing_locked", lambda _id: False + ) + await schedule_checker_task._check_and_trigger_schedules() + return session + + +@pytest.mark.asyncio +async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch): + """A due BookStack connector must dispatch index_bookstack_pages_task. + + Regression test for the connector type missing from the scheduler's + task_map, which made periodic BookStack syncs silently no-op with only a + "No task found" warning. + """ + from app.tasks.celery_tasks import connector_tasks + + task_mock = MagicMock() + monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock) + + connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR) + session = await _run_checker(monkeypatch, connector) + + task_mock.delay.assert_called_once_with( + connector.id, + connector.search_space_id, + str(connector.user_id), + None, + None, + ) + # The next run must be rescheduled, otherwise the connector stays "due" + # and is re-examined every minute. + assert connector.next_scheduled_at > datetime.now(UTC) + assert session.commits == 1 + + +@pytest.mark.asyncio +async def test_unmapped_connector_type_does_not_dispatch(monkeypatch): + """Connector types absent from task_map are skipped without dispatching.""" + from app.tasks.celery_tasks import connector_tasks + + task_mock = MagicMock() + monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock) + + connector = _due_connector(SearchSourceConnectorType.TAVILY_API) + session = await _run_checker(monkeypatch, connector) + + task_mock.delay.assert_not_called() + assert session.commits == 0 From 6fafedca6a95f4fcfaa202d417379d3e61f2c908 Mon Sep 17 00:00:00 2001 From: Dustin Persek Date: Sun, 5 Jul 2026 22:33:09 -0400 Subject: [PATCH 2/2] Support separate embedding base URL --- docker/.env.example | 5 ++ surfsense_backend/.env.example | 4 + surfsense_backend/app/config/__init__.py | 14 ++-- .../app/config/embedding_settings.py | 48 ++++++++++++ .../unit/config/test_embedding_settings.py | 76 +++++++++++++++++++ .../docker-installation/docker-compose.mdx | 1 + .../content/docs/local-models/ollama.mdx | 17 +++++ .../content/docs/manual-installation.mdx | 1 + 8 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 surfsense_backend/app/config/embedding_settings.py create mode 100644 surfsense_backend/tests/unit/config/test_embedding_settings.py diff --git a/docker/.env.example b/docker/.env.example index 18142c614..df8fcbfba 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -48,7 +48,12 @@ ETL_SERVICE=DOCLING # Local: sentence-transformers/all-MiniLM-L6-v2 # OpenAI: openai://text-embedding-ada-002 (set OPENAI_API_KEY below) # Cohere: cohere://embed-english-light-v3.0 (set COHERE_API_KEY below) +# Ollama or OpenAI-compatible embedding endpoint: +# EMBEDDING_MODEL=litellm://ollama/nomic-embed-text +# EMBEDDING_BASE_URL=http://host.docker.internal:11434 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 +# EMBEDDING_BASE_URL= +# OLLAMA_EMBEDDING_BASE_URL= # ------------------------------------------------------------------------------ # How You Access SurfSense diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index a1d410eef..5bf888167 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -214,6 +214,10 @@ COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callb # # Get Cohere embeddings # embeddings = AutoEmbeddings.get_embeddings("cohere://embed-english-light-v3.0", api_key="...") EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 +# Optional: use a separate endpoint for Chonkie/LiteLLM embedding models, for +# example EMBEDDING_MODEL=litellm://ollama/nomic-embed-text. +# EMBEDDING_BASE_URL=http://host.docker.internal:11434 +# OLLAMA_EMBEDDING_BASE_URL=http://host.docker.internal:11434 # Rerankers Config RERANKERS_ENABLED=TRUE or FALSE(Default: FALSE) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 405718f0c..fc7c42311 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -9,6 +9,11 @@ from chonkie import AutoEmbeddings, CodeChunker, RecursiveChunker from dotenv import load_dotenv from rerankers import Reranker +from app.config.embedding_settings import ( + build_embedding_kwargs, + resolve_embedding_base_url, +) + # Get the base directory of the project BASE_DIR = Path(__file__).resolve().parent.parent.parent @@ -876,16 +881,13 @@ class Config: # Chonkie Configuration | Edit this to your needs EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL") + EMBEDDING_BASE_URL = resolve_embedding_base_url() # Azure OpenAI credentials from environment variables AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT") AZURE_OPENAI_API_KEY = os.getenv("AZURE_OPENAI_API_KEY") - # Pass Azure credentials to embeddings when using Azure OpenAI - embedding_kwargs = {} - if AZURE_OPENAI_ENDPOINT: - embedding_kwargs["azure_endpoint"] = AZURE_OPENAI_ENDPOINT - if AZURE_OPENAI_API_KEY: - embedding_kwargs["azure_api_key"] = AZURE_OPENAI_API_KEY + # Pass provider-specific settings to embeddings when supported. + embedding_kwargs = build_embedding_kwargs(embedding_model=EMBEDDING_MODEL) embedding_model_instance = AutoEmbeddings.get_embeddings( EMBEDDING_MODEL, diff --git a/surfsense_backend/app/config/embedding_settings.py b/surfsense_backend/app/config/embedding_settings.py new file mode 100644 index 000000000..571ada346 --- /dev/null +++ b/surfsense_backend/app/config/embedding_settings.py @@ -0,0 +1,48 @@ +import os +from collections.abc import Mapping + +EMBEDDING_BASE_URL_ENV = "EMBEDDING_BASE_URL" +OLLAMA_EMBEDDING_BASE_URL_ENV = "OLLAMA_EMBEDDING_BASE_URL" + + +def _clean_env_value(value: str | None) -> str | None: + if value is None: + return None + stripped = value.strip() + return stripped or None + + +def resolve_embedding_base_url(environ: Mapping[str, str] | None = None) -> str | None: + """Return the configured embedding endpoint, if any.""" + environ = os.environ if environ is None else environ + return _clean_env_value(environ.get(EMBEDDING_BASE_URL_ENV)) or _clean_env_value( + environ.get(OLLAMA_EMBEDDING_BASE_URL_ENV) + ) + + +def _supports_embedding_api_base(embedding_model: str | None) -> bool: + return (embedding_model or "").startswith("litellm://") + + +def build_embedding_kwargs( + environ: Mapping[str, str] | None = None, + *, + embedding_model: str | None = None, +) -> dict[str, str]: + """Build keyword arguments for Chonkie's embedding provider.""" + environ = os.environ if environ is None else environ + + embedding_kwargs: dict[str, str] = {} + embedding_base_url = resolve_embedding_base_url(environ) + if embedding_base_url and _supports_embedding_api_base(embedding_model): + embedding_kwargs["api_base"] = embedding_base_url + + azure_openai_endpoint = _clean_env_value(environ.get("AZURE_OPENAI_ENDPOINT")) + azure_openai_api_key = _clean_env_value(environ.get("AZURE_OPENAI_API_KEY")) + + if azure_openai_endpoint: + embedding_kwargs["azure_endpoint"] = azure_openai_endpoint + if azure_openai_api_key: + embedding_kwargs["azure_api_key"] = azure_openai_api_key + + return embedding_kwargs diff --git a/surfsense_backend/tests/unit/config/test_embedding_settings.py b/surfsense_backend/tests/unit/config/test_embedding_settings.py new file mode 100644 index 000000000..7483f9457 --- /dev/null +++ b/surfsense_backend/tests/unit/config/test_embedding_settings.py @@ -0,0 +1,76 @@ +import pytest + +from app.config.embedding_settings import ( + build_embedding_kwargs, + resolve_embedding_base_url, +) + +pytestmark = pytest.mark.unit + + +def test_resolve_embedding_base_url_prefers_generic_value() -> None: + environ = { + "EMBEDDING_BASE_URL": " http://embed-host:11434 ", + "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434", + } + + assert resolve_embedding_base_url(environ) == "http://embed-host:11434" + + +def test_resolve_embedding_base_url_falls_back_to_ollama_specific_value() -> None: + environ = { + "EMBEDDING_BASE_URL": " ", + "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434", + } + + assert resolve_embedding_base_url(environ) == "http://ollama-embed:11434" + + +def test_build_embedding_kwargs_maps_base_url_to_litellm_api_base() -> None: + kwargs = build_embedding_kwargs( + {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"}, + embedding_model="litellm://ollama/nomic-embed-text", + ) + + assert kwargs == {"api_base": "http://host.docker.internal:11435"} + + +def test_build_embedding_kwargs_does_not_leak_api_base_to_other_providers() -> None: + kwargs = build_embedding_kwargs( + {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"}, + embedding_model="cohere://embed-english-light-v3.0", + ) + + assert kwargs == {} + + +def test_build_embedding_kwargs_preserves_azure_settings() -> None: + kwargs = build_embedding_kwargs( + { + "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com", + "AZURE_OPENAI_API_KEY": "test-key", + }, + embedding_model="azure_openai://text-embedding-3-small", + ) + + assert kwargs == { + "azure_endpoint": "https://example.openai.azure.com", + "azure_api_key": "test-key", + } + + +def test_build_embedding_kwargs_combines_litellm_and_azure_env_when_set() -> None: + kwargs = build_embedding_kwargs( + { + "EMBEDDING_BASE_URL": "http://host.docker.internal:4000/v1", + "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com", + "AZURE_OPENAI_API_KEY": "test-key", + }, + embedding_model="litellm://openai/text-embedding-3-small", + ) + + assert kwargs == { + "api_base": "http://host.docker.internal:4000/v1", + "azure_endpoint": "https://example.openai.azure.com", + "azure_api_key": "test-key", + } diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index bf71c077b..d9d713334 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -39,6 +39,7 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y | `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` | | `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` | | `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` | +| `EMBEDDING_BASE_URL` | Optional separate endpoint for Chonkie/LiteLLM embedding models. Use with values like `EMBEDDING_MODEL=litellm://ollama/nomic-embed-text`. `OLLAMA_EMBEDDING_BASE_URL` is also accepted as an Ollama-specific fallback. | *(empty)* | | `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` | | `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` | | `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` | diff --git a/surfsense_web/content/docs/local-models/ollama.mdx b/surfsense_web/content/docs/local-models/ollama.mdx index b062d98b0..ec4c50638 100644 --- a/surfsense_web/content/docs/local-models/ollama.mdx +++ b/surfsense_web/content/docs/local-models/ollama.mdx @@ -50,6 +50,23 @@ http://:11434 Replace `` with the LAN IP or domain for that machine. +## Embeddings on a Separate Ollama Server + +Search-space model connections configure chat and completion models. The global +embedding model is configured in the backend environment. + +Use Chonkie's LiteLLM embedding provider when embeddings run on Ollama: + +```dotenv +EMBEDDING_MODEL=litellm://ollama/nomic-embed-text +EMBEDDING_BASE_URL=http://host.docker.internal:11434 +``` + +If chat and embeddings run on different Ollama instances, keep the chat model +connection pointed at the chat server and set `EMBEDDING_BASE_URL` to the +embedding server. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an +Ollama-specific fallback when `EMBEDDING_BASE_URL` is not set. + ## Add the Connection 1. Open Search Space Settings. diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index ab09b4155..b115fa03d 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -85,6 +85,7 @@ Edit the `.env` file and set the following variables: | GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | | GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | | EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) | +| EMBEDDING_BASE_URL | (Optional) Separate endpoint for Chonkie/LiteLLM embedding models, such as `http://localhost:11434` with `EMBEDDING_MODEL=litellm://ollama/nomic-embed-text`. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an Ollama-specific fallback. | | RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) | | RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) | | RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) (required if RERANKERS_ENABLED=TRUE) |