Merge upstream/dev into feature/tiktok-scraper

Sync with the advanced dev tip so the PR compares against current dev
(drops already-merged commits from the diff). No history rewritten.

docker/.env.example

@@ -48,7 +48,12 @@ ETL_SERVICE=DOCLING
 # Local: sentence-transformers/all-MiniLM-L6-v2
 # OpenAI: openai://text-embedding-ada-002 (set OPENAI_API_KEY below)
 # Cohere: cohere://embed-english-light-v3.0 (set COHERE_API_KEY below)
+# Ollama or OpenAI-compatible embedding endpoint:
+#   EMBEDDING_MODEL=litellm://ollama/nomic-embed-text
+#   EMBEDDING_BASE_URL=http://host.docker.internal:11434
 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+# EMBEDDING_BASE_URL=
+# OLLAMA_EMBEDDING_BASE_URL=
 
 # ------------------------------------------------------------------------------
 # How You Access SurfSense

surfsense_backend/.env.example

@@ -209,6 +209,10 @@ COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callb
 #     # Get Cohere embeddings
 #     embeddings = AutoEmbeddings.get_embeddings("cohere://embed-english-light-v3.0", api_key="...")
 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+# Optional: use a separate endpoint for Chonkie/LiteLLM embedding models, for
+# example EMBEDDING_MODEL=litellm://ollama/nomic-embed-text.
+# EMBEDDING_BASE_URL=http://host.docker.internal:11434
+# OLLAMA_EMBEDDING_BASE_URL=http://host.docker.internal:11434
 
 # Rerankers Config
 RERANKERS_ENABLED=TRUE or FALSE(Default: FALSE)

surfsense_backend/app/config/__init__.py

@@ -9,6 +9,11 @@ from chonkie import AutoEmbeddings, CodeChunker, RecursiveChunker
 from dotenv import load_dotenv
 from rerankers import Reranker
 
+from app.config.embedding_settings import (
+    build_embedding_kwargs,
+    resolve_embedding_base_url,
+)
+
 # Get the base directory of the project
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
 
@@ -946,16 +951,13 @@ class Config:
 
     # Chonkie Configuration | Edit this to your needs
     EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
+    EMBEDDING_BASE_URL = resolve_embedding_base_url()
     # Azure OpenAI credentials from environment variables
     AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
     AZURE_OPENAI_API_KEY = os.getenv("AZURE_OPENAI_API_KEY")
 
-    # Pass Azure credentials to embeddings when using Azure OpenAI
-    embedding_kwargs = {}
-    if AZURE_OPENAI_ENDPOINT:
-        embedding_kwargs["azure_endpoint"] = AZURE_OPENAI_ENDPOINT
-    if AZURE_OPENAI_API_KEY:
-        embedding_kwargs["azure_api_key"] = AZURE_OPENAI_API_KEY
+    # Pass provider-specific settings to embeddings when supported.
+    embedding_kwargs = build_embedding_kwargs(embedding_model=EMBEDDING_MODEL)
 
     embedding_model_instance = AutoEmbeddings.get_embeddings(
         EMBEDDING_MODEL,

surfsense_backend/app/config/embedding_settings.py

@@ -0,0 +1,48 @@
+import os
+from collections.abc import Mapping
+
+EMBEDDING_BASE_URL_ENV = "EMBEDDING_BASE_URL"
+OLLAMA_EMBEDDING_BASE_URL_ENV = "OLLAMA_EMBEDDING_BASE_URL"
+
+
+def _clean_env_value(value: str | None) -> str | None:
+    if value is None:
+        return None
+    stripped = value.strip()
+    return stripped or None
+
+
+def resolve_embedding_base_url(environ: Mapping[str, str] | None = None) -> str | None:
+    """Return the configured embedding endpoint, if any."""
+    environ = os.environ if environ is None else environ
+    return _clean_env_value(environ.get(EMBEDDING_BASE_URL_ENV)) or _clean_env_value(
+        environ.get(OLLAMA_EMBEDDING_BASE_URL_ENV)
+    )
+
+
+def _supports_embedding_api_base(embedding_model: str | None) -> bool:
+    return (embedding_model or "").startswith("litellm://")
+
+
+def build_embedding_kwargs(
+    environ: Mapping[str, str] | None = None,
+    *,
+    embedding_model: str | None = None,
+) -> dict[str, str]:
+    """Build keyword arguments for Chonkie's embedding provider."""
+    environ = os.environ if environ is None else environ
+
+    embedding_kwargs: dict[str, str] = {}
+    embedding_base_url = resolve_embedding_base_url(environ)
+    if embedding_base_url and _supports_embedding_api_base(embedding_model):
+        embedding_kwargs["api_base"] = embedding_base_url
+
+    azure_openai_endpoint = _clean_env_value(environ.get("AZURE_OPENAI_ENDPOINT"))
+    azure_openai_api_key = _clean_env_value(environ.get("AZURE_OPENAI_API_KEY"))
+
+    if azure_openai_endpoint:
+        embedding_kwargs["azure_endpoint"] = azure_openai_endpoint
+    if azure_openai_api_key:
+        embedding_kwargs["azure_api_key"] = azure_openai_api_key
+
+    return embedding_kwargs

surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py

@@ -48,6 +48,7 @@ async def _check_and_trigger_schedules():
             # Live connectors (Linear, Slack, Jira, ClickUp, Airtable, Discord,
             # Teams, Gmail, Calendar, Luma) use real-time tools instead.
             from app.tasks.celery_tasks.connector_tasks import (
+                index_bookstack_pages_task,
                 index_confluence_pages_task,
                 index_elasticsearch_documents_task,
                 index_github_repos_task,
@@ -57,6 +58,7 @@ async def _check_and_trigger_schedules():
 
             task_map = {
                 SearchSourceConnectorType.NOTION_CONNECTOR: index_notion_pages_task,
+                SearchSourceConnectorType.BOOKSTACK_CONNECTOR: index_bookstack_pages_task,
                 SearchSourceConnectorType.GITHUB_CONNECTOR: index_github_repos_task,
                 SearchSourceConnectorType.CONFLUENCE_CONNECTOR: index_confluence_pages_task,
                 SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: index_elasticsearch_documents_task,

surfsense_backend/tests/unit/config/test_embedding_settings.py

@@ -0,0 +1,76 @@
+import pytest
+
+from app.config.embedding_settings import (
+    build_embedding_kwargs,
+    resolve_embedding_base_url,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_resolve_embedding_base_url_prefers_generic_value() -> None:
+    environ = {
+        "EMBEDDING_BASE_URL": " http://embed-host:11434 ",
+        "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434",
+    }
+
+    assert resolve_embedding_base_url(environ) == "http://embed-host:11434"
+
+
+def test_resolve_embedding_base_url_falls_back_to_ollama_specific_value() -> None:
+    environ = {
+        "EMBEDDING_BASE_URL": " ",
+        "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434",
+    }
+
+    assert resolve_embedding_base_url(environ) == "http://ollama-embed:11434"
+
+
+def test_build_embedding_kwargs_maps_base_url_to_litellm_api_base() -> None:
+    kwargs = build_embedding_kwargs(
+        {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"},
+        embedding_model="litellm://ollama/nomic-embed-text",
+    )
+
+    assert kwargs == {"api_base": "http://host.docker.internal:11435"}
+
+
+def test_build_embedding_kwargs_does_not_leak_api_base_to_other_providers() -> None:
+    kwargs = build_embedding_kwargs(
+        {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"},
+        embedding_model="cohere://embed-english-light-v3.0",
+    )
+
+    assert kwargs == {}
+
+
+def test_build_embedding_kwargs_preserves_azure_settings() -> None:
+    kwargs = build_embedding_kwargs(
+        {
+            "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com",
+            "AZURE_OPENAI_API_KEY": "test-key",
+        },
+        embedding_model="azure_openai://text-embedding-3-small",
+    )
+
+    assert kwargs == {
+        "azure_endpoint": "https://example.openai.azure.com",
+        "azure_api_key": "test-key",
+    }
+
+
+def test_build_embedding_kwargs_combines_litellm_and_azure_env_when_set() -> None:
+    kwargs = build_embedding_kwargs(
+        {
+            "EMBEDDING_BASE_URL": "http://host.docker.internal:4000/v1",
+            "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com",
+            "AZURE_OPENAI_API_KEY": "test-key",
+        },
+        embedding_model="litellm://openai/text-embedding-3-small",
+    )
+
+    assert kwargs == {
+        "api_base": "http://host.docker.internal:4000/v1",
+        "azure_endpoint": "https://example.openai.azure.com",
+        "azure_api_key": "test-key",
+    }

surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py

@@ -0,0 +1,129 @@
+"""Unit tests for the periodic schedule checker's connector-to-task dispatch."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from app.db import SearchSourceConnectorType
+from app.tasks.celery_tasks import schedule_checker_task
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScalars:
+    def __init__(self, connectors):
+        self._connectors = connectors
+
+    def all(self):
+        return self._connectors
+
+
+class _FakeDueConnectorsResult:
+    def __init__(self, connectors):
+        self._connectors = connectors
+
+    def scalars(self):
+        return _FakeScalars(self._connectors)
+
+
+class _FakeEmptyResult:
+    def first(self):
+        return None
+
+
+class _FakeSession:
+    """Session stub: first execute() returns due connectors, later ones no rows."""
+
+    def __init__(self, connectors):
+        self._results = [_FakeDueConnectorsResult(connectors)]
+        self.commits = 0
+
+    async def execute(self, _query):
+        if self._results:
+            return self._results.pop(0)
+        return _FakeEmptyResult()
+
+    async def commit(self):
+        self.commits += 1
+
+    async def rollback(self):
+        pass
+
+
+def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace:
+    return SimpleNamespace(
+        id=42,
+        connector_type=connector_type,
+        search_space_id=7,
+        user_id="00000000-0000-0000-0000-000000000001",
+        config={},
+        periodic_indexing_enabled=True,
+        indexing_frequency_minutes=60,
+        next_scheduled_at=datetime.now(UTC) - timedelta(minutes=5),
+    )
+
+
+async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespace):
+    session = _FakeSession([connector])
+
+    @asynccontextmanager
+    async def _session_ctx():
+        yield session
+
+    monkeypatch.setattr(
+        schedule_checker_task, "get_celery_session_maker", lambda: _session_ctx
+    )
+    monkeypatch.setattr(
+        schedule_checker_task, "is_connector_indexing_locked", lambda _id: False
+    )
+    await schedule_checker_task._check_and_trigger_schedules()
+    return session
+
+
+@pytest.mark.asyncio
+async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
+    """A due BookStack connector must dispatch index_bookstack_pages_task.
+
+    Regression test for the connector type missing from the scheduler's
+    task_map, which made periodic BookStack syncs silently no-op with only a
+    "No task found" warning.
+    """
+    from app.tasks.celery_tasks import connector_tasks
+
+    task_mock = MagicMock()
+    monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock)
+
+    connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR)
+    session = await _run_checker(monkeypatch, connector)
+
+    task_mock.delay.assert_called_once_with(
+        connector.id,
+        connector.search_space_id,
+        str(connector.user_id),
+        None,
+        None,
+    )
+    # The next run must be rescheduled, otherwise the connector stays "due"
+    # and is re-examined every minute.
+    assert connector.next_scheduled_at > datetime.now(UTC)
+    assert session.commits == 1
+
+
+@pytest.mark.asyncio
+async def test_unmapped_connector_type_does_not_dispatch(monkeypatch):
+    """Connector types absent from task_map are skipped without dispatching."""
+    from app.tasks.celery_tasks import connector_tasks
+
+    task_mock = MagicMock()
+    monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock)
+
+    connector = _due_connector(SearchSourceConnectorType.TAVILY_API)
+    session = await _run_checker(monkeypatch, connector)
+
+    task_mock.delay.assert_not_called()
+    assert session.commits == 0

surfsense_web/content/docs/docker-installation/index.mdx

@@ -59,6 +59,7 @@ The defaults give you local email/password auth, local document parsing with Doc
 
 - **Authentication** — switch to Google OAuth login.
 - **Document parsing** — switch to Unstructured or LlamaCloud (both need API keys).
+- **Embedding endpoint** — set `EMBEDDING_BASE_URL` for Chonkie/LiteLLM embedding models, or `OLLAMA_EMBEDDING_BASE_URL` as an Ollama-specific fallback.
 - **Connector credentials** — OAuth apps for [external connectors](/docs/connectors/external) that need them on self-hosted deployments.
 - **Messaging channels** — Telegram, WhatsApp, Slack, and Discord bots (see [Messaging Channels](/docs/messaging-channels)).
 

surfsense_web/content/docs/local-models/ollama.mdx

@@ -50,6 +50,23 @@ http://<host>:11434
 
 Replace `<host>` with the LAN IP or domain for that machine.
 
+## Embeddings on a Separate Ollama Server
+
+Search-space model connections configure chat and completion models. The global
+embedding model is configured in the backend environment.
+
+Use Chonkie's LiteLLM embedding provider when embeddings run on Ollama:
+
+```dotenv
+EMBEDDING_MODEL=litellm://ollama/nomic-embed-text
+EMBEDDING_BASE_URL=http://host.docker.internal:11434
+```
+
+If chat and embeddings run on different Ollama instances, keep the chat model
+connection pointed at the chat server and set `EMBEDDING_BASE_URL` to the
+embedding server. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an
+Ollama-specific fallback when `EMBEDDING_BASE_URL` is not set.
+
 ## Add the Connection
 
 1. Open Workspace Settings.

surfsense_web/content/docs/manual-installation.mdx

@@ -57,6 +57,8 @@ Copy-Item -Path .env.example -Destination .env
 
 `.env.example` is the source of truth for configuration — every variable is documented inline with comments and sensible defaults. At minimum, set your PostgreSQL connection string and a JWT secret key (generate one with `openssl rand -base64 32`). Everything else — auth type, ETL service, embeddings, TTS/STT, connector credentials — is optional and explained in the file itself.
 
+For separate embedding servers, set `EMBEDDING_BASE_URL` with Chonkie/LiteLLM embedding models; `OLLAMA_EMBEDDING_BASE_URL` is also supported as an Ollama-specific fallback.
+
 ### 2. Install Dependencies
 
 ```bash