From fb423560d5aee09e5cebbb56b83a8ff413f00901 Mon Sep 17 00:00:00 2001
From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Date: Wed, 24 Jun 2026 01:57:36 -0700
Subject: [PATCH 001/160] fix: wire video_presentation fan-in as a barrier so
generate_slide_scene_codes fires once
---
.../app/agents/video_presentation/graph.py | 7 +-
.../agents/test_video_presentation_graph.py | 144 ++++++++++++++++++
2 files changed, 148 insertions(+), 3 deletions(-)
create mode 100644 surfsense_backend/tests/unit/agents/test_video_presentation_graph.py
diff --git a/surfsense_backend/app/agents/video_presentation/graph.py b/surfsense_backend/app/agents/video_presentation/graph.py
index 1d87bcd76..5aed3d3d1 100644
--- a/surfsense_backend/app/agents/video_presentation/graph.py
+++ b/surfsense_backend/app/agents/video_presentation/graph.py
@@ -24,9 +24,10 @@ def build_graph():
workflow.add_edge("create_presentation_slides", "create_slide_audio")
workflow.add_edge("create_presentation_slides", "assign_slide_themes")
- # Fan-in: scene code generation waits for both audio and themes.
- workflow.add_edge("create_slide_audio", "generate_slide_scene_codes")
- workflow.add_edge("assign_slide_themes", "generate_slide_scene_codes")
+ # Fan-in: wait for BOTH audio and themes (barrier).
+ workflow.add_edge(
+ ["create_slide_audio", "assign_slide_themes"], "generate_slide_scene_codes"
+ )
workflow.add_edge("generate_slide_scene_codes", "__end__")
diff --git a/surfsense_backend/tests/unit/agents/test_video_presentation_graph.py b/surfsense_backend/tests/unit/agents/test_video_presentation_graph.py
new file mode 100644
index 000000000..3283af782
--- /dev/null
+++ b/surfsense_backend/tests/unit/agents/test_video_presentation_graph.py
@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import importlib
+import operator
+import sys
+import types
+from typing import Annotated, Any
+
+import pytest
+from langgraph.graph import END, START, StateGraph
+from typing_extensions import TypedDict
+
+pytestmark = pytest.mark.unit
+
+_GRAPH_MODULE = "app.agents.video_presentation.graph"
+_NODES_MODULE = "app.agents.video_presentation.nodes"
+_TARGET_NODE = "generate_slide_scene_codes"
+_BARRIER_SOURCES = ("create_slide_audio", "assign_slide_themes")
+
+
+class _DelayedState(TypedDict, total=False):
+ calls: Annotated[list[str], operator.add]
+ audio_ready: bool
+ themes_ready: bool
+
+
+def _config(thread_id: str = "video-presentation-graph-test") -> dict[str, Any]:
+ return {
+ "configurable": {
+ "search_space_id": 1,
+ "thread_id": thread_id,
+ "video_title": "Test deck",
+ }
+ }
+
+
+def _input_state() -> dict[str, Any]:
+ return {"db_session": None, "source_content": "source material"}
+
+
+def _build_video_graph(monkeypatch: pytest.MonkeyPatch, calls: list[str]):
+ nodes = types.ModuleType(_NODES_MODULE)
+
+ async def create_presentation_slides(_state, config):
+ _ = config
+ calls.append("create_presentation_slides")
+ return {"slides": [{"slide_number": 1, "title": "Intro"}]}
+
+ async def create_slide_audio(_state, config):
+ _ = config
+ calls.append("create_slide_audio")
+ return {"slide_audio_results": [{"slide_number": 1}]}
+
+ async def assign_slide_themes(_state, config):
+ _ = config
+ calls.append("assign_slide_themes")
+ return {"slide_theme_assignments": {1: ("corporate", "light")}}
+
+ async def generate_slide_scene_codes(state, config):
+ _ = config
+ calls.append(_TARGET_NODE)
+ assert state.slide_audio_results == [{"slide_number": 1}]
+ assert state.slide_theme_assignments == {1: ("corporate", "light")}
+ return {"slide_scene_codes": [{"slide_number": 1, "code": "", "title": ""}]}
+
+ nodes.create_presentation_slides = create_presentation_slides
+ nodes.create_slide_audio = create_slide_audio
+ nodes.assign_slide_themes = assign_slide_themes
+ nodes.generate_slide_scene_codes = generate_slide_scene_codes
+
+ monkeypatch.setitem(sys.modules, _NODES_MODULE, nodes)
+ monkeypatch.delitem(sys.modules, _GRAPH_MODULE, raising=False)
+
+ graph_module = importlib.import_module(_GRAPH_MODULE)
+ return graph_module.build_graph()
+
+
+@pytest.mark.asyncio
+async def test_video_presentation_graph_generates_scene_codes_once(monkeypatch):
+ calls: list[str] = []
+ graph = _build_video_graph(monkeypatch, calls)
+
+ await graph.ainvoke(_input_state(), _config())
+
+ assert calls.count(_TARGET_NODE) == 1
+ scene_index = calls.index(_TARGET_NODE)
+ assert calls.index("create_slide_audio") < scene_index
+ assert calls.index("assign_slide_themes") < scene_index
+
+
+def test_video_presentation_graph_registers_single_barrier_trigger(monkeypatch):
+ graph = _build_video_graph(monkeypatch, [])
+
+ assert graph.builder.waiting_edges == {(_BARRIER_SOURCES, _TARGET_NODE)}
+ assert not {
+ edge
+ for edge in graph.builder.edges
+ if edge[0] in _BARRIER_SOURCES and edge[1] == _TARGET_NODE
+ }
+
+ join_channel = f"join:{'+'.join(_BARRIER_SOURCES)}:{_TARGET_NODE}"
+ assert join_channel in graph.channels
+ assert graph.nodes[_TARGET_NODE].triggers.count(join_channel) == 1
+
+
+@pytest.mark.asyncio
+async def test_barrier_fires_once_when_one_branch_finishes_a_superstep_later():
+ def create_presentation_slides(_state):
+ return {"calls": ["create_presentation_slides"]}
+
+ def create_slide_audio(_state):
+ return {"calls": ["create_slide_audio"]}
+
+ def finish_slide_audio(_state):
+ return {"calls": ["finish_slide_audio"], "audio_ready": True}
+
+ def assign_slide_themes(_state):
+ return {"calls": ["assign_slide_themes"], "themes_ready": True}
+
+ def generate_slide_scene_codes(state):
+ assert state["audio_ready"] is True
+ assert state["themes_ready"] is True
+ return {"calls": [_TARGET_NODE]}
+
+ workflow = StateGraph(_DelayedState)
+ workflow.add_node("create_presentation_slides", create_presentation_slides)
+ workflow.add_node("create_slide_audio", create_slide_audio)
+ workflow.add_node("finish_slide_audio", finish_slide_audio)
+ workflow.add_node("assign_slide_themes", assign_slide_themes)
+ workflow.add_node(_TARGET_NODE, generate_slide_scene_codes)
+
+ workflow.add_edge(START, "create_presentation_slides")
+ workflow.add_edge("create_presentation_slides", "create_slide_audio")
+ workflow.add_edge("create_presentation_slides", "assign_slide_themes")
+ workflow.add_edge("create_slide_audio", "finish_slide_audio")
+ workflow.add_edge(["finish_slide_audio", "assign_slide_themes"], _TARGET_NODE)
+ workflow.add_edge(_TARGET_NODE, END)
+
+ result = await workflow.compile().ainvoke({"calls": []})
+
+ calls = result["calls"]
+ assert calls.count(_TARGET_NODE) == 1
+ assert calls.index("assign_slide_themes") < calls.index(_TARGET_NODE)
+ assert calls.index("finish_slide_audio") < calls.index(_TARGET_NODE)
From f44cb8a493f0d36f85a53a6944c44afa1872c528 Mon Sep 17 00:00:00 2001
From: pelazas
Date: Sat, 4 Jul 2026 09:53:51 +0200
Subject: [PATCH 002/160] fix: dispatch periodic BookStack indexing from the
scheduler task_map
BOOKSTACK_CONNECTOR was missing from check_periodic_schedules' task_map,
so connectors with periodic indexing enabled fell through to the
"No task found" warning branch every minute and never synced.
Fixes #891
Co-Authored-By: Claude Fable 5
---
.../celery_tasks/schedule_checker_task.py | 2 +
.../tests/unit/tasks/celery_tasks/__init__.py | 0
.../test_schedule_checker_task.py | 129 ++++++++++++++++++
3 files changed, 131 insertions(+)
create mode 100644 surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py
create mode 100644 surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py
diff --git a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py
index e88fb58b9..5f088e4d8 100644
--- a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py
+++ b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py
@@ -48,6 +48,7 @@ async def _check_and_trigger_schedules():
# Live connectors (Linear, Slack, Jira, ClickUp, Airtable, Discord,
# Teams, Gmail, Calendar, Luma) use real-time tools instead.
from app.tasks.celery_tasks.connector_tasks import (
+ index_bookstack_pages_task,
index_confluence_pages_task,
index_crawled_urls_task,
index_elasticsearch_documents_task,
@@ -58,6 +59,7 @@ async def _check_and_trigger_schedules():
task_map = {
SearchSourceConnectorType.NOTION_CONNECTOR: index_notion_pages_task,
+ SearchSourceConnectorType.BOOKSTACK_CONNECTOR: index_bookstack_pages_task,
SearchSourceConnectorType.GITHUB_CONNECTOR: index_github_repos_task,
SearchSourceConnectorType.CONFLUENCE_CONNECTOR: index_confluence_pages_task,
SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: index_elasticsearch_documents_task,
diff --git a/surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py b/surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py b/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py
new file mode 100644
index 000000000..4bca74135
--- /dev/null
+++ b/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py
@@ -0,0 +1,129 @@
+"""Unit tests for the periodic schedule checker's connector-to-task dispatch."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from app.db import SearchSourceConnectorType
+from app.tasks.celery_tasks import schedule_checker_task
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScalars:
+ def __init__(self, connectors):
+ self._connectors = connectors
+
+ def all(self):
+ return self._connectors
+
+
+class _FakeDueConnectorsResult:
+ def __init__(self, connectors):
+ self._connectors = connectors
+
+ def scalars(self):
+ return _FakeScalars(self._connectors)
+
+
+class _FakeEmptyResult:
+ def first(self):
+ return None
+
+
+class _FakeSession:
+ """Session stub: first execute() returns due connectors, later ones no rows."""
+
+ def __init__(self, connectors):
+ self._results = [_FakeDueConnectorsResult(connectors)]
+ self.commits = 0
+
+ async def execute(self, _query):
+ if self._results:
+ return self._results.pop(0)
+ return _FakeEmptyResult()
+
+ async def commit(self):
+ self.commits += 1
+
+ async def rollback(self):
+ pass
+
+
+def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace:
+ return SimpleNamespace(
+ id=42,
+ connector_type=connector_type,
+ search_space_id=7,
+ user_id="00000000-0000-0000-0000-000000000001",
+ config={},
+ periodic_indexing_enabled=True,
+ indexing_frequency_minutes=60,
+ next_scheduled_at=datetime.now(UTC) - timedelta(minutes=5),
+ )
+
+
+async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespace):
+ session = _FakeSession([connector])
+
+ @asynccontextmanager
+ async def _session_ctx():
+ yield session
+
+ monkeypatch.setattr(
+ schedule_checker_task, "get_celery_session_maker", lambda: _session_ctx
+ )
+ monkeypatch.setattr(
+ schedule_checker_task, "is_connector_indexing_locked", lambda _id: False
+ )
+ await schedule_checker_task._check_and_trigger_schedules()
+ return session
+
+
+@pytest.mark.asyncio
+async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch):
+ """A due BookStack connector must dispatch index_bookstack_pages_task.
+
+ Regression test for the connector type missing from the scheduler's
+ task_map, which made periodic BookStack syncs silently no-op with only a
+ "No task found" warning.
+ """
+ from app.tasks.celery_tasks import connector_tasks
+
+ task_mock = MagicMock()
+ monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock)
+
+ connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR)
+ session = await _run_checker(monkeypatch, connector)
+
+ task_mock.delay.assert_called_once_with(
+ connector.id,
+ connector.search_space_id,
+ str(connector.user_id),
+ None,
+ None,
+ )
+ # The next run must be rescheduled, otherwise the connector stays "due"
+ # and is re-examined every minute.
+ assert connector.next_scheduled_at > datetime.now(UTC)
+ assert session.commits == 1
+
+
+@pytest.mark.asyncio
+async def test_unmapped_connector_type_does_not_dispatch(monkeypatch):
+ """Connector types absent from task_map are skipped without dispatching."""
+ from app.tasks.celery_tasks import connector_tasks
+
+ task_mock = MagicMock()
+ monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock)
+
+ connector = _due_connector(SearchSourceConnectorType.TAVILY_API)
+ session = await _run_checker(monkeypatch, connector)
+
+ task_mock.delay.assert_not_called()
+ assert session.commits == 0
From 6fafedca6a95f4fcfaa202d417379d3e61f2c908 Mon Sep 17 00:00:00 2001
From: Dustin Persek
Date: Sun, 5 Jul 2026 22:33:09 -0400
Subject: [PATCH 003/160] Support separate embedding base URL
---
docker/.env.example | 5 ++
surfsense_backend/.env.example | 4 +
surfsense_backend/app/config/__init__.py | 14 ++--
.../app/config/embedding_settings.py | 48 ++++++++++++
.../unit/config/test_embedding_settings.py | 76 +++++++++++++++++++
.../docker-installation/docker-compose.mdx | 1 +
.../content/docs/local-models/ollama.mdx | 17 +++++
.../content/docs/manual-installation.mdx | 1 +
8 files changed, 160 insertions(+), 6 deletions(-)
create mode 100644 surfsense_backend/app/config/embedding_settings.py
create mode 100644 surfsense_backend/tests/unit/config/test_embedding_settings.py
diff --git a/docker/.env.example b/docker/.env.example
index 18142c614..df8fcbfba 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -48,7 +48,12 @@ ETL_SERVICE=DOCLING
# Local: sentence-transformers/all-MiniLM-L6-v2
# OpenAI: openai://text-embedding-ada-002 (set OPENAI_API_KEY below)
# Cohere: cohere://embed-english-light-v3.0 (set COHERE_API_KEY below)
+# Ollama or OpenAI-compatible embedding endpoint:
+# EMBEDDING_MODEL=litellm://ollama/nomic-embed-text
+# EMBEDDING_BASE_URL=http://host.docker.internal:11434
EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+# EMBEDDING_BASE_URL=
+# OLLAMA_EMBEDDING_BASE_URL=
# ------------------------------------------------------------------------------
# How You Access SurfSense
diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example
index a1d410eef..5bf888167 100644
--- a/surfsense_backend/.env.example
+++ b/surfsense_backend/.env.example
@@ -214,6 +214,10 @@ COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callb
# # Get Cohere embeddings
# embeddings = AutoEmbeddings.get_embeddings("cohere://embed-english-light-v3.0", api_key="...")
EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+# Optional: use a separate endpoint for Chonkie/LiteLLM embedding models, for
+# example EMBEDDING_MODEL=litellm://ollama/nomic-embed-text.
+# EMBEDDING_BASE_URL=http://host.docker.internal:11434
+# OLLAMA_EMBEDDING_BASE_URL=http://host.docker.internal:11434
# Rerankers Config
RERANKERS_ENABLED=TRUE or FALSE(Default: FALSE)
diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py
index 405718f0c..fc7c42311 100644
--- a/surfsense_backend/app/config/__init__.py
+++ b/surfsense_backend/app/config/__init__.py
@@ -9,6 +9,11 @@ from chonkie import AutoEmbeddings, CodeChunker, RecursiveChunker
from dotenv import load_dotenv
from rerankers import Reranker
+from app.config.embedding_settings import (
+ build_embedding_kwargs,
+ resolve_embedding_base_url,
+)
+
# Get the base directory of the project
BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -876,16 +881,13 @@ class Config:
# Chonkie Configuration | Edit this to your needs
EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
+ EMBEDDING_BASE_URL = resolve_embedding_base_url()
# Azure OpenAI credentials from environment variables
AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
AZURE_OPENAI_API_KEY = os.getenv("AZURE_OPENAI_API_KEY")
- # Pass Azure credentials to embeddings when using Azure OpenAI
- embedding_kwargs = {}
- if AZURE_OPENAI_ENDPOINT:
- embedding_kwargs["azure_endpoint"] = AZURE_OPENAI_ENDPOINT
- if AZURE_OPENAI_API_KEY:
- embedding_kwargs["azure_api_key"] = AZURE_OPENAI_API_KEY
+ # Pass provider-specific settings to embeddings when supported.
+ embedding_kwargs = build_embedding_kwargs(embedding_model=EMBEDDING_MODEL)
embedding_model_instance = AutoEmbeddings.get_embeddings(
EMBEDDING_MODEL,
diff --git a/surfsense_backend/app/config/embedding_settings.py b/surfsense_backend/app/config/embedding_settings.py
new file mode 100644
index 000000000..571ada346
--- /dev/null
+++ b/surfsense_backend/app/config/embedding_settings.py
@@ -0,0 +1,48 @@
+import os
+from collections.abc import Mapping
+
+EMBEDDING_BASE_URL_ENV = "EMBEDDING_BASE_URL"
+OLLAMA_EMBEDDING_BASE_URL_ENV = "OLLAMA_EMBEDDING_BASE_URL"
+
+
+def _clean_env_value(value: str | None) -> str | None:
+ if value is None:
+ return None
+ stripped = value.strip()
+ return stripped or None
+
+
+def resolve_embedding_base_url(environ: Mapping[str, str] | None = None) -> str | None:
+ """Return the configured embedding endpoint, if any."""
+ environ = os.environ if environ is None else environ
+ return _clean_env_value(environ.get(EMBEDDING_BASE_URL_ENV)) or _clean_env_value(
+ environ.get(OLLAMA_EMBEDDING_BASE_URL_ENV)
+ )
+
+
+def _supports_embedding_api_base(embedding_model: str | None) -> bool:
+ return (embedding_model or "").startswith("litellm://")
+
+
+def build_embedding_kwargs(
+ environ: Mapping[str, str] | None = None,
+ *,
+ embedding_model: str | None = None,
+) -> dict[str, str]:
+ """Build keyword arguments for Chonkie's embedding provider."""
+ environ = os.environ if environ is None else environ
+
+ embedding_kwargs: dict[str, str] = {}
+ embedding_base_url = resolve_embedding_base_url(environ)
+ if embedding_base_url and _supports_embedding_api_base(embedding_model):
+ embedding_kwargs["api_base"] = embedding_base_url
+
+ azure_openai_endpoint = _clean_env_value(environ.get("AZURE_OPENAI_ENDPOINT"))
+ azure_openai_api_key = _clean_env_value(environ.get("AZURE_OPENAI_API_KEY"))
+
+ if azure_openai_endpoint:
+ embedding_kwargs["azure_endpoint"] = azure_openai_endpoint
+ if azure_openai_api_key:
+ embedding_kwargs["azure_api_key"] = azure_openai_api_key
+
+ return embedding_kwargs
diff --git a/surfsense_backend/tests/unit/config/test_embedding_settings.py b/surfsense_backend/tests/unit/config/test_embedding_settings.py
new file mode 100644
index 000000000..7483f9457
--- /dev/null
+++ b/surfsense_backend/tests/unit/config/test_embedding_settings.py
@@ -0,0 +1,76 @@
+import pytest
+
+from app.config.embedding_settings import (
+ build_embedding_kwargs,
+ resolve_embedding_base_url,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_resolve_embedding_base_url_prefers_generic_value() -> None:
+ environ = {
+ "EMBEDDING_BASE_URL": " http://embed-host:11434 ",
+ "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434",
+ }
+
+ assert resolve_embedding_base_url(environ) == "http://embed-host:11434"
+
+
+def test_resolve_embedding_base_url_falls_back_to_ollama_specific_value() -> None:
+ environ = {
+ "EMBEDDING_BASE_URL": " ",
+ "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434",
+ }
+
+ assert resolve_embedding_base_url(environ) == "http://ollama-embed:11434"
+
+
+def test_build_embedding_kwargs_maps_base_url_to_litellm_api_base() -> None:
+ kwargs = build_embedding_kwargs(
+ {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"},
+ embedding_model="litellm://ollama/nomic-embed-text",
+ )
+
+ assert kwargs == {"api_base": "http://host.docker.internal:11435"}
+
+
+def test_build_embedding_kwargs_does_not_leak_api_base_to_other_providers() -> None:
+ kwargs = build_embedding_kwargs(
+ {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"},
+ embedding_model="cohere://embed-english-light-v3.0",
+ )
+
+ assert kwargs == {}
+
+
+def test_build_embedding_kwargs_preserves_azure_settings() -> None:
+ kwargs = build_embedding_kwargs(
+ {
+ "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com",
+ "AZURE_OPENAI_API_KEY": "test-key",
+ },
+ embedding_model="azure_openai://text-embedding-3-small",
+ )
+
+ assert kwargs == {
+ "azure_endpoint": "https://example.openai.azure.com",
+ "azure_api_key": "test-key",
+ }
+
+
+def test_build_embedding_kwargs_combines_litellm_and_azure_env_when_set() -> None:
+ kwargs = build_embedding_kwargs(
+ {
+ "EMBEDDING_BASE_URL": "http://host.docker.internal:4000/v1",
+ "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com",
+ "AZURE_OPENAI_API_KEY": "test-key",
+ },
+ embedding_model="litellm://openai/text-embedding-3-small",
+ )
+
+ assert kwargs == {
+ "api_base": "http://host.docker.internal:4000/v1",
+ "azure_endpoint": "https://example.openai.azure.com",
+ "azure_api_key": "test-key",
+ }
diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx
index bf71c077b..d9d713334 100644
--- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx
+++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx
@@ -39,6 +39,7 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y
| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` |
| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` |
| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` |
+| `EMBEDDING_BASE_URL` | Optional separate endpoint for Chonkie/LiteLLM embedding models. Use with values like `EMBEDDING_MODEL=litellm://ollama/nomic-embed-text`. `OLLAMA_EMBEDDING_BASE_URL` is also accepted as an Ollama-specific fallback. | *(empty)* |
| `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` |
| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` |
| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` |
diff --git a/surfsense_web/content/docs/local-models/ollama.mdx b/surfsense_web/content/docs/local-models/ollama.mdx
index b062d98b0..ec4c50638 100644
--- a/surfsense_web/content/docs/local-models/ollama.mdx
+++ b/surfsense_web/content/docs/local-models/ollama.mdx
@@ -50,6 +50,23 @@ http://:11434
Replace `` with the LAN IP or domain for that machine.
+## Embeddings on a Separate Ollama Server
+
+Search-space model connections configure chat and completion models. The global
+embedding model is configured in the backend environment.
+
+Use Chonkie's LiteLLM embedding provider when embeddings run on Ollama:
+
+```dotenv
+EMBEDDING_MODEL=litellm://ollama/nomic-embed-text
+EMBEDDING_BASE_URL=http://host.docker.internal:11434
+```
+
+If chat and embeddings run on different Ollama instances, keep the chat model
+connection pointed at the chat server and set `EMBEDDING_BASE_URL` to the
+embedding server. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an
+Ollama-specific fallback when `EMBEDDING_BASE_URL` is not set.
+
## Add the Connection
1. Open Search Space Settings.
diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx
index ab09b4155..b115fa03d 100644
--- a/surfsense_web/content/docs/manual-installation.mdx
+++ b/surfsense_web/content/docs/manual-installation.mdx
@@ -85,6 +85,7 @@ Edit the `.env` file and set the following variables:
| GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) |
| GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) |
| EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) |
+| EMBEDDING_BASE_URL | (Optional) Separate endpoint for Chonkie/LiteLLM embedding models, such as `http://localhost:11434` with `EMBEDDING_MODEL=litellm://ollama/nomic-embed-text`. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an Ollama-specific fallback. |
| RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) |
| RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) |
| RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) (required if RERANKERS_ENABLED=TRUE) |
From 97f6497e2a401e4879cc726f4ccdd5ef15ae108f Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 004/160] feat(mcp): add per-request API key identity auth
slice
---
.../src/surfsense_mcp/core/auth/__init__.py | 7 +++
.../src/surfsense_mcp/core/auth/headers.py | 26 +++++++++
.../src/surfsense_mcp/core/auth/identity.py | 42 +++++++++++++++
.../src/surfsense_mcp/core/auth/middleware.py | 54 +++++++++++++++++++
4 files changed, 129 insertions(+)
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/headers.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/identity.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py
new file mode 100644
index 000000000..e837c6d61
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py
@@ -0,0 +1,7 @@
+"""Per-request caller identity: header parsing, request-scoped storage, and the
+ASGI middleware that binds them together for the remote transport."""
+
+from .identity import current_api_key, current_identity
+from .middleware import ApiKeyIdentityMiddleware
+
+__all__ = ["current_api_key", "current_identity", "ApiKeyIdentityMiddleware"]
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py
new file mode 100644
index 000000000..c06989909
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py
@@ -0,0 +1,26 @@
+"""Extraction of a SurfSense API key from request headers.
+
+Pure and side-effect free: given the request headers, return the caller's key
+or ``None``. Isolated from transport and state so the parsing rules stay
+trivially unit-testable.
+"""
+
+from __future__ import annotations
+
+from starlette.datastructures import Headers
+
+_BEARER_PREFIX = "bearer "
+
+
+def extract_api_key(headers: Headers) -> str | None:
+ """Return the caller's key from the ``Authorization: Bearer`` slot the
+ backend already expects, falling back to ``X-API-Key`` for clients that can
+ only send custom headers."""
+ authorization = headers.get("authorization", "")
+ if authorization[: len(_BEARER_PREFIX)].lower() == _BEARER_PREFIX:
+ token = authorization[len(_BEARER_PREFIX) :].strip()
+ if token:
+ return token
+
+ fallback = headers.get("x-api-key", "").strip()
+ return fallback or None
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py
new file mode 100644
index 000000000..5f242e94e
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py
@@ -0,0 +1,42 @@
+"""Request-scoped caller identity.
+
+Over streamable-http one process serves many users, so the caller's key lives
+in a contextvar for the life of a single request: the auth middleware binds it,
+and the client reads it when building the outbound backend call. Under stdio
+there is no request, the contextvar stays empty, and the env key is used.
+
+The contextvar is request-scoped, not stored state — it is re-derived from the
+header on every request, which is what keeps the server stateless.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar, Token
+
+_LOCAL_IDENTITY = "__local__"
+
+_api_key: ContextVar[str | None] = ContextVar("surfsense_api_key", default=None)
+
+
+def bind_api_key(api_key: str | None) -> Token:
+ """Bind the caller's key to the current request; returns a reset token."""
+ return _api_key.set(api_key)
+
+
+def unbind_api_key(token: Token) -> None:
+ """Release the binding once the request is done."""
+ _api_key.reset(token)
+
+
+def current_api_key() -> str | None:
+ """The caller's key for the in-flight request, or ``None`` under stdio."""
+ return _api_key.get()
+
+
+def current_identity() -> str:
+ """Stable per-caller key for scoping request state.
+
+ The token identifies the account, so state keyed on it is naturally
+ per-user and survives reconnects. Under stdio all calls share one identity.
+ """
+ return _api_key.get() or _LOCAL_IDENTITY
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
new file mode 100644
index 000000000..2f2356ad7
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
@@ -0,0 +1,54 @@
+"""ASGI middleware that establishes the caller's identity for each request.
+
+A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``:
+the latter runs the endpoint in a separate task, so a contextvar set in it does
+not reach the tool handler. A pure middleware sets the key in the request's own
+task, from which the SDK's per-request handling inherits it (verified).
+
+Requests without a key are rejected here so no tool ever runs unauthenticated.
+"""
+
+from __future__ import annotations
+
+from starlette.datastructures import Headers
+from starlette.responses import JSONResponse
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+from .headers import extract_api_key
+from .identity import bind_api_key, unbind_api_key
+
+
+class ApiKeyIdentityMiddleware:
+ """Binds the per-request API key into the identity contextvar, or 401s."""
+
+ def __init__(self, app: ASGIApp) -> None:
+ self._app = app
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ if scope["type"] != "http":
+ await self._app(scope, receive, send)
+ return
+
+ api_key = extract_api_key(Headers(scope=scope))
+ if api_key is None:
+ await _unauthenticated()(scope, receive, send)
+ return
+
+ token = bind_api_key(api_key)
+ try:
+ await self._app(scope, receive, send)
+ finally:
+ unbind_api_key(token)
+
+
+def _unauthenticated() -> JSONResponse:
+ return JSONResponse(
+ {
+ "error": "unauthorized",
+ "message": (
+ "Missing SurfSense API key. Send 'Authorization: Bearer "
+ "ss_pat_...' (or an X-API-Key header)."
+ ),
+ },
+ status_code=401,
+ )
From 80e032ec3a2d43779b9bb51029b8d71804716c2f Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 005/160] feat(mcp): assemble streamable-http transport app
---
.../surfsense_mcp/core/transport/__init__.py | 5 ++++
.../src/surfsense_mcp/core/transport/http.py | 27 +++++++++++++++++++
2 files changed, 32 insertions(+)
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/transport/http.py
diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py
new file mode 100644
index 000000000..53aaa6b8c
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py
@@ -0,0 +1,5 @@
+"""Transport wiring for the MCP server (streamable-http app assembly)."""
+
+from .http import build_http_app
+
+__all__ = ["build_http_app"]
diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py
new file mode 100644
index 000000000..2f2cf8f53
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py
@@ -0,0 +1,27 @@
+"""Assemble the streamable-http ASGI app for the remote transport.
+
+Wraps the SDK's MCP endpoint with the API-key identity middleware and CORS.
+CORS sits outermost so browser preflight (which carries no key) is answered
+before the identity middleware, and clients can read the ``Mcp-Session-Id``
+header the streamable-http protocol relies on.
+"""
+
+from __future__ import annotations
+
+from mcp.server.fastmcp import FastMCP
+from starlette.middleware.cors import CORSMiddleware
+from starlette.types import ASGIApp
+
+from ..auth.middleware import ApiKeyIdentityMiddleware
+
+
+def build_http_app(mcp: FastMCP) -> ASGIApp:
+ """Return the MCP streamable-http app wrapped with identity + CORS."""
+ app: ASGIApp = ApiKeyIdentityMiddleware(mcp.streamable_http_app())
+ return CORSMiddleware(
+ app,
+ allow_origins=["*"],
+ allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
+ allow_headers=["*"],
+ expose_headers=["Mcp-Session-Id"],
+ )
From 8a54e0293eccb04ac46ef2e7dc83fd1402fb5240 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 006/160] refactor(mcp): resolve API key per request in client
---
.../src/surfsense_mcp/core/client.py | 36 +++++++++++++++----
1 file changed, 29 insertions(+), 7 deletions(-)
diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py
index 55f9d6048..cfe1a73ee 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/client.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/client.py
@@ -10,10 +10,11 @@ from typing import Any
import httpx
+from .auth.identity import current_api_key
from .errors import ToolError
_FAILURE_HINTS: dict[int, str] = {
- 401: "Authentication failed — check that SURFSENSE_API_KEY is a valid, unexpired key.",
+ 401: "Authentication failed — the SurfSense API key is invalid or expired.",
402: "The workspace is out of credits for this operation.",
403: (
"Access denied — the token lacks permission, or API access is disabled "
@@ -27,17 +28,31 @@ _FAILURE_HINTS: dict[int, str] = {
class SurfSenseClient:
"""Issues authenticated requests against ``{base_url}{api_prefix}``."""
- def __init__(self, *, api_base: str, api_key: str, timeout: float) -> None:
+ def __init__(
+ self, *, api_base: str, timeout: float, fallback_api_key: str | None = None
+ ) -> None:
self._api_base = api_base
+ # The key is resolved per request (one client serves many users over
+ # http), so none is baked into the shared client. ``fallback_api_key``
+ # is the env-supplied key used under stdio, where there is no header.
+ self._fallback_api_key = fallback_api_key
self._http = httpx.AsyncClient(
base_url=api_base,
- headers={
- "Authorization": f"Bearer {api_key}",
- "Accept": "application/json",
- },
+ headers={"Accept": "application/json"},
timeout=timeout,
)
+ def _auth_headers(self) -> dict[str, str]:
+ """Resolve the caller's key: the per-request header, else the env key."""
+ api_key = current_api_key() or self._fallback_api_key
+ if not api_key:
+ raise ToolError(
+ "No SurfSense API key supplied. Send it as an 'Authorization: "
+ "Bearer ss_pat_...' header (remote server), or set the "
+ "SURFSENSE_API_KEY environment variable (stdio)."
+ )
+ return {"Authorization": f"Bearer {api_key}"}
+
async def request(
self,
method: str,
@@ -53,9 +68,16 @@ class SurfSenseClient:
# as a value (e.g. int("") on folder_id) and fail.
if params is not None:
params = {key: value for key, value in params.items() if value is not None}
+ headers = self._auth_headers()
try:
response = await self._http.request(
- method, path, params=params, json=json, data=data, files=files
+ method,
+ path,
+ params=params,
+ json=json,
+ data=data,
+ files=files,
+ headers=headers,
)
except httpx.RequestError as exc:
raise ToolError(
From 3b4678ade46df2d53be6657771007ae86f5c7f0e Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 007/160] feat(mcp): make api_key optional, add http host/port
config
---
surfsense_mcp/src/surfsense_mcp/config.py | 25 +++++++++++++-------
surfsense_mcp/src/surfsense_mcp/selfcheck.py | 2 ++
2 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/surfsense_mcp/src/surfsense_mcp/config.py b/surfsense_mcp/src/surfsense_mcp/config.py
index 5d3646545..18d0f1514 100644
--- a/surfsense_mcp/src/surfsense_mcp/config.py
+++ b/surfsense_mcp/src/surfsense_mcp/config.py
@@ -12,6 +12,8 @@ from dataclasses import dataclass
DEFAULT_BASE_URL = "http://localhost:8000"
DEFAULT_API_PREFIX = "/api/v1"
DEFAULT_TIMEOUT_SECONDS = 180.0
+DEFAULT_HTTP_HOST = "127.0.0.1"
+DEFAULT_HTTP_PORT = 8080
@dataclass(frozen=True)
@@ -19,10 +21,12 @@ class Settings:
"""Resolved configuration for a server process."""
base_url: str
- api_key: str
+ api_key: str | None
api_prefix: str
timeout: float
default_workspace: str | None
+ host: str
+ port: int
@property
def api_base(self) -> str:
@@ -30,13 +34,9 @@ class Settings:
@classmethod
def from_env(cls) -> Settings:
- api_key = os.environ.get("SURFSENSE_API_KEY", "").strip()
- if not api_key:
- raise SystemExit(
- "SURFSENSE_API_KEY is required. Create an API key in SurfSense "
- "(Settings -> API) and pass it via the SURFSENSE_API_KEY "
- "environment variable."
- )
+ # Optional here: remote (http) callers pass their own key per request in
+ # a header. ``__main__`` enforces it for stdio, its only source of a key.
+ api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() or None
base_url = (
os.environ.get("SURFSENSE_BASE_URL", DEFAULT_BASE_URL).strip().rstrip("/")
@@ -53,10 +53,19 @@ class Settings:
default_workspace = os.environ.get("SURFSENSE_WORKSPACE", "").strip() or None
+ host = os.environ.get("SURFSENSE_MCP_HOST", "").strip() or DEFAULT_HTTP_HOST
+ raw_port = os.environ.get("SURFSENSE_MCP_PORT", "").strip()
+ try:
+ port = int(raw_port) if raw_port else DEFAULT_HTTP_PORT
+ except ValueError:
+ port = DEFAULT_HTTP_PORT
+
return cls(
base_url=base_url,
api_key=api_key,
api_prefix=api_prefix,
timeout=timeout,
default_workspace=default_workspace,
+ host=host,
+ port=port,
)
diff --git a/surfsense_mcp/src/surfsense_mcp/selfcheck.py b/surfsense_mcp/src/surfsense_mcp/selfcheck.py
index e38edc909..20224c173 100644
--- a/surfsense_mcp/src/surfsense_mcp/selfcheck.py
+++ b/surfsense_mcp/src/surfsense_mcp/selfcheck.py
@@ -47,6 +47,8 @@ async def _collect_tools() -> dict[str, object]:
api_prefix="/api/v1",
timeout=5.0,
default_workspace=None,
+ host="127.0.0.1",
+ port=8080,
)
mcp, _client = build_server(settings)
tools = await mcp.list_tools()
From 93bd022c02e3d4d60c40c60b009a9a88f5f1fbb3 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 008/160] feat(mcp): serve stateless streamable-http with stdio
dispatch
---
surfsense_mcp/src/surfsense_mcp/__main__.py | 33 +++++++++++++++++++--
surfsense_mcp/src/surfsense_mcp/server.py | 11 ++++++-
2 files changed, 40 insertions(+), 4 deletions(-)
diff --git a/surfsense_mcp/src/surfsense_mcp/__main__.py b/surfsense_mcp/src/surfsense_mcp/__main__.py
index 6c878ebfa..37a2a1785 100644
--- a/surfsense_mcp/src/surfsense_mcp/__main__.py
+++ b/surfsense_mcp/src/surfsense_mcp/__main__.py
@@ -1,7 +1,12 @@
"""Entry point: load settings from the environment and run the MCP server.
-Defaults to stdio (what Cursor, Claude Code, and Claude Desktop launch). stdout
-is the protocol channel, so every log line goes to stderr.
+Two transports share one build:
+- ``stdio`` (default): Cursor/Claude launch one process per user; the key comes
+ from the environment, so it is required here.
+- ``streamable-http``: one process serves many users, each passing their own key
+ per request; the key is enforced by the transport's auth middleware instead.
+
+For stdio, stdout is the protocol channel, so every log line goes to stderr.
"""
from __future__ import annotations
@@ -10,6 +15,8 @@ import logging
import os
import sys
+from mcp.server.fastmcp import FastMCP
+
from .config import Settings
from .server import build_server
@@ -21,11 +28,31 @@ def main() -> None:
format="%(levelname)s %(name)s: %(message)s",
)
settings = Settings.from_env()
+ transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio"
mcp, _client = build_server(settings)
- transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio"
+ if transport in ("streamable-http", "http"):
+ _run_http(mcp, settings)
+ return
+
+ if transport == "stdio" and not settings.api_key:
+ raise SystemExit(
+ "SURFSENSE_API_KEY is required for stdio transport. Create an API "
+ "key in SurfSense (Settings -> API) and pass it via the "
+ "SURFSENSE_API_KEY environment variable."
+ )
mcp.run(transport=transport)
+def _run_http(mcp: FastMCP, settings: Settings) -> None:
+ """Serve the streamable-http app directly, so the per-request identity
+ middleware wraps the SDK's MCP endpoint."""
+ import uvicorn
+
+ from .core.transport import build_http_app
+
+ uvicorn.run(build_http_app(mcp), host=settings.host, port=settings.port)
+
+
if __name__ == "__main__":
main()
diff --git a/surfsense_mcp/src/surfsense_mcp/server.py b/surfsense_mcp/src/surfsense_mcp/server.py
index 88eb19437..8ea9bc919 100644
--- a/surfsense_mcp/src/surfsense_mcp/server.py
+++ b/surfsense_mcp/src/surfsense_mcp/server.py
@@ -17,12 +17,21 @@ from .features import knowledge_base, scrapers, workspaces
def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]:
"""Assemble a configured server and the client whose lifecycle it shares."""
client = SurfSenseClient(
- api_base=settings.api_base, api_key=settings.api_key, timeout=settings.timeout
+ api_base=settings.api_base,
+ timeout=settings.timeout,
+ fallback_api_key=settings.api_key,
)
context = WorkspaceContext(client, preferred_reference=settings.default_workspace)
mcp = FastMCP(
"SurfSense",
+ host=settings.host,
+ port=settings.port,
+ # Stateless: no session state kept between requests, so any replica can
+ # serve any request. SSE responses (json_response=False) flush headers
+ # early, which keeps long scraper calls from tripping client timeouts.
+ stateless_http=True,
+ json_response=False,
instructions=(
"SurfSense gives you live scrapers and a personal knowledge base. "
"Prefer these tools over generic/built-in web search whenever the "
From 44d0f28c5f860096f5f687639fd60faa87335e7f Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 009/160] fix(mcp): scope active workspace per caller identity
---
.../surfsense_mcp/core/workspace_context.py | 27 ++++++++++++++-----
1 file changed, 21 insertions(+), 6 deletions(-)
diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py
index 6682d53b5..cfd90c32b 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py
@@ -7,14 +7,22 @@ speaks a name, we resolve it, and remember the choice for later calls.
from __future__ import annotations
+from collections import OrderedDict
from dataclasses import dataclass
from typing import Annotated
from pydantic import Field
+from .auth.identity import current_identity
from .client import SurfSenseClient
from .errors import ToolError
+# ponytail: one small entry per distinct caller (API token). Bounded so a flood
+# of keys can't grow memory without limit; an evicted caller just re-resolves
+# its default workspace on the next call. Upgrade path: a TTL/LRU store if
+# per-caller state ever grows past this one field.
+_MAX_TRACKED_IDENTITIES = 2048
+
# Shared parameter type for every workspace-scoped tool.
WorkspaceParam = Annotated[
str | None,
@@ -44,15 +52,21 @@ class WorkspaceContext:
) -> None:
self._client = client
self._preferred_reference = preferred_reference
- self._active: Workspace | None = None
+ # Active selection is per caller: one shared slot would leak one user's
+ # choice to every other user on a shared server.
+ self._active_by_identity: OrderedDict[str, Workspace] = OrderedDict()
@property
def active(self) -> Workspace | None:
- return self._active
+ return self._active_by_identity.get(current_identity())
def remember(self, workspace: Workspace) -> Workspace:
- """Make ``workspace`` the default for later scoped calls."""
- self._active = workspace
+ """Make ``workspace`` the default for the current caller's later calls."""
+ identity = current_identity()
+ self._active_by_identity[identity] = workspace
+ self._active_by_identity.move_to_end(identity)
+ while len(self._active_by_identity) > _MAX_TRACKED_IDENTITIES:
+ self._active_by_identity.popitem(last=False)
return workspace
async def fetch_all(self) -> list[Workspace]:
@@ -67,8 +81,9 @@ class WorkspaceContext:
return self.remember(await self._match(reference))
async def _resolve_default(self) -> Workspace:
- if self._active is not None:
- return self._active
+ active = self.active
+ if active is not None:
+ return active
if self._preferred_reference:
return self.remember(await self._match(self._preferred_reference))
return self.remember(await self._only_workspace_or_prompt())
From fa7d5f83f0ea8db081199d6e2afb99154c25c827 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 010/160] test(mcp): cover per-request auth and workspace
isolation
---
surfsense_mcp/tests/test_auth_headers.py | 40 +++++++
surfsense_mcp/tests/test_client_errors.py | 2 +-
surfsense_mcp/tests/test_client_params.py | 4 +-
surfsense_mcp/tests/test_request_auth.py | 100 ++++++++++++++++++
surfsense_mcp/tests/test_workspace_context.py | 28 +++++
5 files changed, 172 insertions(+), 2 deletions(-)
create mode 100644 surfsense_mcp/tests/test_auth_headers.py
create mode 100644 surfsense_mcp/tests/test_request_auth.py
diff --git a/surfsense_mcp/tests/test_auth_headers.py b/surfsense_mcp/tests/test_auth_headers.py
new file mode 100644
index 000000000..efb2d85a9
--- /dev/null
+++ b/surfsense_mcp/tests/test_auth_headers.py
@@ -0,0 +1,40 @@
+"""API key extraction from request headers: Bearer, fallback, and rejection."""
+
+from __future__ import annotations
+
+from starlette.datastructures import Headers
+
+from surfsense_mcp.core.auth.headers import extract_api_key
+
+
+def _headers(**pairs: str) -> Headers:
+ return Headers(pairs)
+
+
+def test_reads_bearer_token():
+ assert extract_api_key(_headers(authorization="Bearer ss_pat_abc")) == "ss_pat_abc"
+
+
+def test_bearer_scheme_is_case_insensitive():
+ assert extract_api_key(_headers(authorization="bearer ss_pat_abc")) == "ss_pat_abc"
+
+
+def test_falls_back_to_x_api_key():
+ assert extract_api_key(Headers({"x-api-key": "ss_pat_xyz"})) == "ss_pat_xyz"
+
+
+def test_bearer_wins_over_fallback():
+ headers = Headers({"authorization": "Bearer primary", "x-api-key": "secondary"})
+ assert extract_api_key(headers) == "primary"
+
+
+def test_missing_headers_return_none():
+ assert extract_api_key(_headers()) is None
+
+
+def test_empty_bearer_is_rejected():
+ assert extract_api_key(_headers(authorization="Bearer ")) is None
+
+
+def test_non_bearer_authorization_is_ignored():
+ assert extract_api_key(_headers(authorization="Basic abc123")) is None
diff --git a/surfsense_mcp/tests/test_client_errors.py b/surfsense_mcp/tests/test_client_errors.py
index 648bfde3e..0525fb523 100644
--- a/surfsense_mcp/tests/test_client_errors.py
+++ b/surfsense_mcp/tests/test_client_errors.py
@@ -15,7 +15,7 @@ def _response(status: int, **kwargs) -> httpx.Response:
def test_explains_401_with_token_hint():
message = SurfSenseClient._explain_failure(_response(401, json={"detail": "bad"}))
- assert "SURFSENSE_API_KEY" in message
+ assert "API key" in message
assert "bad" in message
diff --git a/surfsense_mcp/tests/test_client_params.py b/surfsense_mcp/tests/test_client_params.py
index 4840ad488..e9cd33a0c 100644
--- a/surfsense_mcp/tests/test_client_params.py
+++ b/surfsense_mcp/tests/test_client_params.py
@@ -25,7 +25,9 @@ def _capture(client: SurfSenseClient) -> dict:
def test_none_params_are_dropped():
- client = SurfSenseClient(api_base="http://test/api/v1", api_key="ss_pat_x", timeout=5)
+ client = SurfSenseClient(
+ api_base="http://test/api/v1", timeout=5, fallback_api_key="ss_pat_x"
+ )
seen = _capture(client)
asyncio.run(
client.request(
diff --git a/surfsense_mcp/tests/test_request_auth.py b/surfsense_mcp/tests/test_request_auth.py
new file mode 100644
index 000000000..c548dff7a
--- /dev/null
+++ b/surfsense_mcp/tests/test_request_auth.py
@@ -0,0 +1,100 @@
+"""Per-request key resolution and the Authorization header the backend receives.
+
+Covers the security-critical behaviors: the per-request key wins over the env
+fallback, the fallback covers stdio, a missing key is refused, and concurrent
+callers never see each other's key.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+import pytest
+
+from surfsense_mcp.core.auth import identity
+from surfsense_mcp.core.client import SurfSenseClient
+from surfsense_mcp.core.errors import ToolError
+
+
+def _client_recording_auth(seen: dict, *, fallback: str | None) -> SurfSenseClient:
+ async def handler(request: httpx.Request) -> httpx.Response:
+ seen["authorization"] = request.headers.get("authorization")
+ return httpx.Response(200, json={"ok": True})
+
+ client = SurfSenseClient(
+ api_base="http://test/api/v1", timeout=5, fallback_api_key=fallback
+ )
+ client._http = httpx.AsyncClient(
+ base_url="http://test/api/v1", transport=httpx.MockTransport(handler)
+ )
+ return client
+
+
+async def _get(client: SurfSenseClient) -> None:
+ await client.request("GET", "/workspaces")
+
+
+def test_request_key_is_sent_as_bearer():
+ seen: dict = {}
+ client = _client_recording_auth(seen, fallback=None)
+
+ async def run() -> None:
+ token = identity.bind_api_key("ss_pat_request")
+ try:
+ await _get(client)
+ finally:
+ identity.unbind_api_key(token)
+
+ asyncio.run(run())
+ assert seen["authorization"] == "Bearer ss_pat_request"
+
+
+def test_request_key_overrides_env_fallback():
+ seen: dict = {}
+ client = _client_recording_auth(seen, fallback="ss_pat_env")
+
+ async def run() -> None:
+ token = identity.bind_api_key("ss_pat_request")
+ try:
+ await _get(client)
+ finally:
+ identity.unbind_api_key(token)
+
+ asyncio.run(run())
+ assert seen["authorization"] == "Bearer ss_pat_request"
+
+
+def test_env_fallback_used_without_request_key():
+ seen: dict = {}
+ client = _client_recording_auth(seen, fallback="ss_pat_env")
+ asyncio.run(_get(client))
+ assert seen["authorization"] == "Bearer ss_pat_env"
+
+
+def test_missing_key_is_refused():
+ client = _client_recording_auth({}, fallback=None)
+ with pytest.raises(ToolError):
+ asyncio.run(_get(client))
+
+
+def test_concurrent_callers_do_not_leak_keys():
+ seen_by_caller: dict[str, str | None] = {}
+
+ async def call_as(key: str) -> None:
+ # Each caller runs in its own task, so the contextvar is isolated.
+ recorded: dict = {}
+ client = _client_recording_auth(recorded, fallback=None)
+ token = identity.bind_api_key(key)
+ try:
+ await _get(client)
+ finally:
+ identity.unbind_api_key(token)
+ seen_by_caller[key] = recorded["authorization"]
+
+ async def run() -> None:
+ await asyncio.gather(call_as("ss_pat_A"), call_as("ss_pat_B"))
+
+ asyncio.run(run())
+ assert seen_by_caller["ss_pat_A"] == "Bearer ss_pat_A"
+ assert seen_by_caller["ss_pat_B"] == "Bearer ss_pat_B"
diff --git a/surfsense_mcp/tests/test_workspace_context.py b/surfsense_mcp/tests/test_workspace_context.py
index 9ba9c2ca3..d2024e5d5 100644
--- a/surfsense_mcp/tests/test_workspace_context.py
+++ b/surfsense_mcp/tests/test_workspace_context.py
@@ -6,6 +6,7 @@ import asyncio
import pytest
+from surfsense_mcp.core.auth import identity
from surfsense_mcp.core.errors import ToolError
from surfsense_mcp.core.workspace_context import WorkspaceContext
@@ -96,3 +97,30 @@ def test_resolution_is_remembered_as_active():
assert ctx.active is not None and ctx.active.id == 2
# a later default call reuses the active selection without re-choosing
assert asyncio.run(ctx.resolve(None)).id == 2
+
+
+def test_active_workspace_is_isolated_per_identity():
+ ctx = _context(_rows(("A", 1), ("B", 2)))
+
+ async def select_as(key: str, reference: str) -> None:
+ token = identity.bind_api_key(key)
+ try:
+ await ctx.resolve(reference)
+ finally:
+ identity.unbind_api_key(token)
+
+ async def active_for(key: str) -> int | None:
+ token = identity.bind_api_key(key)
+ try:
+ return ctx.active.id if ctx.active else None
+ finally:
+ identity.unbind_api_key(token)
+
+ asyncio.run(select_as("ss_pat_A", "A"))
+ asyncio.run(select_as("ss_pat_B", "B"))
+
+ # Each caller keeps its own selection; no bleed across identities.
+ assert asyncio.run(active_for("ss_pat_A")) == 1
+ assert asyncio.run(active_for("ss_pat_B")) == 2
+ # An unknown caller has no active selection.
+ assert asyncio.run(active_for("ss_pat_C")) is None
From 0d4033682d6bda104d7a16622715bba4976ae393 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:52:30 +0200
Subject: [PATCH 011/160] build(mcp): declare starlette and uvicorn
dependencies
---
surfsense_mcp/pyproject.toml | 2 ++
surfsense_mcp/uv.lock | 4 ++++
2 files changed, 6 insertions(+)
diff --git a/surfsense_mcp/pyproject.toml b/surfsense_mcp/pyproject.toml
index 87db0c223..489bbf455 100644
--- a/surfsense_mcp/pyproject.toml
+++ b/surfsense_mcp/pyproject.toml
@@ -8,6 +8,8 @@ license = { text = "Apache-2.0" }
dependencies = [
"mcp>=1.26.0",
"httpx>=0.27.0",
+ "starlette>=0.37",
+ "uvicorn>=0.30",
]
[project.scripts]
diff --git a/surfsense_mcp/uv.lock b/surfsense_mcp/uv.lock
index bc97a9b2a..d1bfab702 100644
--- a/surfsense_mcp/uv.lock
+++ b/surfsense_mcp/uv.lock
@@ -718,6 +718,8 @@ source = { editable = "." }
dependencies = [
{ name = "httpx" },
{ name = "mcp" },
+ { name = "starlette" },
+ { name = "uvicorn" },
]
[package.dev-dependencies]
@@ -729,6 +731,8 @@ dev = [
requires-dist = [
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "mcp", specifier = ">=1.26.0" },
+ { name = "starlette", specifier = ">=0.37" },
+ { name = "uvicorn", specifier = ">=0.30" },
]
[package.metadata.requires-dev]
From a7215c09dc44a58d0a8686d74cd577deb3c6e15f Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 16:55:20 +0200
Subject: [PATCH 012/160] docs(mcp): tighten comments to intent-only
---
.../src/surfsense_mcp/core/auth/headers.py | 6 ++----
.../src/surfsense_mcp/core/auth/identity.py | 18 +++++-------------
.../src/surfsense_mcp/core/auth/middleware.py | 4 ++--
surfsense_mcp/src/surfsense_mcp/core/client.py | 5 ++---
4 files changed, 11 insertions(+), 22 deletions(-)
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py
index c06989909..3bca45967 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py
@@ -1,8 +1,6 @@
-"""Extraction of a SurfSense API key from request headers.
+"""Extract a SurfSense API key from request headers.
-Pure and side-effect free: given the request headers, return the caller's key
-or ``None``. Isolated from transport and state so the parsing rules stay
-trivially unit-testable.
+Pure header parsing, kept separate from transport and state.
"""
from __future__ import annotations
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py
index 5f242e94e..0c8f6b315 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py
@@ -1,12 +1,9 @@
"""Request-scoped caller identity.
-Over streamable-http one process serves many users, so the caller's key lives
-in a contextvar for the life of a single request: the auth middleware binds it,
-and the client reads it when building the outbound backend call. Under stdio
-there is no request, the contextvar stays empty, and the env key is used.
-
-The contextvar is request-scoped, not stored state — it is re-derived from the
-header on every request, which is what keeps the server stateless.
+Over streamable-http one process serves many users, so the caller's key lives in
+a contextvar for the life of a request: the auth middleware binds it and the
+client reads it when calling the backend. Under stdio there is no request, so the
+contextvar is empty and the env key is used instead.
"""
from __future__ import annotations
@@ -24,7 +21,6 @@ def bind_api_key(api_key: str | None) -> Token:
def unbind_api_key(token: Token) -> None:
- """Release the binding once the request is done."""
_api_key.reset(token)
@@ -34,9 +30,5 @@ def current_api_key() -> str | None:
def current_identity() -> str:
- """Stable per-caller key for scoping request state.
-
- The token identifies the account, so state keyed on it is naturally
- per-user and survives reconnects. Under stdio all calls share one identity.
- """
+ """Stable per-caller key for scoping request state; shared under stdio."""
return _api_key.get() or _LOCAL_IDENTITY
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
index 2f2356ad7..354ae0a89 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
@@ -2,8 +2,8 @@
A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``:
the latter runs the endpoint in a separate task, so a contextvar set in it does
-not reach the tool handler. A pure middleware sets the key in the request's own
-task, from which the SDK's per-request handling inherits it (verified).
+not reach the tool handler. A pure middleware binds the key in the request's own
+task, from which the SDK's per-request handling inherits it.
Requests without a key are rejected here so no tool ever runs unauthenticated.
"""
diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py
index cfe1a73ee..9ae446ae2 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/client.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/client.py
@@ -32,9 +32,8 @@ class SurfSenseClient:
self, *, api_base: str, timeout: float, fallback_api_key: str | None = None
) -> None:
self._api_base = api_base
- # The key is resolved per request (one client serves many users over
- # http), so none is baked into the shared client. ``fallback_api_key``
- # is the env-supplied key used under stdio, where there is no header.
+ # Resolved per request, so no key is baked into the shared client. The
+ # fallback is the env key used under stdio, where there is no header.
self._fallback_api_key = fallback_api_key
self._http = httpx.AsyncClient(
base_url=api_base,
From 2acc1426bf8294f0fd03c5a6e1e7f97b6aba9310 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 17:22:37 +0200
Subject: [PATCH 013/160] refactor(mcp): split scraper and knowledge-base
features into SRP modules
---
.../surfsense_mcp/core/workspace_context.py | 43 +-
.../surfsense_mcp/core/workspace_matching.py | 51 ++
.../features/knowledge_base/__init__.py | 371 +-----------
.../features/knowledge_base/annotations.py | 34 ++
.../features/knowledge_base/document_tools.py | 185 ++++++
.../features/knowledge_base/search_tools.py | 188 ++++++
.../features/scrapers/__init__.py | 562 +-----------------
.../features/scrapers/annotations.py | 13 +
.../features/scrapers/platforms/__init__.py | 1 +
.../scrapers/platforms/google_maps.py | 151 +++++
.../scrapers/platforms/google_search.py | 79 +++
.../features/scrapers/platforms/reddit.py | 98 +++
.../features/scrapers/platforms/web.py | 89 +++
.../features/scrapers/platforms/youtube.py | 131 ++++
.../features/scrapers/run_history.py | 112 ++++
15 files changed, 1153 insertions(+), 955 deletions(-)
create mode 100644 surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py
create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py
diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py
index cfd90c32b..0c5423e6b 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py
@@ -16,6 +16,7 @@ from pydantic import Field
from .auth.identity import current_identity
from .client import SurfSenseClient
from .errors import ToolError
+from .workspace_matching import as_int, match_by_name, name_list
# ponytail: one small entry per distinct caller (API token). Bounded so a flood
# of keys can't grow memory without limit; an evicted caller just re-resolves
@@ -99,43 +100,20 @@ class WorkspaceContext:
)
raise ToolError(
"No workspace selected. Choose one first with surfsense_select_workspace, "
- f"or pass 'workspace'. Available: {_name_list(workspaces)}."
+ f"or pass 'workspace'. Available: {name_list(workspaces)}."
)
async def _match(self, reference: str | int) -> Workspace:
workspaces = await self.fetch_all()
- as_id = _as_int(reference)
+ as_id = as_int(reference)
if as_id is not None:
found = next((w for w in workspaces if w.id == as_id), None)
if found is None:
raise ToolError(
- f"No workspace with id {as_id}. Available: {_name_list(workspaces)}."
+ f"No workspace with id {as_id}. Available: {name_list(workspaces)}."
)
return found
- return _match_by_name(str(reference), workspaces)
-
-
-def _match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace:
- """Match on name: exact, then case-insensitive, then unique substring."""
- needle = reference.strip()
- exact = [w for w in workspaces if w.name == needle]
- if exact:
- return exact[0]
- lowered = needle.casefold()
- insensitive = [w for w in workspaces if w.name.casefold() == lowered]
- if insensitive:
- return insensitive[0]
- partial = [w for w in workspaces if lowered in w.name.casefold()]
- if len(partial) == 1:
- return partial[0]
- if len(partial) > 1:
- raise ToolError(
- f"'{reference}' matches several workspaces: {_name_list(partial)}. "
- "Use a more specific name or the id."
- )
- raise ToolError(
- f"No workspace named '{reference}'. Available: {_name_list(workspaces)}."
- )
+ return match_by_name(str(reference), workspaces)
def _to_workspace(row: dict) -> Workspace:
@@ -146,14 +124,3 @@ def _to_workspace(row: dict) -> Workspace:
is_owner=row.get("is_owner", False),
member_count=row.get("member_count", 1),
)
-
-
-def _as_int(reference: str | int) -> int | None:
- if isinstance(reference, int):
- return reference
- text = reference.strip()
- return int(text) if text.isdigit() else None
-
-
-def _name_list(workspaces: list[Workspace]) -> str:
- return ", ".join(f"{w.name} (id {w.id})" for w in workspaces)
diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py
new file mode 100644
index 000000000..33bfc50a5
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py
@@ -0,0 +1,51 @@
+"""Resolve a user-supplied workspace reference to a single workspace.
+
+Pure matching over an already-fetched list: name (exact, then case-insensitive,
+then unique substring) or numeric id. Kept apart from WorkspaceContext so the
+resolution rules can be read and tested without the network.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .errors import ToolError
+
+if TYPE_CHECKING:
+ from .workspace_context import Workspace
+
+
+def match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace:
+ """Match on name: exact, then case-insensitive, then unique substring."""
+ needle = reference.strip()
+ exact = [w for w in workspaces if w.name == needle]
+ if exact:
+ return exact[0]
+ lowered = needle.casefold()
+ insensitive = [w for w in workspaces if w.name.casefold() == lowered]
+ if insensitive:
+ return insensitive[0]
+ partial = [w for w in workspaces if lowered in w.name.casefold()]
+ if len(partial) == 1:
+ return partial[0]
+ if len(partial) > 1:
+ raise ToolError(
+ f"'{reference}' matches several workspaces: {name_list(partial)}. "
+ "Use a more specific name or the id."
+ )
+ raise ToolError(
+ f"No workspace named '{reference}'. Available: {name_list(workspaces)}."
+ )
+
+
+def as_int(reference: str | int) -> int | None:
+ """Return the reference as an id, or None when it isn't numeric."""
+ if isinstance(reference, int):
+ return reference
+ text = reference.strip()
+ return int(text) if text.isdigit() else None
+
+
+def name_list(workspaces: list[Workspace]) -> str:
+ """Render workspaces as a human-readable 'name (id N)' list."""
+ return ", ".join(f"{w.name} (id {w.id})" for w in workspaces)
diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py
index 7fb1802ae..1a971bfe4 100644
--- a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py
+++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py
@@ -1,379 +1,22 @@
"""Knowledge-base tools: search the KB and manage its documents.
Semantic search plus the document lifecycle — list, read, add text, upload a
-file, update, and delete — over a workspace's knowledge base. Search and reads
-default to the active workspace; document ids identify a single document across
-the whole account, so id-addressed tools need no workspace.
+file, update, and delete — over a workspace's knowledge base. Read tools live in
+search_tools, mutations in document_tools.
"""
from __future__ import annotations
-import mimetypes
-from pathlib import Path
-from typing import Annotated
-
from mcp.server.fastmcp import FastMCP
-from mcp.types import ToolAnnotations
-from pydantic import Field
from ...core.client import SurfSenseClient
-from ...core.errors import ToolError
-from ...core.rendering import ResponseFormatParam, clip, to_json
-from ...core.workspace_context import WorkspaceContext, WorkspaceParam
-from .note_ingestion import build_note_document
-
-_READ = ToolAnnotations(
- readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False
-)
-_WRITE = ToolAnnotations(
- readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False
-)
-_DELETE = ToolAnnotations(
- readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False
-)
-
-_DOCUMENT_ID = Annotated[
- int,
- Field(
- description="Document id from surfsense_search_knowledge_base or "
- "surfsense_list_documents results."
- ),
-]
-
-_DOCUMENT_TYPES = Annotated[
- list[str] | None,
- Field(
- description="Restrict to these document types, e.g. "
- "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types."
- ),
-]
+from ...core.workspace_context import WorkspaceContext
+from . import document_tools, search_tools
def register(
mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
) -> None:
- """Register the knowledge-base tools on the server."""
-
- @mcp.tool(
- name="surfsense_search_knowledge_base",
- title="Search knowledge base",
- annotations=_READ,
- structured_output=False,
- )
- async def search_knowledge_base(
- query: Annotated[
- str,
- Field(
- min_length=1,
- description="Natural-language search, e.g. "
- "'notebooklm user complaints'.",
- ),
- ],
- top_k: Annotated[
- int, Field(ge=1, le=20, description="Maximum documents to return.")
- ] = 5,
- document_types: _DOCUMENT_TYPES = None,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Search the workspace's knowledge base by meaning and keywords.
-
- Use this FIRST when a question might be answered by content already
- stored in SurfSense — notes, uploaded files, saved pages, past
- research. Do NOT use it to fetch new data from the web; use the
- scraper tools for that. Returns the most relevant documents with the
- passages that matched, ranked by relevance score.
- Example: query='pricing feedback', top_k=5.
- """
- resolved = await context.resolve(workspace)
- hits = await client.request(
- "POST",
- "/documents/search-semantic",
- json={
- "workspace_id": resolved.id,
- "query": query,
- "top_k": max(1, min(top_k, 20)),
- "document_types": document_types,
- },
- )
- items = (hits or {}).get("items", [])
- if response_format == "json":
- return to_json(items)
- return _render_search(query, items)
-
- @mcp.tool(
- name="surfsense_list_documents",
- title="List documents",
- annotations=_READ,
- structured_output=False,
- )
- async def list_documents(
- document_types: _DOCUMENT_TYPES = None,
- folder_id: Annotated[
- int | None,
- Field(description="Only documents in this folder. Omit for all."),
- ] = None,
- page: Annotated[
- int, Field(ge=0, description="Zero-based page number.")
- ] = 0,
- page_size: Annotated[
- int, Field(ge=1, description="Documents per page.")
- ] = 20,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """List documents in the workspace's knowledge base, newest first.
-
- Use this to browse or inventory what is stored; to find documents
- about a topic, prefer surfsense_search_knowledge_base. Returns each
- document's title, id, type, and update time, plus a has_more flag —
- request the next page by increasing page.
- Example: document_types=['FILE'], page=0, page_size=20.
- """
- resolved = await context.resolve(workspace)
- result = await client.request(
- "GET",
- "/documents",
- params={
- "workspace_id": resolved.id,
- "page": page,
- "page_size": page_size,
- "document_types": _join(document_types),
- "folder_id": folder_id,
- },
- )
- if response_format == "json":
- return to_json(result)
- return _render_document_list(result)
-
- @mcp.tool(
- name="surfsense_get_document",
- title="Read one document",
- annotations=_READ,
- structured_output=False,
- )
- async def get_document(
- document_id: _DOCUMENT_ID,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Read one document's full content and metadata by id.
-
- Use this after surfsense_search_knowledge_base or
- surfsense_list_documents to open a specific document — search results
- only include the matching passages, this returns the whole text.
- """
- document = await client.request("GET", f"/documents/{document_id}")
- if response_format == "json":
- return clip(to_json(document))
- return _render_document(document)
-
- @mcp.tool(
- name="surfsense_add_document",
- title="Add a note",
- annotations=_WRITE,
- structured_output=False,
- )
- async def add_document(
- title: Annotated[
- str,
- Field(min_length=1, description="Short descriptive title for the note."),
- ],
- content: Annotated[
- str,
- Field(
- min_length=1,
- description="The note's body; plain text or markdown.",
- ),
- ],
- source_url: Annotated[
- str | None,
- Field(description="Where the text came from, if anywhere."),
- ] = None,
- workspace: WorkspaceParam = None,
- ) -> str:
- """Save a text or markdown note into the workspace's knowledge base.
-
- Use this to store notes, summaries, or findings so they become
- searchable later — e.g. after finishing a piece of research. For files
- on disk use surfsense_upload_file instead. Indexing is asynchronous,
- so the note may take a moment to appear in search.
- Example: title='NotebookLM subreddits', content='- r/notebooklm ...'.
- """
- resolved = await context.resolve(workspace)
- await client.request(
- "POST",
- "/documents",
- json=build_note_document(
- workspace_id=resolved.id,
- title=title,
- content=content,
- source_url=source_url,
- ),
- )
- return (
- f"Queued '{title}' for indexing in '{resolved.name}'. "
- "It will be searchable once processing completes."
- )
-
- @mcp.tool(
- name="surfsense_upload_file",
- title="Upload a file",
- annotations=_WRITE,
- structured_output=False,
- )
- async def upload_file(
- file_path: Annotated[
- str,
- Field(
- description="Path to a local file, e.g. "
- "'C:/Users/me/report.pdf' or '~/notes/summary.md'."
- ),
- ],
- use_vision_llm: Annotated[
- bool,
- Field(
- description="True reads scanned or image-heavy files with a "
- "vision model (slower)."
- ),
- ] = False,
- workspace: WorkspaceParam = None,
- ) -> str:
- """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base.
-
- Use this to ingest a file from disk so its content becomes searchable;
- for text you already have in hand use surfsense_add_document instead.
- The file is parsed, chunked, and indexed asynchronously. Duplicate
- files are detected and skipped.
- Example: file_path='C:/Users/me/report.pdf'.
- """
- resolved = await context.resolve(workspace)
- payload = _read_upload(file_path)
- result = await client.request(
- "POST",
- "/documents/fileupload",
- data={
- "workspace_id": str(resolved.id),
- "use_vision_llm": str(use_vision_llm).lower(),
- "processing_mode": "basic",
- },
- files=[("files", payload)],
- )
- pending = (result or {}).get("pending_files", 0)
- skipped = (result or {}).get("skipped_duplicates", 0)
- note = " (already present, skipped)" if skipped and not pending else ""
- return (
- f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. "
- "It will be searchable once processing completes."
- )
-
- @mcp.tool(
- name="surfsense_update_document",
- title="Replace a document's content",
- annotations=_WRITE,
- structured_output=False,
- )
- async def update_document(
- document_id: _DOCUMENT_ID,
- content: Annotated[
- str,
- Field(
- min_length=1,
- description="New full text; replaces the existing content "
- "entirely.",
- ),
- ],
- ) -> str:
- """Replace a document's stored content by id.
-
- Use this to correct or rewrite a document's text. The new content
- REPLACES the old entirely — to append, read the document first with
- surfsense_get_document and resend the combined text. Search chunks are
- not re-indexed by this call.
- """
- existing = await client.request("GET", f"/documents/{document_id}")
- await client.request(
- "PUT",
- f"/documents/{document_id}",
- json={
- "document_type": existing["document_type"],
- "workspace_id": existing["workspace_id"],
- "content": content,
- },
- )
- return f"Updated document {document_id} ('{existing.get('title', '')}')."
-
- @mcp.tool(
- name="surfsense_delete_document",
- title="Delete a document",
- annotations=_DELETE,
- structured_output=False,
- )
- async def delete_document(document_id: _DOCUMENT_ID) -> str:
- """Permanently delete a document from the knowledge base by id.
-
- Use this only when the user explicitly asks to remove a document —
- deletion cannot be undone. The document stops appearing in searches
- immediately.
- """
- await client.request("DELETE", f"/documents/{document_id}")
- return f"Deleted document {document_id}."
-
-
-def _read_upload(file_path: str) -> tuple[str, bytes, str]:
- path = Path(file_path).expanduser()
- if not path.is_file():
- raise ToolError(f"No file at '{file_path}'.")
- mime, _ = mimetypes.guess_type(path.name)
- return (path.name, path.read_bytes(), mime or "application/octet-stream")
-
-
-def _join(values: list[str] | None) -> str | None:
- return ",".join(values) if values else None
-
-
-def _render_search(query: str, items: list[dict]) -> str:
- if not items:
- return f'No matches for "{query}".'
- lines = [f'# {len(items)} result(s) for "{query}"', ""]
- for hit in items:
- lines.append(
- f"## {hit.get('title', 'Untitled')} "
- f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}"
- )
- for chunk in hit.get("chunks", []):
- excerpt = clip(chunk.get("content", "").strip(), 500)
- lines.append(f"> {excerpt}")
- lines.append("")
- return "\n".join(lines).strip()
-
-
-def _render_document_list(result: dict | None) -> str:
- items = (result or {}).get("items", [])
- if not items:
- return "No documents found."
- lines = ["# Documents", ""]
- for doc in items:
- lines.append(
- f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · "
- f"{doc.get('document_type')} · updated {doc.get('updated_at')}"
- )
- total = (result or {}).get("total", len(items))
- page = (result or {}).get("page", 0)
- has_more = (result or {}).get("has_more", False)
- lines.append("")
- lines.append(
- f"_Page {page} · showing {len(items)} of {total}"
- + (" · more available_" if has_more else "_")
- )
- return "\n".join(lines)
-
-
-def _render_document(document: dict) -> str:
- content = clip(document.get("content", "") or "(empty)")
- return (
- f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n"
- f"- type: {document.get('document_type')}\n"
- f"- workspace: {document.get('workspace_id')}\n"
- f"- updated: {document.get('updated_at')}\n\n"
- f"{content}"
- )
+ """Register every knowledge-base tool on the server."""
+ search_tools.register(mcp, client, context)
+ document_tools.register(mcp, client, context)
diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py
new file mode 100644
index 000000000..16322506c
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py
@@ -0,0 +1,34 @@
+"""Tool-call policy hints and shared parameter types for knowledge-base tools."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.types import ToolAnnotations
+from pydantic import Field
+
+READ = ToolAnnotations(
+ readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False
+)
+WRITE = ToolAnnotations(
+ readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False
+)
+DELETE = ToolAnnotations(
+ readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False
+)
+
+DocumentId = Annotated[
+ int,
+ Field(
+ description="Document id from surfsense_search_knowledge_base or "
+ "surfsense_list_documents results."
+ ),
+]
+
+DocumentTypes = Annotated[
+ list[str] | None,
+ Field(
+ description="Restrict to these document types, e.g. "
+ "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types."
+ ),
+]
diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py
new file mode 100644
index 000000000..497a2526c
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py
@@ -0,0 +1,185 @@
+"""Knowledge-base write tools: add a note, upload a file, update, and delete.
+
+Add and upload target the active workspace; update and delete address a document
+by its account-unique id, so they need no workspace.
+"""
+
+from __future__ import annotations
+
+import mimetypes
+from pathlib import Path
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ...core.client import SurfSenseClient
+from ...core.errors import ToolError
+from ...core.workspace_context import WorkspaceContext, WorkspaceParam
+from .annotations import DELETE, WRITE, DocumentId
+from .note_ingestion import build_note_document
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the knowledge-base write and delete tools."""
+
+ @mcp.tool(
+ name="surfsense_add_document",
+ title="Add a note",
+ annotations=WRITE,
+ structured_output=False,
+ )
+ async def add_document(
+ title: Annotated[
+ str,
+ Field(min_length=1, description="Short descriptive title for the note."),
+ ],
+ content: Annotated[
+ str,
+ Field(
+ min_length=1,
+ description="The note's body; plain text or markdown.",
+ ),
+ ],
+ source_url: Annotated[
+ str | None,
+ Field(description="Where the text came from, if anywhere."),
+ ] = None,
+ workspace: WorkspaceParam = None,
+ ) -> str:
+ """Save a text or markdown note into the workspace's knowledge base.
+
+ Use this to store notes, summaries, or findings so they become
+ searchable later — e.g. after finishing a piece of research. For files
+ on disk use surfsense_upload_file instead. Indexing is asynchronous,
+ so the note may take a moment to appear in search.
+ Example: title='NotebookLM subreddits', content='- r/notebooklm ...'.
+ """
+ resolved = await context.resolve(workspace)
+ await client.request(
+ "POST",
+ "/documents",
+ json=build_note_document(
+ workspace_id=resolved.id,
+ title=title,
+ content=content,
+ source_url=source_url,
+ ),
+ )
+ return (
+ f"Queued '{title}' for indexing in '{resolved.name}'. "
+ "It will be searchable once processing completes."
+ )
+
+ @mcp.tool(
+ name="surfsense_upload_file",
+ title="Upload a file",
+ annotations=WRITE,
+ structured_output=False,
+ )
+ async def upload_file(
+ file_path: Annotated[
+ str,
+ Field(
+ description="Path to a local file, e.g. "
+ "'C:/Users/me/report.pdf' or '~/notes/summary.md'."
+ ),
+ ],
+ use_vision_llm: Annotated[
+ bool,
+ Field(
+ description="True reads scanned or image-heavy files with a "
+ "vision model (slower)."
+ ),
+ ] = False,
+ workspace: WorkspaceParam = None,
+ ) -> str:
+ """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base.
+
+ Use this to ingest a file from disk so its content becomes searchable;
+ for text you already have in hand use surfsense_add_document instead.
+ The file is parsed, chunked, and indexed asynchronously. Duplicate
+ files are detected and skipped.
+ Example: file_path='C:/Users/me/report.pdf'.
+ """
+ resolved = await context.resolve(workspace)
+ payload = _read_upload(file_path)
+ result = await client.request(
+ "POST",
+ "/documents/fileupload",
+ data={
+ "workspace_id": str(resolved.id),
+ "use_vision_llm": str(use_vision_llm).lower(),
+ "processing_mode": "basic",
+ },
+ files=[("files", payload)],
+ )
+ pending = (result or {}).get("pending_files", 0)
+ skipped = (result or {}).get("skipped_duplicates", 0)
+ note = " (already present, skipped)" if skipped and not pending else ""
+ return (
+ f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. "
+ "It will be searchable once processing completes."
+ )
+
+ @mcp.tool(
+ name="surfsense_update_document",
+ title="Replace a document's content",
+ annotations=WRITE,
+ structured_output=False,
+ )
+ async def update_document(
+ document_id: DocumentId,
+ content: Annotated[
+ str,
+ Field(
+ min_length=1,
+ description="New full text; replaces the existing content "
+ "entirely.",
+ ),
+ ],
+ ) -> str:
+ """Replace a document's stored content by id.
+
+ Use this to correct or rewrite a document's text. The new content
+ REPLACES the old entirely — to append, read the document first with
+ surfsense_get_document and resend the combined text. Search chunks are
+ not re-indexed by this call.
+ """
+ existing = await client.request("GET", f"/documents/{document_id}")
+ await client.request(
+ "PUT",
+ f"/documents/{document_id}",
+ json={
+ "document_type": existing["document_type"],
+ "workspace_id": existing["workspace_id"],
+ "content": content,
+ },
+ )
+ return f"Updated document {document_id} ('{existing.get('title', '')}')."
+
+ @mcp.tool(
+ name="surfsense_delete_document",
+ title="Delete a document",
+ annotations=DELETE,
+ structured_output=False,
+ )
+ async def delete_document(document_id: DocumentId) -> str:
+ """Permanently delete a document from the knowledge base by id.
+
+ Use this only when the user explicitly asks to remove a document —
+ deletion cannot be undone. The document stops appearing in searches
+ immediately.
+ """
+ await client.request("DELETE", f"/documents/{document_id}")
+ return f"Deleted document {document_id}."
+
+
+def _read_upload(file_path: str) -> tuple[str, bytes, str]:
+ path = Path(file_path).expanduser()
+ if not path.is_file():
+ raise ToolError(f"No file at '{file_path}'.")
+ mime, _ = mimetypes.guess_type(path.name)
+ return (path.name, path.read_bytes(), mime or "application/octet-stream")
diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py
new file mode 100644
index 000000000..a9e60810d
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py
@@ -0,0 +1,188 @@
+"""Knowledge-base read tools: semantic search, list, and read one document.
+
+Search and list default to the active workspace; a document read is addressed by
+id, which is unique across the account, so it needs no workspace.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ...core.client import SurfSenseClient
+from ...core.rendering import ResponseFormatParam, clip, to_json
+from ...core.workspace_context import WorkspaceContext, WorkspaceParam
+from .annotations import READ, DocumentId, DocumentTypes
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the knowledge-base read tools."""
+
+ @mcp.tool(
+ name="surfsense_search_knowledge_base",
+ title="Search knowledge base",
+ annotations=READ,
+ structured_output=False,
+ )
+ async def search_knowledge_base(
+ query: Annotated[
+ str,
+ Field(
+ min_length=1,
+ description="Natural-language search, e.g. "
+ "'notebooklm user complaints'.",
+ ),
+ ],
+ top_k: Annotated[
+ int, Field(ge=1, le=20, description="Maximum documents to return.")
+ ] = 5,
+ document_types: DocumentTypes = None,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Search the workspace's knowledge base by meaning and keywords.
+
+ Use this FIRST when a question might be answered by content already
+ stored in SurfSense — notes, uploaded files, saved pages, past
+ research. Do NOT use it to fetch new data from the web; use the
+ scraper tools for that. Returns the most relevant documents with the
+ passages that matched, ranked by relevance score.
+ Example: query='pricing feedback', top_k=5.
+ """
+ resolved = await context.resolve(workspace)
+ hits = await client.request(
+ "POST",
+ "/documents/search-semantic",
+ json={
+ "workspace_id": resolved.id,
+ "query": query,
+ "top_k": max(1, min(top_k, 20)),
+ "document_types": document_types,
+ },
+ )
+ items = (hits or {}).get("items", [])
+ if response_format == "json":
+ return to_json(items)
+ return _render_search(query, items)
+
+ @mcp.tool(
+ name="surfsense_list_documents",
+ title="List documents",
+ annotations=READ,
+ structured_output=False,
+ )
+ async def list_documents(
+ document_types: DocumentTypes = None,
+ folder_id: Annotated[
+ int | None,
+ Field(description="Only documents in this folder. Omit for all."),
+ ] = None,
+ page: Annotated[
+ int, Field(ge=0, description="Zero-based page number.")
+ ] = 0,
+ page_size: Annotated[
+ int, Field(ge=1, description="Documents per page.")
+ ] = 20,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """List documents in the workspace's knowledge base, newest first.
+
+ Use this to browse or inventory what is stored; to find documents
+ about a topic, prefer surfsense_search_knowledge_base. Returns each
+ document's title, id, type, and update time, plus a has_more flag —
+ request the next page by increasing page.
+ Example: document_types=['FILE'], page=0, page_size=20.
+ """
+ resolved = await context.resolve(workspace)
+ result = await client.request(
+ "GET",
+ "/documents",
+ params={
+ "workspace_id": resolved.id,
+ "page": page,
+ "page_size": page_size,
+ "document_types": _join(document_types),
+ "folder_id": folder_id,
+ },
+ )
+ if response_format == "json":
+ return to_json(result)
+ return _render_document_list(result)
+
+ @mcp.tool(
+ name="surfsense_get_document",
+ title="Read one document",
+ annotations=READ,
+ structured_output=False,
+ )
+ async def get_document(
+ document_id: DocumentId,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Read one document's full content and metadata by id.
+
+ Use this after surfsense_search_knowledge_base or
+ surfsense_list_documents to open a specific document — search results
+ only include the matching passages, this returns the whole text.
+ """
+ document = await client.request("GET", f"/documents/{document_id}")
+ if response_format == "json":
+ return clip(to_json(document))
+ return _render_document(document)
+
+
+def _join(values: list[str] | None) -> str | None:
+ return ",".join(values) if values else None
+
+
+def _render_search(query: str, items: list[dict]) -> str:
+ if not items:
+ return f'No matches for "{query}".'
+ lines = [f'# {len(items)} result(s) for "{query}"', ""]
+ for hit in items:
+ lines.append(
+ f"## {hit.get('title', 'Untitled')} "
+ f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}"
+ )
+ for chunk in hit.get("chunks", []):
+ excerpt = clip(chunk.get("content", "").strip(), 500)
+ lines.append(f"> {excerpt}")
+ lines.append("")
+ return "\n".join(lines).strip()
+
+
+def _render_document_list(result: dict | None) -> str:
+ items = (result or {}).get("items", [])
+ if not items:
+ return "No documents found."
+ lines = ["# Documents", ""]
+ for doc in items:
+ lines.append(
+ f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · "
+ f"{doc.get('document_type')} · updated {doc.get('updated_at')}"
+ )
+ total = (result or {}).get("total", len(items))
+ page = (result or {}).get("page", 0)
+ has_more = (result or {}).get("has_more", False)
+ lines.append("")
+ lines.append(
+ f"_Page {page} · showing {len(items)} of {total}"
+ + (" · more available_" if has_more else "_")
+ )
+ return "\n".join(lines)
+
+
+def _render_document(document: dict) -> str:
+ content = clip(document.get("content", "") or "(empty)")
+ return (
+ f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n"
+ f"- type: {document.get('document_type')}\n"
+ f"- workspace: {document.get('workspace_id')}\n"
+ f"- updated: {document.get('updated_at')}\n\n"
+ f"{content}"
+ )
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py
index aa8265148..dfa2f3ab2 100644
--- a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py
@@ -1,570 +1,26 @@
"""Scraper tools: one MCP surface per SurfSense platform capability.
Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that
-maps a natural-language request to the workspace's scraper door. Two more tools
+maps a natural-language request to the workspace's scraper. Two run-history tools
list and fetch past runs, so a large result truncated inline can be retrieved in
-full later.
+full later. Each platform lives in its own module under platforms/.
"""
from __future__ import annotations
-from typing import Annotated, Literal
-
from mcp.server.fastmcp import FastMCP
-from mcp.types import ToolAnnotations
-from pydantic import Field
from ...core.client import SurfSenseClient
-from ...core.rendering import ResponseFormatParam, clip, to_json
-from ...core.workspace_context import WorkspaceContext, WorkspaceParam
-from .capability import run_scraper
+from ...core.workspace_context import WorkspaceContext
+from . import run_history
+from .platforms import google_maps, google_search, reddit, web, youtube
-# Scrapers reach the open web and record a billable run; they are neither
-# read-only nor idempotent, but they do not mutate the knowledge base.
-_SCRAPE = ToolAnnotations(
- readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True
-)
-_READ_RUNS = ToolAnnotations(
- readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False
-)
-
-RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"]
-RedditTime = Literal["hour", "day", "week", "month", "year", "all"]
-CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"]
-ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"]
+_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history)
def register(
mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
) -> None:
- """Register the scraper and run-history tools on the server."""
-
- @mcp.tool(
- name="surfsense_web_crawl",
- title="Crawl web pages",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def web_crawl(
- start_urls: Annotated[
- list[str],
- Field(
- min_length=1,
- description="Full URLs to fetch, e.g. "
- "['https://example.com/blog/post'].",
- ),
- ],
- max_crawl_depth: Annotated[
- int,
- Field(
- ge=0,
- description="Link-hops to follow from start_urls within the "
- "same site. 0 fetches only start_urls.",
- ),
- ] = 0,
- max_crawl_pages: Annotated[
- int, Field(ge=1, description="Stop after this many pages in total.")
- ] = 10,
- max_length: Annotated[
- int, Field(ge=1, description="Max characters kept per page.")
- ] = 50_000,
- include_url_patterns: Annotated[
- list[str] | None,
- Field(
- description="Regexes; only discovered links matching one are "
- "followed, e.g. ['/docs/.*']."
- ),
- ] = None,
- exclude_url_patterns: Annotated[
- list[str] | None,
- Field(description="Regexes; discovered links matching one are skipped."),
- ] = None,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Fetch specific web pages and return their cleaned content as markdown.
-
- Use this to read a page the user names, or to spider a site from a
- starting URL. Do NOT use it to find pages on a topic — use
- surfsense_google_search for discovery. Returns one item per crawled
- page: url, title, and the page text as markdown.
- Example: start_urls=['https://blog.example.com'], max_crawl_depth=1,
- include_url_patterns=['/2026/'].
- """
- return await run_scraper(
- client,
- context,
- platform="web",
- verb="crawl",
- payload={
- "startUrls": start_urls,
- "maxCrawlDepth": max_crawl_depth,
- "maxCrawlPages": max_crawl_pages,
- "maxLength": max_length,
- "includeUrlPatterns": include_url_patterns,
- "excludeUrlPatterns": exclude_url_patterns,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_google_search",
- title="Scrape Google Search",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def google_search(
- queries: Annotated[
- list[str],
- Field(
- min_length=1,
- description="Search terms or full Google Search URLs, e.g. "
- "['best rss readers 2026'].",
- ),
- ],
- max_pages_per_query: Annotated[
- int, Field(ge=1, description="Result pages to fetch per query.")
- ] = 1,
- country_code: Annotated[
- str | None,
- Field(description="Two-letter country to search from, e.g. 'us'."),
- ] = None,
- language_code: Annotated[
- str, Field(description="Results language, e.g. 'en'. Empty for default.")
- ] = "",
- site: Annotated[
- str | None,
- Field(
- description="Restrict results to one domain, e.g. 'example.com'."
- ),
- ] = None,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Scrape Google Search result pages for one or more queries.
-
- Use this to discover pages on the open web by topic; follow up with
- surfsense_web_crawl to read a result in full. Do NOT use it for
- Reddit, YouTube, or Google Maps research — the dedicated tools return
- richer data. Returns each query's parsed results: title, url, and
- snippet per organic result.
- Example: queries=['notebooklm review'], site='news.ycombinator.com'.
- """
- return await run_scraper(
- client,
- context,
- platform="google_search",
- verb="scrape",
- payload={
- "queries": queries,
- "max_pages_per_query": max_pages_per_query,
- "country_code": country_code,
- "language_code": language_code,
- "site": site,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_reddit_scrape",
- title="Search or scrape Reddit",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def reddit_scrape(
- urls: Annotated[
- list[str] | None,
- Field(
- description="Reddit URLs: a post, a subreddit like "
- "'https://reddit.com/r/LocalLLaMA', a user page, or a search "
- "URL. Provide urls OR search_queries."
- ),
- ] = None,
- search_queries: Annotated[
- list[str] | None,
- Field(
- description="Terms to search Reddit for, e.g. "
- "['NotebookLM alternatives']. Provide search_queries OR urls."
- ),
- ] = None,
- community: Annotated[
- str | None,
- Field(
- description="Restrict a search to one subreddit, name without "
- "'r/', e.g. 'ArtificialInteligence'."
- ),
- ] = None,
- sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new",
- time_filter: Annotated[
- RedditTime | None,
- Field(description="Time window; only valid with sort='top'."),
- ] = None,
- max_items: Annotated[
- int, Field(ge=1, description="Maximum posts to return.")
- ] = 10,
- skip_comments: Annotated[
- bool,
- Field(
- description="True fetches posts only (faster); False also "
- "fetches each post's comment thread."
- ),
- ] = False,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Search or scrape Reddit: posts, comments, subreddits, and users.
-
- Use this for ANY Reddit research — finding relevant subreddits or
- communities for a topic, top posts, or discussions — instead of a
- generic web search. Returns posts (title, text, score, subreddit, url)
- with comment threads unless skip_comments is set. Every post carries
- its subreddit, so to find communities for a topic, search posts and
- aggregate their subreddits.
- Example: search_queries=['NotebookLM'], sort='top', time_filter='month'.
- """
- return await run_scraper(
- client,
- context,
- platform="reddit",
- verb="scrape",
- payload={
- "urls": urls,
- "search_queries": search_queries,
- "community": community,
- "sort": sort,
- "time_filter": time_filter,
- "max_items": max_items,
- "skip_comments": skip_comments,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_youtube_scrape",
- title="Search or scrape YouTube",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def youtube_scrape(
- urls: Annotated[
- list[str] | None,
- Field(
- description="YouTube URLs: video, channel, playlist, shorts, "
- "or hashtag pages. Provide urls OR search_queries."
- ),
- ] = None,
- search_queries: Annotated[
- list[str] | None,
- Field(
- description="Terms to search YouTube for, e.g. "
- "['NotebookLM tutorial']. Provide search_queries OR urls."
- ),
- ] = None,
- max_results: Annotated[
- int, Field(ge=1, description="Maximum videos to return.")
- ] = 10,
- download_subtitles: Annotated[
- bool,
- Field(description="True also fetches each video's transcript."),
- ] = False,
- subtitles_language: Annotated[
- str, Field(description="Transcript language code, e.g. 'en'.")
- ] = "en",
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Search or scrape YouTube videos, optionally with transcripts.
-
- Use this for YouTube research: finding videos on a topic, or reading a
- video's details or transcript. For a video's comment section use
- surfsense_youtube_comments instead. Returns per-video metadata (title,
- channel, views, description, url) and, if requested, the transcript.
- Example: search_queries=['NotebookLM tutorial'], download_subtitles=True.
- """
- return await run_scraper(
- client,
- context,
- platform="youtube",
- verb="scrape",
- payload={
- "urls": urls,
- "search_queries": search_queries,
- "max_results": max_results,
- "download_subtitles": download_subtitles,
- "subtitles_language": subtitles_language,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_youtube_comments",
- title="Fetch YouTube comments",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def youtube_comments(
- urls: Annotated[
- list[str],
- Field(
- min_length=1,
- description="YouTube video URLs, e.g. "
- "['https://www.youtube.com/watch?v=abc123'].",
- ),
- ],
- max_comments: Annotated[
- int,
- Field(
- ge=1,
- description="Maximum comments per video, counting top-level "
- "comments and replies together.",
- ),
- ] = 20,
- sort_by: Annotated[
- CommentSort, Field(description="Comment ordering.")
- ] = "NEWEST_FIRST",
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Fetch the comments (and replies) on one or more YouTube videos.
-
- Use this when the user wants a video's discussion or audience reaction
- rather than the video itself; get video URLs from
- surfsense_youtube_scrape if you only have a topic. Returns comment
- text, author, likes, and replies.
- Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50.
- """
- return await run_scraper(
- client,
- context,
- platform="youtube",
- verb="comments",
- payload={
- "urls": urls,
- "max_comments": max_comments,
- "sort_by": sort_by,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_google_maps_scrape",
- title="Find places on Google Maps",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def google_maps_scrape(
- search_queries: Annotated[
- list[str] | None,
- Field(
- description="Place searches, e.g. ['coffee shops']. Provide "
- "search_queries OR urls OR place_ids."
- ),
- ] = None,
- urls: Annotated[
- list[str] | None,
- Field(description="Google Maps URLs of specific places."),
- ] = None,
- place_ids: Annotated[
- list[str] | None,
- Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."),
- ] = None,
- location: Annotated[
- str | None,
- Field(
- description="Geographic scope for a search, e.g. "
- "'Seattle, USA'."
- ),
- ] = None,
- max_places: Annotated[
- int, Field(ge=1, description="Maximum places to return.")
- ] = 10,
- include_details: Annotated[
- bool,
- Field(
- description="True adds opening hours and extra contact info "
- "(slower)."
- ),
- ] = False,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Find places on Google Maps by search, URL, or place id.
-
- Use this for local-business and location research: names, addresses,
- ratings, categories, coordinates, place ids. For a place's customer
- reviews use surfsense_google_maps_reviews instead.
- Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5.
- """
- return await run_scraper(
- client,
- context,
- platform="google_maps",
- verb="scrape",
- payload={
- "search_queries": search_queries,
- "urls": urls,
- "place_ids": place_ids,
- "location": location,
- "max_places": max_places,
- "include_details": include_details,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_google_maps_reviews",
- title="Fetch Google Maps reviews",
- annotations=_SCRAPE,
- structured_output=False,
- )
- async def google_maps_reviews(
- urls: Annotated[
- list[str] | None,
- Field(
- description="Google Maps URLs of places. Provide urls OR "
- "place_ids."
- ),
- ] = None,
- place_ids: Annotated[
- list[str] | None,
- Field(
- description="Google place ids from surfsense_google_maps_scrape."
- ),
- ] = None,
- max_reviews: Annotated[
- int, Field(ge=1, description="Maximum reviews per place.")
- ] = 20,
- sort_by: Annotated[
- ReviewSort, Field(description="Review ordering.")
- ] = "newest",
- language: Annotated[
- str, Field(description="Reviews language code, e.g. 'en'.")
- ] = "en",
- start_date: Annotated[
- str | None,
- Field(
- description="ISO date like '2026-01-01'; keeps only reviews on "
- "or after that day."
- ),
- ] = None,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Fetch customer reviews for Google Maps places by URL or place id.
-
- Use this to read feedback on specific places; get urls or place_ids
- from surfsense_google_maps_scrape first if you only have a name.
- Returns review text, rating, author, and date per review.
- Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'.
- """
- return await run_scraper(
- client,
- context,
- platform="google_maps",
- verb="reviews",
- payload={
- "urls": urls,
- "place_ids": place_ids,
- "max_reviews": max_reviews,
- "sort_by": sort_by,
- "language": language,
- "start_date": start_date,
- },
- workspace=workspace,
- response_format=response_format,
- )
-
- @mcp.tool(
- name="surfsense_list_scraper_runs",
- title="List past scraper runs",
- annotations=_READ_RUNS,
- structured_output=False,
- )
- async def list_scraper_runs(
- limit: Annotated[
- int, Field(ge=1, description="Maximum runs to list.")
- ] = 20,
- capability: Annotated[
- str | None,
- Field(
- description="Filter by capability slug, e.g. 'web.crawl' or "
- "'reddit.scrape'."
- ),
- ] = None,
- status: Annotated[
- str | None,
- Field(description="Filter by run status: 'success' or 'error'."),
- ] = None,
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """List recent scraper runs in the workspace, newest first.
-
- Use this to find the run_id of an earlier scrape — for example when an
- inline result was truncated — then fetch it in full with
- surfsense_get_scraper_run. Returns each run's id, capability, status,
- item count, and creation time.
- Example: capability='reddit.scrape', status='success'.
- """
- resolved = await context.resolve(workspace)
- runs = await client.request(
- "GET",
- f"/workspaces/{resolved.id}/scrapers/runs",
- params={
- "limit": limit,
- "capability": capability,
- "status": status,
- },
- )
- if response_format == "json":
- return to_json(runs)
- return _render_runs(runs)
-
- @mcp.tool(
- name="surfsense_get_scraper_run",
- title="Fetch one scraper run in full",
- annotations=_READ_RUNS,
- structured_output=False,
- )
- async def get_scraper_run(
- run_id: Annotated[
- str,
- Field(
- description="Run id from surfsense_list_scraper_runs or a "
- "prior scrape's output."
- ),
- ],
- workspace: WorkspaceParam = None,
- response_format: ResponseFormatParam = "markdown",
- ) -> str:
- """Fetch a single scraper run in full, including its stored output.
-
- Use this to retrieve the complete, untruncated result of an earlier
- scrape. Do NOT re-run a scraper just to recover a truncated result —
- fetch the stored run instead.
- """
- resolved = await context.resolve(workspace)
- run = await client.request(
- "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}"
- )
- if response_format == "json":
- return clip(to_json(run))
- return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```"
-
-
-def _render_runs(runs: list[dict] | None) -> str:
- if not runs:
- return "No scraper runs found."
- lines = ["# Scraper runs", ""]
- for run in runs:
- lines.append(
- f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · "
- f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}"
- )
- return "\n".join(lines)
+ """Register every scraper and run-history tool on the server."""
+ for module in _REGISTRARS:
+ module.register(mcp, client, context)
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py
new file mode 100644
index 000000000..cb872cc90
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py
@@ -0,0 +1,13 @@
+"""Tool-call policy hints shared across scraper tools."""
+
+from __future__ import annotations
+
+from mcp.types import ToolAnnotations
+
+SCRAPE = ToolAnnotations(
+ readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True
+)
+
+READ_RUNS = ToolAnnotations(
+ readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False
+)
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py
new file mode 100644
index 000000000..f61704380
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py
@@ -0,0 +1 @@
+"""One module per scraper platform; each exposes register(mcp, client, context)."""
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py
new file mode 100644
index 000000000..e1613ca4e
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py
@@ -0,0 +1,151 @@
+"""Google Maps scraper tools: places and reviews."""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"]
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the Google Maps place and review tools."""
+
+ @mcp.tool(
+ name="surfsense_google_maps_scrape",
+ title="Find places on Google Maps",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def google_maps_scrape(
+ search_queries: Annotated[
+ list[str] | None,
+ Field(
+ description="Place searches, e.g. ['coffee shops']. Provide "
+ "search_queries OR urls OR place_ids."
+ ),
+ ] = None,
+ urls: Annotated[
+ list[str] | None,
+ Field(description="Google Maps URLs of specific places."),
+ ] = None,
+ place_ids: Annotated[
+ list[str] | None,
+ Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."),
+ ] = None,
+ location: Annotated[
+ str | None,
+ Field(
+ description="Geographic scope for a search, e.g. "
+ "'Seattle, USA'."
+ ),
+ ] = None,
+ max_places: Annotated[
+ int, Field(ge=1, description="Maximum places to return.")
+ ] = 10,
+ include_details: Annotated[
+ bool,
+ Field(
+ description="True adds opening hours and extra contact info "
+ "(slower)."
+ ),
+ ] = False,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Find places on Google Maps by search, URL, or place id.
+
+ Use this for local-business and location research: names, addresses,
+ ratings, categories, coordinates, place ids. For a place's customer
+ reviews use surfsense_google_maps_reviews instead.
+ Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="google_maps",
+ verb="scrape",
+ payload={
+ "search_queries": search_queries,
+ "urls": urls,
+ "place_ids": place_ids,
+ "location": location,
+ "max_places": max_places,
+ "include_details": include_details,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
+
+ @mcp.tool(
+ name="surfsense_google_maps_reviews",
+ title="Fetch Google Maps reviews",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def google_maps_reviews(
+ urls: Annotated[
+ list[str] | None,
+ Field(
+ description="Google Maps URLs of places. Provide urls OR "
+ "place_ids."
+ ),
+ ] = None,
+ place_ids: Annotated[
+ list[str] | None,
+ Field(
+ description="Google place ids from surfsense_google_maps_scrape."
+ ),
+ ] = None,
+ max_reviews: Annotated[
+ int, Field(ge=1, description="Maximum reviews per place.")
+ ] = 20,
+ sort_by: Annotated[
+ ReviewSort, Field(description="Review ordering.")
+ ] = "newest",
+ language: Annotated[
+ str, Field(description="Reviews language code, e.g. 'en'.")
+ ] = "en",
+ start_date: Annotated[
+ str | None,
+ Field(
+ description="ISO date like '2026-01-01'; keeps only reviews on "
+ "or after that day."
+ ),
+ ] = None,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Fetch customer reviews for Google Maps places by URL or place id.
+
+ Use this to read feedback on specific places; get urls or place_ids
+ from surfsense_google_maps_scrape first if you only have a name.
+ Returns review text, rating, author, and date per review.
+ Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="google_maps",
+ verb="reviews",
+ payload={
+ "urls": urls,
+ "place_ids": place_ids,
+ "max_reviews": max_reviews,
+ "sort_by": sort_by,
+ "language": language,
+ "start_date": start_date,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py
new file mode 100644
index 000000000..cc1a1f8ed
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py
@@ -0,0 +1,79 @@
+"""Google Search scraper tool."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the Google Search tool."""
+
+ @mcp.tool(
+ name="surfsense_google_search",
+ title="Scrape Google Search",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def google_search(
+ queries: Annotated[
+ list[str],
+ Field(
+ min_length=1,
+ description="Search terms or full Google Search URLs, e.g. "
+ "['best rss readers 2026'].",
+ ),
+ ],
+ max_pages_per_query: Annotated[
+ int, Field(ge=1, description="Result pages to fetch per query.")
+ ] = 1,
+ country_code: Annotated[
+ str | None,
+ Field(description="Two-letter country to search from, e.g. 'us'."),
+ ] = None,
+ language_code: Annotated[
+ str, Field(description="Results language, e.g. 'en'. Empty for default.")
+ ] = "",
+ site: Annotated[
+ str | None,
+ Field(
+ description="Restrict results to one domain, e.g. 'example.com'."
+ ),
+ ] = None,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Scrape Google Search result pages for one or more queries.
+
+ Use this to discover pages on the open web by topic; follow up with
+ surfsense_web_crawl to read a result in full. Do NOT use it for
+ Reddit, YouTube, or Google Maps research — the dedicated tools return
+ richer data. Returns each query's parsed results: title, url, and
+ snippet per organic result.
+ Example: queries=['notebooklm review'], site='news.ycombinator.com'.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="google_search",
+ verb="scrape",
+ payload={
+ "queries": queries,
+ "max_pages_per_query": max_pages_per_query,
+ "country_code": country_code,
+ "language_code": language_code,
+ "site": site,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py
new file mode 100644
index 000000000..035193ebc
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py
@@ -0,0 +1,98 @@
+"""Reddit scraper tool."""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"]
+RedditTime = Literal["hour", "day", "week", "month", "year", "all"]
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the Reddit tool."""
+
+ @mcp.tool(
+ name="surfsense_reddit_scrape",
+ title="Search or scrape Reddit",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def reddit_scrape(
+ urls: Annotated[
+ list[str] | None,
+ Field(
+ description="Reddit URLs: a post, a subreddit like "
+ "'https://reddit.com/r/LocalLLaMA', a user page, or a search "
+ "URL. Provide urls OR search_queries."
+ ),
+ ] = None,
+ search_queries: Annotated[
+ list[str] | None,
+ Field(
+ description="Terms to search Reddit for, e.g. "
+ "['NotebookLM alternatives']. Provide search_queries OR urls."
+ ),
+ ] = None,
+ community: Annotated[
+ str | None,
+ Field(
+ description="Restrict a search to one subreddit, name without "
+ "'r/', e.g. 'ArtificialInteligence'."
+ ),
+ ] = None,
+ sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new",
+ time_filter: Annotated[
+ RedditTime | None,
+ Field(description="Time window; only valid with sort='top'."),
+ ] = None,
+ max_items: Annotated[
+ int, Field(ge=1, description="Maximum posts to return.")
+ ] = 10,
+ skip_comments: Annotated[
+ bool,
+ Field(
+ description="True fetches posts only (faster); False also "
+ "fetches each post's comment thread."
+ ),
+ ] = False,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Search or scrape Reddit: posts, comments, subreddits, and users.
+
+ Use this for ANY Reddit research — finding relevant subreddits or
+ communities for a topic, top posts, or discussions — instead of a
+ generic web search. Returns posts (title, text, score, subreddit, url)
+ with comment threads unless skip_comments is set. Every post carries
+ its subreddit, so to find communities for a topic, search posts and
+ aggregate their subreddits.
+ Example: search_queries=['NotebookLM'], sort='top', time_filter='month'.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="reddit",
+ verb="scrape",
+ payload={
+ "urls": urls,
+ "search_queries": search_queries,
+ "community": community,
+ "sort": sort,
+ "time_filter": time_filter,
+ "max_items": max_items,
+ "skip_comments": skip_comments,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py
new file mode 100644
index 000000000..9c24a4352
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py
@@ -0,0 +1,89 @@
+"""Web crawl scraper tool."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the web crawl tool."""
+
+ @mcp.tool(
+ name="surfsense_web_crawl",
+ title="Crawl web pages",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def web_crawl(
+ start_urls: Annotated[
+ list[str],
+ Field(
+ min_length=1,
+ description="Full URLs to fetch, e.g. "
+ "['https://example.com/blog/post'].",
+ ),
+ ],
+ max_crawl_depth: Annotated[
+ int,
+ Field(
+ ge=0,
+ description="Link-hops to follow from start_urls within the "
+ "same site. 0 fetches only start_urls.",
+ ),
+ ] = 0,
+ max_crawl_pages: Annotated[
+ int, Field(ge=1, description="Stop after this many pages in total.")
+ ] = 10,
+ max_length: Annotated[
+ int, Field(ge=1, description="Max characters kept per page.")
+ ] = 50_000,
+ include_url_patterns: Annotated[
+ list[str] | None,
+ Field(
+ description="Regexes; only discovered links matching one are "
+ "followed, e.g. ['/docs/.*']."
+ ),
+ ] = None,
+ exclude_url_patterns: Annotated[
+ list[str] | None,
+ Field(description="Regexes; discovered links matching one are skipped."),
+ ] = None,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Fetch specific web pages and return their cleaned content as markdown.
+
+ Use this to read a page the user names, or to spider a site from a
+ starting URL. Do NOT use it to find pages on a topic — use
+ surfsense_google_search for discovery. Returns one item per crawled
+ page: url, title, and the page text as markdown.
+ Example: start_urls=['https://blog.example.com'], max_crawl_depth=1,
+ include_url_patterns=['/2026/'].
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="web",
+ verb="crawl",
+ payload={
+ "startUrls": start_urls,
+ "maxCrawlDepth": max_crawl_depth,
+ "maxCrawlPages": max_crawl_pages,
+ "maxLength": max_length,
+ "includeUrlPatterns": include_url_patterns,
+ "excludeUrlPatterns": exclude_url_patterns,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py
new file mode 100644
index 000000000..5582c82bb
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py
@@ -0,0 +1,131 @@
+"""YouTube scraper tools: videos and comments."""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"]
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the YouTube video and comment tools."""
+
+ @mcp.tool(
+ name="surfsense_youtube_scrape",
+ title="Search or scrape YouTube",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def youtube_scrape(
+ urls: Annotated[
+ list[str] | None,
+ Field(
+ description="YouTube URLs: video, channel, playlist, shorts, "
+ "or hashtag pages. Provide urls OR search_queries."
+ ),
+ ] = None,
+ search_queries: Annotated[
+ list[str] | None,
+ Field(
+ description="Terms to search YouTube for, e.g. "
+ "['NotebookLM tutorial']. Provide search_queries OR urls."
+ ),
+ ] = None,
+ max_results: Annotated[
+ int, Field(ge=1, description="Maximum videos to return.")
+ ] = 10,
+ download_subtitles: Annotated[
+ bool,
+ Field(description="True also fetches each video's transcript."),
+ ] = False,
+ subtitles_language: Annotated[
+ str, Field(description="Transcript language code, e.g. 'en'.")
+ ] = "en",
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Search or scrape YouTube videos, optionally with transcripts.
+
+ Use this for YouTube research: finding videos on a topic, or reading a
+ video's details or transcript. For a video's comment section use
+ surfsense_youtube_comments instead. Returns per-video metadata (title,
+ channel, views, description, url) and, if requested, the transcript.
+ Example: search_queries=['NotebookLM tutorial'], download_subtitles=True.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="youtube",
+ verb="scrape",
+ payload={
+ "urls": urls,
+ "search_queries": search_queries,
+ "max_results": max_results,
+ "download_subtitles": download_subtitles,
+ "subtitles_language": subtitles_language,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
+
+ @mcp.tool(
+ name="surfsense_youtube_comments",
+ title="Fetch YouTube comments",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def youtube_comments(
+ urls: Annotated[
+ list[str],
+ Field(
+ min_length=1,
+ description="YouTube video URLs, e.g. "
+ "['https://www.youtube.com/watch?v=abc123'].",
+ ),
+ ],
+ max_comments: Annotated[
+ int,
+ Field(
+ ge=1,
+ description="Maximum comments per video, counting top-level "
+ "comments and replies together.",
+ ),
+ ] = 20,
+ sort_by: Annotated[
+ CommentSort, Field(description="Comment ordering.")
+ ] = "NEWEST_FIRST",
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Fetch the comments (and replies) on one or more YouTube videos.
+
+ Use this when the user wants a video's discussion or audience reaction
+ rather than the video itself; get video URLs from
+ surfsense_youtube_scrape if you only have a topic. Returns comment
+ text, author, likes, and replies.
+ Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="youtube",
+ verb="comments",
+ payload={
+ "urls": urls,
+ "max_comments": max_comments,
+ "sort_by": sort_by,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py
new file mode 100644
index 000000000..9274a1a69
--- /dev/null
+++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py
@@ -0,0 +1,112 @@
+"""Scraper run history: list past runs and fetch one in full.
+
+A scrape whose inline result was truncated is retrievable here by run id, so the
+model never re-runs a scraper just to recover output.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ...core.client import SurfSenseClient
+from ...core.rendering import ResponseFormatParam, clip, to_json
+from ...core.workspace_context import WorkspaceContext, WorkspaceParam
+from .annotations import READ_RUNS
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the run-history tools."""
+
+ @mcp.tool(
+ name="surfsense_list_scraper_runs",
+ title="List past scraper runs",
+ annotations=READ_RUNS,
+ structured_output=False,
+ )
+ async def list_scraper_runs(
+ limit: Annotated[
+ int, Field(ge=1, description="Maximum runs to list.")
+ ] = 20,
+ capability: Annotated[
+ str | None,
+ Field(
+ description="Filter by capability slug, e.g. 'web.crawl' or "
+ "'reddit.scrape'."
+ ),
+ ] = None,
+ status: Annotated[
+ str | None,
+ Field(description="Filter by run status: 'success' or 'error'."),
+ ] = None,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """List recent scraper runs in the workspace, newest first.
+
+ Use this to find the run_id of an earlier scrape — for example when an
+ inline result was truncated — then fetch it in full with
+ surfsense_get_scraper_run. Returns each run's id, capability, status,
+ item count, and creation time.
+ Example: capability='reddit.scrape', status='success'.
+ """
+ resolved = await context.resolve(workspace)
+ runs = await client.request(
+ "GET",
+ f"/workspaces/{resolved.id}/scrapers/runs",
+ params={
+ "limit": limit,
+ "capability": capability,
+ "status": status,
+ },
+ )
+ if response_format == "json":
+ return to_json(runs)
+ return _render_runs(runs)
+
+ @mcp.tool(
+ name="surfsense_get_scraper_run",
+ title="Fetch one scraper run in full",
+ annotations=READ_RUNS,
+ structured_output=False,
+ )
+ async def get_scraper_run(
+ run_id: Annotated[
+ str,
+ Field(
+ description="Run id from surfsense_list_scraper_runs or a "
+ "prior scrape's output."
+ ),
+ ],
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Fetch a single scraper run in full, including its stored output.
+
+ Use this to retrieve the complete, untruncated result of an earlier
+ scrape. Do NOT re-run a scraper just to recover a truncated result —
+ fetch the stored run instead.
+ """
+ resolved = await context.resolve(workspace)
+ run = await client.request(
+ "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}"
+ )
+ if response_format == "json":
+ return clip(to_json(run))
+ return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```"
+
+
+def _render_runs(runs: list[dict] | None) -> str:
+ if not runs:
+ return "No scraper runs found."
+ lines = ["# Scraper runs", ""]
+ for run in runs:
+ lines.append(
+ f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · "
+ f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}"
+ )
+ return "\n".join(lines)
From 2b9daead58dcd4f98108d779edd55c1dc19765b3 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 18:05:56 +0200
Subject: [PATCH 014/160] feat(mcp): add health endpoint with public-path auth
exemption
---
.../src/surfsense_mcp/core/auth/middleware.py | 8 +++++--
.../src/surfsense_mcp/core/transport/http.py | 21 +++++++++++++------
2 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
index 354ae0a89..8ba6ce939 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py
@@ -6,10 +6,13 @@ not reach the tool handler. A pure middleware binds the key in the request's own
task, from which the SDK's per-request handling inherits it.
Requests without a key are rejected here so no tool ever runs unauthenticated.
+Paths in ``public_paths`` (e.g. the health probe) skip the check entirely.
"""
from __future__ import annotations
+from collections.abc import Iterable
+
from starlette.datastructures import Headers
from starlette.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
@@ -21,11 +24,12 @@ from .identity import bind_api_key, unbind_api_key
class ApiKeyIdentityMiddleware:
"""Binds the per-request API key into the identity contextvar, or 401s."""
- def __init__(self, app: ASGIApp) -> None:
+ def __init__(self, app: ASGIApp, public_paths: Iterable[str] = ()) -> None:
self._app = app
+ self._public_paths = frozenset(public_paths)
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
- if scope["type"] != "http":
+ if scope["type"] != "http" or scope["path"] in self._public_paths:
await self._app(scope, receive, send)
return
diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py
index 2f2cf8f53..e375e8b98 100644
--- a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py
+++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py
@@ -1,23 +1,32 @@
-"""Assemble the streamable-http ASGI app for the remote transport.
+"""Wrap the SDK's MCP endpoint with identity + CORS for the remote transport.
-Wraps the SDK's MCP endpoint with the API-key identity middleware and CORS.
-CORS sits outermost so browser preflight (which carries no key) is answered
-before the identity middleware, and clients can read the ``Mcp-Session-Id``
-header the streamable-http protocol relies on.
+CORS is outermost so keyless browser preflight is answered before the identity
+middleware. ``/health`` is a public path, exempt from the key check.
"""
from __future__ import annotations
from mcp.server.fastmcp import FastMCP
+from starlette.requests import Request
+from starlette.responses import JSONResponse
from starlette.middleware.cors import CORSMiddleware
from starlette.types import ASGIApp
from ..auth.middleware import ApiKeyIdentityMiddleware
+HEALTH_PATH = "/health"
+
+
+async def _health(_request: Request) -> JSONResponse:
+ return JSONResponse({"status": "ok"})
+
def build_http_app(mcp: FastMCP) -> ASGIApp:
"""Return the MCP streamable-http app wrapped with identity + CORS."""
- app: ASGIApp = ApiKeyIdentityMiddleware(mcp.streamable_http_app())
+ mcp.custom_route(HEALTH_PATH, methods=["GET"])(_health)
+ app: ASGIApp = ApiKeyIdentityMiddleware(
+ mcp.streamable_http_app(), public_paths={HEALTH_PATH}
+ )
return CORSMiddleware(
app,
allow_origins=["*"],
From 9aef0c27a28181e34275e8b629f27d7ee94877f2 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 18:06:00 +0200
Subject: [PATCH 015/160] docs(mcp): document hosted /mcp remote endpoint
---
README.es.md | 2 +-
README.hi.md | 2 +-
README.md | 2 +-
README.pt-BR.md | 2 +-
README.zh-CN.md | 2 +-
surfsense_mcp/README.md | 68 +++++++++++++------
.../content/docs/how-to/mcp-server.mdx | 63 +++++++++--------
7 files changed, 88 insertions(+), 53 deletions(-)
diff --git a/README.es.md b/README.es.md
index 82cfcae1d..70ccd5eb1 100644
--- a/README.es.md
+++ b/README.es.md
@@ -138,7 +138,7 @@ Agrega el servidor MCP de SurfSense a Claude, Cursor o tu propio framework de ag
{
"mcpServers": {
"surfsense": {
- "url": "https://mcp.surfsense.com",
+ "url": "https://mcp.surfsense.com/mcp",
"headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" }
}
}
diff --git a/README.hi.md b/README.hi.md
index 1c9cfa4a7..cbf9ae8fa 100644
--- a/README.hi.md
+++ b/README.hi.md
@@ -138,7 +138,7 @@ SurfSense MCP सर्वर को Claude, Cursor या अपने एज
{
"mcpServers": {
"surfsense": {
- "url": "https://mcp.surfsense.com",
+ "url": "https://mcp.surfsense.com/mcp",
"headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" }
}
}
diff --git a/README.md b/README.md
index a05548d66..8f2e65118 100644
--- a/README.md
+++ b/README.md
@@ -138,7 +138,7 @@ Add the SurfSense MCP server to Claude, Cursor, or your own agent framework:
{
"mcpServers": {
"surfsense": {
- "url": "https://mcp.surfsense.com",
+ "url": "https://mcp.surfsense.com/mcp",
"headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" }
}
}
diff --git a/README.pt-BR.md b/README.pt-BR.md
index 4f2b945f4..076736bcf 100644
--- a/README.pt-BR.md
+++ b/README.pt-BR.md
@@ -138,7 +138,7 @@ Adicione o servidor MCP do SurfSense ao Claude, ao Cursor ou ao seu próprio fra
{
"mcpServers": {
"surfsense": {
- "url": "https://mcp.surfsense.com",
+ "url": "https://mcp.surfsense.com/mcp",
"headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" }
}
}
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 22410a727..7d2ae35c3 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -138,7 +138,7 @@ curl -X POST "$SURFSENSE_API_URL/workspaces/$WORKSPACE_ID/scrapers/reddit/scrape
{
"mcpServers": {
"surfsense": {
- "url": "https://mcp.surfsense.com",
+ "url": "https://mcp.surfsense.com/mcp",
"headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" }
}
}
diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md
index 47dabf55c..cc5ef19c5 100644
--- a/surfsense_mcp/README.md
+++ b/surfsense_mcp/README.md
@@ -2,9 +2,15 @@
A [Model Context Protocol](https://modelcontextprotocol.io/) server that exposes
SurfSense to MCP clients like **Claude Code**, **Cursor**, and **Claude Desktop**.
-It talks to a running SurfSense backend purely over its REST API using a SurfSense
-API key — it imports no backend code and can point at any instance (local or
-hosted) by changing two environment variables.
+It talks to a SurfSense backend purely over its REST API using a SurfSense API
+key — it imports no backend code.
+
+Connect it two ways:
+
+- **Hosted** (recommended) — point your client at `https://mcp.surfsense.com/mcp`
+ and pass your API key in a header. Nothing to install or keep running.
+- **Self-host (stdio)** — run the server yourself against any backend (cloud or
+ your own). Best for self-hosters and clients without remote-server support.
## Tools
@@ -29,15 +35,42 @@ Workspace-scoped tools default to the active workspace; pass `workspace` (a name
or id) to override for a single call. Ids never need to be typed by hand — the
model carries them between calls.
-## Prerequisites
+## Get an API key
-1. A running SurfSense backend (default `http://localhost:8000`).
-2. A **SurfSense API key**: SurfSense → Settings → API → create key (`ss_pat_…`).
-3. **API access enabled** on the workspace(s) you want to use (workspace settings).
+1. SurfSense → **API Playground → API Keys**: create a personal key (`ss_pat_…`).
+ It is shown only once.
+2. Toggle **API key access** on for the workspace(s) you want to use.
-## Setup
+## Connect (hosted)
-Uses [uv](https://github.com/astral-sh/uv):
+Point your client at the hosted server and send the key as a Bearer token. For
+clients that read an `mcpServers` map (Cursor, Claude Desktop, and others):
+
+```json
+{
+ "mcpServers": {
+ "surfsense": {
+ "url": "https://mcp.surfsense.com/mcp",
+ "headers": { "Authorization": "Bearer ss_pat_your_key_here" }
+ }
+ }
+}
+```
+
+Claude Code, from a terminal:
+
+```bash
+claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \
+ --header "Authorization: Bearer ss_pat_your_key_here"
+```
+
+Most MCP clients accept this `url` + `headers` form; check your client's docs for
+its exact remote-server field.
+
+## Self-host (stdio)
+
+Run the server yourself when you host your own backend or use a client without
+remote support. It uses [uv](https://github.com/astral-sh/uv):
```bash
cd surfsense_mcp
@@ -45,11 +78,8 @@ uv sync
uv run python -m surfsense_mcp.selfcheck # verify tools register correctly
```
-## Connect it to a client
-
-### Cursor
-
-Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`):
+Then add it to your client. Cursor (`~/.cursor/mcp.json` or a project
+`.cursor/mcp.json`):
```json
{
@@ -66,7 +96,7 @@ Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`):
}
```
-### Claude Code
+Claude Code:
```bash
claude mcp add surfsense \
@@ -75,15 +105,13 @@ claude mcp add surfsense \
-- uv run --directory /absolute/path/to/SurfSense/surfsense_mcp python -m surfsense_mcp
```
-### Claude Desktop
-
-Add the same `mcpServers` block as Cursor to
+Claude Desktop: add the same `mcpServers` block as Cursor to
`claude_desktop_config.json` (Settings → Developer → Edit Config).
## Configuration
-See `.env.example`. Secrets are passed as environment variables by the client;
-never commit tokens.
+See `.env.example`. For self-host, secrets are passed as environment variables by
+the client; never commit tokens.
## Backend dependency
diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx
index ffdb4fd56..62628a659 100644
--- a/surfsense_web/content/docs/how-to/mcp-server.mdx
+++ b/surfsense_web/content/docs/how-to/mcp-server.mdx
@@ -4,51 +4,58 @@ description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, C
---
import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
-import { Step, Steps } from 'fumadocs-ui/components/steps';
# SurfSense MCP Server
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
-It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable.
+Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.
-## Prerequisites
+## Create an API key
-
-
+You need a SurfSense API key either way. In SurfSense, open **API Playground → API Keys** in your workspace sidebar:
-### Install uv
+1. Toggle **API key access** on for the workspace.
+2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once.
-The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run:
+## Connect (hosted)
+
+The hosted server runs at `https://mcp.surfsense.com/mcp`. Point your client at it and send the key as a Bearer token — there is nothing to install and no backend to run. For clients that read an `mcpServers` map (Cursor, and others):
+
+```json
+{
+ "mcpServers": {
+ "surfsense": {
+ "url": "https://mcp.surfsense.com/mcp",
+ "headers": { "Authorization": "Bearer ss_pat_your_key_here" }
+ }
+ }
+}
+```
+
+Claude Code, from a terminal:
+
+```bash
+claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \
+ --header "Authorization: Bearer ss_pat_your_key_here"
+```
+
+Most MCP clients accept this `url` + `headers` form; check your client's docs for its exact remote-server field.
+
+## Self-host (stdio)
+
+Run the server yourself when you host your own backend or use a client without remote support. It runs with [uv](https://github.com/astral-sh/uv) — install it once, then from the SurfSense repository run:
```bash
cd surfsense_mcp
uv sync
```
-
-
-
-### Create an API key
-
-In SurfSense, open **API Playground → API Keys** in your workspace sidebar:
-
-1. Toggle **API key access** on for the workspace.
-2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once.
-
-
-
-
-### Know your base URL
+Point the server at your backend with `SURFSENSE_BASE_URL`:
- **SurfSense Cloud**: `https://api.surfsense.com`
- **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000`
-
-
-
-## Connect your agent
-
Every client below launches the same command — `uv run --directory /surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours.
@@ -221,7 +228,7 @@ Run `/mcp` inside Gemini CLI to confirm the server and its tools.
-The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up.
+In this mode your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. (The hosted server above needs none of this.)
## Test it
@@ -236,7 +243,7 @@ That calls `surfsense_list_workspaces` — the simplest end-to-end check of the
## Configuration reference
-All settings are environment variables passed by the client:
+For self-host (stdio), all settings are environment variables passed by the client. The hosted server needs only your API key in the `Authorization` header:
| Variable | Required | Default | Purpose |
|----------|----------|---------|---------|
From 1c5b08561d4505391f5e7ce624908f14c3fa9da3 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Tue, 7 Jul 2026 18:06:04 +0200
Subject: [PATCH 016/160] feat(web): add hosted and self-host MCP client
configs
---
surfsense_web/app/(home)/mcp-server/page.tsx | 21 +-
.../connectors-marketing/api-mcp-tabs.tsx | 2 +-
.../components/mcp/agent-setup-tabs.tsx | 102 +++--
surfsense_web/lib/mcp/clients.ts | 379 +++++++++++++-----
4 files changed, 350 insertions(+), 154 deletions(-)
diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx
index 6dfc69622..5e4961b1e 100644
--- a/surfsense_web/app/(home)/mcp-server/page.tsx
+++ b/surfsense_web/app/(home)/mcp-server/page.tsx
@@ -50,16 +50,13 @@ export const metadata: Metadata = {
},
};
-/* Mirrors surfsense_mcp/README.md: the real Cursor config. */
+/* The hosted Cursor config; mirrors lib/mcp/clients.ts. */
const CURSOR_CONFIG = `{
"mcpServers": {
"surfsense": {
- "command": "uv",
- "args": ["run", "--directory", ".../surfsense_mcp",
- "python", "-m", "surfsense_mcp"],
- "env": {
- "SURFSENSE_BASE_URL": "https://api.surfsense.com",
- "SURFSENSE_API_KEY": "ss_pat_..."
+ "url": "https://mcp.surfsense.com/mcp",
+ "headers": {
+ "Authorization": "Bearer ss_pat_..."
}
}
}
@@ -76,7 +73,7 @@ const STEPS = [
icon: TerminalSquare,
title: "Add the server to your client",
description:
- "Drop the config into Cursor's mcp.json, run claude mcp add for Claude Code, or paste it into Claude Desktop. Point it at the cloud or your own self-hosted instance.",
+ "Point your client at https://mcp.surfsense.com/mcp with your key in an Authorization header — the hosted config for Cursor, Claude Code, and others is one paste. Prefer stdio? Switch to Self-host and run it against your own backend.",
},
{
icon: Server,
@@ -135,7 +132,7 @@ const FAQ: FaqItem[] = [
{
question: "Which MCP clients does it work with?",
answer:
- "Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.",
+ "Any MCP client that speaks remote (streamable HTTP) or stdio. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI all have copy-paste configs on this page — Hosted for the one-paste https://mcp.surfsense.com/mcp endpoint, or Self-host for stdio against your own backend.",
},
{
question: "How is usage billed?",
@@ -278,9 +275,9 @@ export default function McpServerPage() {
Step-by-step setup for every agent
- Pick your client, follow its two steps, and paste the config. Replace the placeholder
- path with your surfsense_mcp checkout and the key with one from API Playground → API
- Keys — or grab a pre-filled config from the playground itself.
+ Pick your client, choose Hosted or Self-host, and
+ paste the config. Replace the key with one from API Playground → API Keys — or grab a
+ pre-filled config from the playground itself.
diff --git a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx
index 6ca19c47b..f25faf95b 100644
--- a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx
+++ b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx
@@ -206,7 +206,7 @@ function buildMcp({ mcpTool }: ApiSample): string {
const config = {
mcpServers: {
surfsense: {
- url: "https://mcp.surfsense.com",
+ url: "https://mcp.surfsense.com/mcp",
headers: { Authorization: "Bearer ${SURFSENSE_API_KEY}" },
},
},
diff --git a/surfsense_web/components/mcp/agent-setup-tabs.tsx b/surfsense_web/components/mcp/agent-setup-tabs.tsx
index 1701b8924..3790441f2 100644
--- a/surfsense_web/components/mcp/agent-setup-tabs.tsx
+++ b/surfsense_web/components/mcp/agent-setup-tabs.tsx
@@ -9,6 +9,8 @@ import {
DEFAULT_SERVER_DIR,
MCP_CLIENTS,
type McpSnippetOptions,
+ type McpTransport,
+ REMOTE_URL,
} from "@/lib/mcp/clients";
function CopyButton({ text }: { text: string }) {
@@ -38,48 +40,82 @@ function CopyButton({ text }: { text: string }) {
);
}
+const TRANSPORTS: { id: McpTransport; label: string; hint: string }[] = [
+ { id: "remote", label: "Hosted", hint: "mcp.surfsense.com — nothing to install" },
+ { id: "stdio", label: "Self-host", hint: "run the server against your own backend" },
+];
+
/**
- * Per-agent MCP setup instructions as tabs: pick a client, follow its steps,
- * copy its exact config. Used on the /mcp-server marketing page and in the
- * API playground; `options` fills in real values where the caller has them.
+ * Per-agent MCP setup instructions as tabs: pick a client, then Hosted or
+ * Self-host, and copy its exact config. Used on the /mcp-server marketing page
+ * and in the API playground; `options` fills in real values where the caller
+ * has them.
*/
export function AgentSetupTabs({ options }: { options?: Partial }) {
+ const [transport, setTransport] = useState("remote");
+
const resolved: McpSnippetOptions = {
- baseUrl: options?.baseUrl || "https://api.surfsense.com",
+ remoteUrl: options?.remoteUrl || REMOTE_URL,
apiKey: options?.apiKey || API_KEY_PLACEHOLDER,
+ baseUrl: options?.baseUrl || "https://api.surfsense.com",
serverDir: options?.serverDir || DEFAULT_SERVER_DIR,
};
+ const active = TRANSPORTS.find((t) => t.id === transport) ?? TRANSPORTS[0];
+
return (
-
-
- {MCP_CLIENTS.map((client) => (
-
- {client.label}
-
- ))}
-
- {MCP_CLIENTS.map((client) => {
- const config = client.buildConfig(resolved);
- return (
-
-
- {client.steps.map((step) => (
-
);
}
diff --git a/surfsense_web/lib/mcp/clients.ts b/surfsense_web/lib/mcp/clients.ts
index 923e99049..795832ad1 100644
--- a/surfsense_web/lib/mcp/clients.ts
+++ b/surfsense_web/lib/mcp/clients.ts
@@ -1,42 +1,75 @@
/**
- * MCP client setup catalog: one entry per popular agent, with the exact
- * config file, steps, and snippet needed to connect the SurfSense MCP server.
- * Shared by the marketing /mcp-server page and the API playground so the
- * instructions can never drift apart.
+ * MCP client setup catalog: one entry per popular agent, each with a hosted
+ * (remote) snippet and a self-host (stdio) snippet, plus the exact config file
+ * and steps. Shared by the marketing /mcp-server page and the API playground so
+ * the instructions can never drift apart.
+ *
+ * Remote snippets point at the hosted server and pass the key as a Bearer token;
+ * every client's exact remote field is verified against its own docs (Windsurf
+ * uses `serverUrl`, Gemini CLI `httpUrl`, VS Code needs `type: "http"`, OpenCode
+ * `type: "remote"` + `oauth: false`, Codex needs the rmcp flag, and Claude
+ * Desktop has no config-file remote support so it uses the `mcp-remote` bridge).
*/
+export type McpTransport = "remote" | "stdio";
+
export interface McpSnippetOptions {
- /** SurfSense backend URL the server should call. */
- baseUrl: string;
+ /** Hosted MCP endpoint (Bearer-authenticated). */
+ remoteUrl: string;
/** API key value or placeholder to show in the snippet. */
apiKey: string;
- /** Absolute path to the surfsense_mcp directory. */
+ /** SurfSense backend URL a self-hosted server should call. */
+ baseUrl: string;
+ /** Absolute path to the surfsense_mcp directory (self-host). */
serverDir: string;
}
+export interface McpSnippet {
+ /** Where the snippet goes: a file path or "Terminal". */
+ configFile: string;
+ language: "json" | "toml" | "bash";
+ steps: string[];
+ build: (options: McpSnippetOptions) => string;
+}
+
export interface McpClient {
id: string;
label: string;
- /** Where the snippet goes: a file path or "Terminal". */
- configFile: string;
- language: "json" | "toml" | "bash";
- steps: string[];
- buildConfig: (options: McpSnippetOptions) => string;
+ remote: McpSnippet;
+ stdio: McpSnippet;
}
+export const REMOTE_URL = "https://mcp.surfsense.com/mcp";
export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp";
export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here";
-function serverArgs(serverDir: string): string[] {
- return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"];
-}
-
function json(value: unknown): string {
return JSON.stringify(value, null, 2);
}
-/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */
-function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string {
+function bearer(apiKey: string): string {
+ return `Bearer ${apiKey}`;
+}
+
+function serverArgs(serverDir: string): string[] {
+ return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"];
+}
+
+/** The `mcpServers` remote shape shared by Cursor, Windsurf, and Gemini CLI. */
+function remoteMcpServers(urlField: "url" | "serverUrl" | "httpUrl") {
+ return ({ remoteUrl, apiKey }: McpSnippetOptions): string =>
+ json({
+ mcpServers: {
+ surfsense: {
+ [urlField]: remoteUrl,
+ headers: { Authorization: bearer(apiKey) },
+ },
+ },
+ });
+}
+
+/** The `mcpServers` stdio shape shared by Cursor, Claude Desktop, Windsurf, Gemini CLI. */
+function stdioMcpServers({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string {
return json({
mcpServers: {
surfsense: {
@@ -52,125 +85,255 @@ export const MCP_CLIENTS: McpClient[] = [
{
id: "claude-code",
label: "Claude Code",
- configFile: "Terminal",
- language: "bash",
- steps: [
- "Run this command in a terminal (any directory).",
- "Start Claude Code and run /mcp — surfsense should be listed as connected.",
- ],
- buildConfig: ({ baseUrl, apiKey, serverDir }) =>
- [
- "claude mcp add surfsense \\",
- ` -e SURFSENSE_BASE_URL=${baseUrl} \\`,
- ` -e SURFSENSE_API_KEY=${apiKey} \\`,
- ` -- uv run --directory ${serverDir} python -m surfsense_mcp`,
- ].join("\n"),
+ remote: {
+ configFile: "Terminal",
+ language: "bash",
+ steps: [
+ "Run this command in a terminal (any directory).",
+ "Start Claude Code and run /mcp — surfsense should be listed as connected.",
+ ],
+ build: ({ remoteUrl, apiKey }) =>
+ [
+ `claude mcp add --transport http surfsense ${remoteUrl} \\`,
+ ` --header "Authorization: ${bearer(apiKey)}"`,
+ ].join("\n"),
+ },
+ stdio: {
+ configFile: "Terminal",
+ language: "bash",
+ steps: [
+ "Run this command in a terminal (any directory).",
+ "Start Claude Code and run /mcp — surfsense should be listed as connected.",
+ ],
+ build: ({ baseUrl, apiKey, serverDir }) =>
+ [
+ "claude mcp add surfsense \\",
+ ` -e SURFSENSE_BASE_URL=${baseUrl} \\`,
+ ` -e SURFSENSE_API_KEY=${apiKey} \\`,
+ ` -- uv run --directory ${serverDir} python -m surfsense_mcp`,
+ ].join("\n"),
+ },
},
{
id: "codex",
label: "Codex",
- configFile: "~/.codex/config.toml",
- language: "toml",
- steps: [
- "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).",
- "Restart Codex; `codex mcp list` should show surfsense.",
- ],
- buildConfig: ({ baseUrl, apiKey, serverDir }) =>
- [
- "[mcp_servers.surfsense]",
- 'command = "uv"',
- `args = ${JSON.stringify(serverArgs(serverDir))}`,
- "",
- "[mcp_servers.surfsense.env]",
- `SURFSENSE_BASE_URL = "${baseUrl}"`,
- `SURFSENSE_API_KEY = "${apiKey}"`,
- ].join("\n"),
+ remote: {
+ configFile: "~/.codex/config.toml",
+ language: "toml",
+ steps: [
+ "Add this to ~/.codex/config.toml. The rmcp flag must sit above every [mcp_servers.*] table.",
+ "Restart Codex; `codex mcp list` should show surfsense.",
+ ],
+ build: ({ remoteUrl, apiKey }) =>
+ [
+ "experimental_use_rmcp_client = true",
+ "",
+ "[mcp_servers.surfsense]",
+ `url = "${remoteUrl}"`,
+ "",
+ "[mcp_servers.surfsense.http_headers]",
+ `Authorization = "${bearer(apiKey)}"`,
+ ].join("\n"),
+ },
+ stdio: {
+ configFile: "~/.codex/config.toml",
+ language: "toml",
+ steps: [
+ "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).",
+ "Restart Codex; `codex mcp list` should show surfsense.",
+ ],
+ build: ({ baseUrl, apiKey, serverDir }) =>
+ [
+ "[mcp_servers.surfsense]",
+ 'command = "uv"',
+ `args = ${JSON.stringify(serverArgs(serverDir))}`,
+ "",
+ "[mcp_servers.surfsense.env]",
+ `SURFSENSE_BASE_URL = "${baseUrl}"`,
+ `SURFSENSE_API_KEY = "${apiKey}"`,
+ ].join("\n"),
+ },
},
{
id: "opencode",
label: "OpenCode",
- configFile: "opencode.json",
- language: "json",
- steps: [
- "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).",
- "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.",
- ],
- buildConfig: ({ baseUrl, apiKey, serverDir }) =>
- json({
- $schema: "https://opencode.ai/config.json",
- mcp: {
- surfsense: {
- type: "local",
- command: ["uv", ...serverArgs(serverDir)],
- enabled: true,
- environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
+ remote: {
+ configFile: "opencode.json",
+ language: "json",
+ steps: [
+ "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).",
+ "`oauth: false` tells OpenCode to use the Bearer key instead of starting an OAuth flow.",
+ ],
+ build: ({ remoteUrl, apiKey }) =>
+ json({
+ $schema: "https://opencode.ai/config.json",
+ mcp: {
+ surfsense: {
+ type: "remote",
+ url: remoteUrl,
+ enabled: true,
+ oauth: false,
+ headers: { Authorization: bearer(apiKey) },
+ },
},
- },
- }),
+ }),
+ },
+ stdio: {
+ configFile: "opencode.json",
+ language: "json",
+ steps: [
+ "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).",
+ "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.",
+ ],
+ build: ({ baseUrl, apiKey, serverDir }) =>
+ json({
+ $schema: "https://opencode.ai/config.json",
+ mcp: {
+ surfsense: {
+ type: "local",
+ command: ["uv", ...serverArgs(serverDir)],
+ enabled: true,
+ environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
+ },
+ },
+ }),
+ },
},
{
id: "cursor",
label: "Cursor",
- configFile: "~/.cursor/mcp.json",
- language: "json",
- steps: [
- "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.",
- "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.",
- ],
- buildConfig: standardJson,
+ remote: {
+ configFile: "~/.cursor/mcp.json",
+ language: "json",
+ steps: [
+ "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.",
+ "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.",
+ ],
+ build: remoteMcpServers("url"),
+ },
+ stdio: {
+ configFile: "~/.cursor/mcp.json",
+ language: "json",
+ steps: [
+ "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.",
+ "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.",
+ ],
+ build: stdioMcpServers,
+ },
},
{
id: "claude-desktop",
label: "Claude Desktop",
- configFile: "claude_desktop_config.json",
- language: "json",
- steps: [
- "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.",
- "Restart Claude Desktop; surfsense appears under the tools icon.",
- ],
- buildConfig: standardJson,
+ remote: {
+ configFile: "claude_desktop_config.json",
+ language: "json",
+ steps: [
+ "Claude Desktop can't take a remote URL directly, so this uses the mcp-remote bridge (needs Node 18+).",
+ "Open Settings → Developer → Edit Config, add this, and restart Claude Desktop.",
+ ],
+ build: ({ remoteUrl, apiKey }) =>
+ json({
+ mcpServers: {
+ surfsense: {
+ command: "npx",
+ args: ["-y", "mcp-remote", remoteUrl, "--header", `Authorization: ${bearer(apiKey)}`],
+ },
+ },
+ }),
+ },
+ stdio: {
+ configFile: "claude_desktop_config.json",
+ language: "json",
+ steps: [
+ "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.",
+ "Restart Claude Desktop; surfsense appears under the tools icon.",
+ ],
+ build: stdioMcpServers,
+ },
},
{
id: "vscode",
label: "VS Code",
- configFile: ".vscode/mcp.json",
- language: "json",
- steps: [
- "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).",
- "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.",
- ],
- buildConfig: ({ baseUrl, apiKey, serverDir }) =>
- json({
- servers: {
- surfsense: {
- type: "stdio",
- command: "uv",
- args: serverArgs(serverDir),
- env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
+ remote: {
+ configFile: ".vscode/mcp.json",
+ language: "json",
+ steps: [
+ "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).",
+ "VS Code requires an explicit `type` field — `http` for the hosted server.",
+ ],
+ build: ({ remoteUrl, apiKey }) =>
+ json({
+ servers: {
+ surfsense: {
+ type: "http",
+ url: remoteUrl,
+ headers: { Authorization: bearer(apiKey) },
+ },
},
- },
- }),
+ }),
+ },
+ stdio: {
+ configFile: ".vscode/mcp.json",
+ language: "json",
+ steps: [
+ "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).",
+ "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.",
+ ],
+ build: ({ baseUrl, apiKey, serverDir }) =>
+ json({
+ servers: {
+ surfsense: {
+ type: "stdio",
+ command: "uv",
+ args: serverArgs(serverDir),
+ env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey },
+ },
+ },
+ }),
+ },
},
{
id: "windsurf",
label: "Windsurf",
- configFile: "~/.codeium/windsurf/mcp_config.json",
- language: "json",
- steps: [
- "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).",
- "Press the refresh button in the MCP panel to pick up the server.",
- ],
- buildConfig: standardJson,
+ remote: {
+ configFile: "~/.codeium/windsurf/mcp_config.json",
+ language: "json",
+ steps: [
+ "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).",
+ "Windsurf uses `serverUrl` (not `url`) for remote servers; press refresh in the MCP panel.",
+ ],
+ build: remoteMcpServers("serverUrl"),
+ },
+ stdio: {
+ configFile: "~/.codeium/windsurf/mcp_config.json",
+ language: "json",
+ steps: [
+ "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).",
+ "Press the refresh button in the MCP panel to pick up the server.",
+ ],
+ build: stdioMcpServers,
+ },
},
{
id: "gemini-cli",
label: "Gemini CLI",
- configFile: "~/.gemini/settings.json",
- language: "json",
- steps: [
- "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).",
- "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.",
- ],
- buildConfig: standardJson,
+ remote: {
+ configFile: "~/.gemini/settings.json",
+ language: "json",
+ steps: [
+ "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).",
+ "Gemini CLI uses `httpUrl` for streamable-HTTP servers; run /mcp to confirm surfsense.",
+ ],
+ build: remoteMcpServers("httpUrl"),
+ },
+ stdio: {
+ configFile: "~/.gemini/settings.json",
+ language: "json",
+ steps: [
+ "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).",
+ "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.",
+ ],
+ build: stdioMcpServers,
+ },
},
];
From ef3f9f2e25ea9661351adb712e2d4f7588af462a Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:39:58 +0200
Subject: [PATCH 017/160] feat(tiktok): item output schemas
---
.../proprietary/platforms/tiktok/__init__.py | 0
.../platforms/tiktok/schemas/__init__.py | 21 +++
.../platforms/tiktok/schemas/items.py | 132 ++++++++++++++++++
3 files changed, 153 insertions(+)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/__init__.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py
new file mode 100644
index 000000000..28326a544
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py
@@ -0,0 +1,21 @@
+"""Apify-compatible input and output contracts for the TikTok scraper."""
+
+from __future__ import annotations
+
+from .items import (
+ AuthorMeta,
+ CommentItem,
+ ErrorItem,
+ MusicMeta,
+ TikTokVideoItem,
+ VideoMeta,
+)
+
+__all__ = [
+ "AuthorMeta",
+ "CommentItem",
+ "ErrorItem",
+ "MusicMeta",
+ "TikTokVideoItem",
+ "VideoMeta",
+]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py
new file mode 100644
index 000000000..c4ff87618
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py
@@ -0,0 +1,132 @@
+# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API
+"""Output items keyed to the Clockworks TikTok actor's shape.
+
+Every model is open (``extra="allow"``) and defaults unsourced fields to
+``None``/``[]`` so the MVP can populate a reliable subset and expand the
+contract additively without breaking consumers.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AuthorMeta(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ id: str | None = None
+ name: str | None = None
+ nickName: str | None = None
+ profileUrl: str | None = None
+ verified: bool | None = None
+ signature: str | None = None
+ avatar: str | None = None
+ privateAccount: bool | None = None
+ fans: int | None = None
+ following: int | None = None
+ heart: int | None = None
+ video: int | None = None
+
+
+class MusicMeta(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ musicId: str | None = None
+ musicName: str | None = None
+ musicAuthor: str | None = None
+ musicOriginal: bool | None = None
+ playUrl: str | None = None
+
+
+class VideoMeta(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ height: int | None = None
+ width: int | None = None
+ duration: int | None = None
+ coverUrl: str | None = None
+ format: str | None = None
+ definition: str | None = None
+
+
+class TikTokVideoItem(BaseModel):
+ """A single scraped post (video or slideshow)."""
+
+ model_config = ConfigDict(extra="allow")
+
+ id: str | None = None
+ text: str | None = None
+ textLanguage: str | None = None
+ createTime: int | None = None
+ createTimeISO: str | None = None
+ isAd: bool | None = None
+
+ authorMeta: AuthorMeta = Field(default_factory=AuthorMeta)
+ musicMeta: MusicMeta = Field(default_factory=MusicMeta)
+ videoMeta: VideoMeta = Field(default_factory=VideoMeta)
+
+ webVideoUrl: str | None = None
+ mediaUrls: list[str] = Field(default_factory=list)
+
+ diggCount: int | None = None
+ shareCount: int | None = None
+ playCount: int | None = None
+ collectCount: int | None = None
+ commentCount: int | None = None
+
+ hashtags: list[dict[str, Any]] = Field(default_factory=list)
+ mentions: list[str] = Field(default_factory=list)
+
+ isSlideshow: bool | None = None
+ isPinned: bool | None = None
+ isSponsored: bool | None = None
+
+ scrapedAt: str | None = None
+
+ def to_output(self) -> dict[str, Any]:
+ return self.model_dump(exclude_none=False)
+
+
+class CommentItem(BaseModel):
+ """A single comment or reply under a post."""
+
+ model_config = ConfigDict(extra="allow")
+
+ id: str | None = None
+ text: str | None = None
+ videoWebUrl: str | None = None
+ diggCount: int | None = None
+ replyCommentTotal: int | None = None
+ createTime: int | None = None
+ createTimeISO: str | None = None
+
+ uid: str | None = None
+ uniqueId: str | None = None
+ nickName: str | None = None
+ avatar: str | None = None
+
+ repliesToId: str | None = None
+ scrapedAt: str | None = None
+
+ def to_output(self) -> dict[str, Any]:
+ return self.model_dump(exclude_none=False)
+
+
+class ErrorItem(BaseModel):
+ """Per-input failure, distinguished from normal items by ``errorCode``.
+
+ Mirrors the actor's convention so a private/deleted/empty target surfaces
+ as an item instead of silently vanishing from the results.
+ """
+
+ model_config = ConfigDict(extra="allow")
+
+ url: str | None = None
+ input: str | None = None
+ error: str | None = None
+ errorCode: str | None = None
+
+ def to_output(self) -> dict[str, Any]:
+ return self.model_dump(exclude_none=False)
From b2d387f994d415a37383d6d2b1e4ff0755e13eae Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:40:07 +0200
Subject: [PATCH 018/160] feat(tiktok): URL target resolver
---
.../platforms/tiktok/targets/__init__.py | 8 +++
.../platforms/tiktok/targets/resolver.py | 50 +++++++++++++++++++
.../platforms/tiktok/targets/types.py | 25 ++++++++++
.../tests/unit/platforms/tiktok/__init__.py | 0
.../platforms/tiktok/test_target_resolver.py | 47 +++++++++++++++++
5 files changed, 130 insertions(+)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/__init__.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py
new file mode 100644
index 000000000..ac7410154
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py
@@ -0,0 +1,8 @@
+"""TikTok URL classification into scrape targets."""
+
+from __future__ import annotations
+
+from .resolver import resolve_target
+from .types import TargetKind, TikTokTarget
+
+__all__ = ["TargetKind", "TikTokTarget", "resolve_target"]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py
new file mode 100644
index 000000000..ffd15ba2a
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py
@@ -0,0 +1,50 @@
+"""Classify a TikTok URL into a :class:`TikTokTarget`, or ``None``."""
+
+from __future__ import annotations
+
+from urllib.parse import parse_qs, unquote, urlparse
+
+from .types import SearchSection, TikTokTarget
+
+_TIKTOK_HOSTS = frozenset({"tiktok.com", "www.tiktok.com", "m.tiktok.com"})
+_SEARCH_SECTIONS: frozenset[SearchSection] = frozenset({"video", "user"})
+
+
+def _is_tiktok_host(hostname: str | None) -> bool:
+ return bool(hostname) and hostname.lower() in _TIKTOK_HOSTS
+
+
+def resolve_target(url: str) -> TikTokTarget | None:
+ parsed = urlparse(url)
+ if not _is_tiktok_host(parsed.hostname):
+ return None
+
+ segments = [s for s in (parsed.path or "").split("/") if s]
+ if not segments:
+ return None
+
+ # Profile / video live under /@username[...].
+ if segments[0].startswith("@"):
+ username = segments[0][1:]
+ if not username:
+ return None
+ if len(segments) >= 3 and segments[1] == "video" and segments[2]:
+ return TikTokTarget("video", segments[2], url, username=username)
+ return TikTokTarget("profile", username, url)
+
+ if segments[0] == "tag" and len(segments) >= 2 and segments[1]:
+ return TikTokTarget("hashtag", unquote(segments[1]), url)
+
+ if segments[0] == "search":
+ query = parse_qs(parsed.query).get("q", [None])[0]
+ if not query:
+ return None
+ section = segments[1] if len(segments) >= 2 else None
+ return TikTokTarget(
+ "search",
+ unquote(query),
+ url,
+ section=section if section in _SEARCH_SECTIONS else None,
+ )
+
+ return None
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py
new file mode 100644
index 000000000..c53d5239d
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py
@@ -0,0 +1,25 @@
+"""Scrape-target value object produced by URL classification."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+TargetKind = Literal["video", "profile", "hashtag", "search"]
+SearchSection = Literal["video", "user"]
+
+
+@dataclass(frozen=True, slots=True)
+class TikTokTarget:
+ """One classified scrape target.
+
+ ``value`` holds the kind-specific identifier: video id, username, hashtag
+ name, or search query. ``username`` is set for videos (needed to build the
+ canonical post URL). ``section`` narrows a search to videos or users.
+ """
+
+ kind: TargetKind
+ value: str
+ url: str
+ username: str | None = None
+ section: SearchSection | None = None
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/__init__.py b/surfsense_backend/tests/unit/platforms/tiktok/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py
new file mode 100644
index 000000000..c118f4f86
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py
@@ -0,0 +1,47 @@
+"""URL classification for the TikTok scraper (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.targets import resolve_target
+
+
+def test_resolve_video_carries_username_and_id():
+ target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173")
+ assert target is not None
+ assert target.kind == "video"
+ assert target.value == "6718335390845095173"
+ assert target.username == "scout2015"
+
+
+def test_resolve_profile():
+ target = resolve_target("https://www.tiktok.com/@scout2015")
+ assert target is not None
+ assert target.kind == "profile"
+ assert target.value == "scout2015"
+
+
+def test_resolve_hashtag():
+ target = resolve_target("https://www.tiktok.com/tag/funny")
+ assert target is not None
+ assert target.kind == "hashtag"
+ assert target.value == "funny"
+
+
+def test_resolve_search_top_video_and_user_sections():
+ top = resolve_target("https://www.tiktok.com/search?q=cats")
+ assert top is not None
+ assert top.kind == "search"
+ assert top.value == "cats"
+ assert top.section is None
+
+ videos = resolve_target("https://www.tiktok.com/search/video?q=cats")
+ assert videos is not None and videos.section == "video"
+
+ users = resolve_target("https://www.tiktok.com/search/user?q=cats")
+ assert users is not None and users.section == "user"
+
+
+def test_resolve_rejects_non_tiktok_and_unknown_paths():
+ assert resolve_target("https://example.com/@scout2015") is None
+ assert resolve_target("https://www.tiktok.com/") is None
+ assert resolve_target("https://www.tiktok.com/foundation") is None
From 8840fd2c53f1dfd10985069723f152898dbbbfb6 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:40:21 +0200
Subject: [PATCH 019/160] feat(tiktok): rehydration blob extractor
---
.../platforms/tiktok/extraction/__init__.py | 7 ++++
.../platforms/tiktok/extraction/hydration.py | 28 ++++++++++++++++
.../unit/platforms/tiktok/test_hydration.py | 32 +++++++++++++++++++
3 files changed, 67 insertions(+)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
new file mode 100644
index 000000000..3bd6b160d
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
@@ -0,0 +1,7 @@
+"""Turn raw TikTok page/API payloads into normalized items."""
+
+from __future__ import annotations
+
+from .hydration import extract_rehydration_data
+
+__all__ = ["extract_rehydration_data"]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py
new file mode 100644
index 000000000..4cc466849
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py
@@ -0,0 +1,28 @@
+"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML.
+
+TikTok server-renders the first page of data into a single script tag; parsing
+it yields page-one items (video/profile/hashtag) without any signed API call.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+_BLOB_RE = re.compile(
+ r'',
+ re.DOTALL,
+)
+
+
+def extract_rehydration_data(html: str) -> dict[str, Any] | None:
+ """Return the parsed rehydration blob, or ``None`` if absent/unparseable."""
+ match = _BLOB_RE.search(html)
+ if not match:
+ return None
+ try:
+ data = json.loads(match.group(1))
+ except (json.JSONDecodeError, ValueError):
+ return None
+ return data if isinstance(data, dict) else None
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py
new file mode 100644
index 000000000..cdca568f7
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py
@@ -0,0 +1,32 @@
+"""Rehydration-blob extraction from TikTok page HTML (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data
+
+_BLOB = (
+ '"
+)
+
+
+def test_extracts_default_scope_from_blob():
+ data = extract_rehydration_data(_BLOB)
+ assert data is not None
+ item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+ assert item["id"] == "123"
+
+
+def test_returns_none_when_blob_absent():
+ assert extract_rehydration_data("no blob here") is None
+
+
+def test_returns_none_when_blob_json_malformed():
+ broken = (
+ ''
+ )
+ assert extract_rehydration_data(broken) is None
From 0bbcf99852c20cb19c03a621149a304dfd2543c0 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:40:38 +0200
Subject: [PATCH 020/160] feat(tiktok): author normalizer
---
.../platforms/tiktok/extraction/__init__.py | 3 +-
.../platforms/tiktok/extraction/author.py | 31 +++++++++++++++++++
2 files changed, 33 insertions(+), 1 deletion(-)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
index 3bd6b160d..fa4df8731 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from .author import parse_author
from .hydration import extract_rehydration_data
-__all__ = ["extract_rehydration_data"]
+__all__ = ["extract_rehydration_data", "parse_author"]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py
new file mode 100644
index 000000000..414de39b6
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py
@@ -0,0 +1,31 @@
+"""Normalize TikTok author/profile payloads into an ``authorMeta`` dict."""
+
+from __future__ import annotations
+
+from typing import Any
+
+_PROFILE_URL = "https://www.tiktok.com/@{username}"
+
+
+def build_author_meta(author: dict[str, Any], stats: dict[str, Any]) -> dict[str, Any]:
+ """Map an author object + its stats to the ``authorMeta`` output shape."""
+ username = author.get("uniqueId")
+ return {
+ "id": author.get("id"),
+ "name": username,
+ "nickName": author.get("nickname"),
+ "profileUrl": _PROFILE_URL.format(username=username) if username else None,
+ "verified": author.get("verified"),
+ "signature": author.get("signature"),
+ "avatar": author.get("avatarLarger") or author.get("avatarMedium"),
+ "privateAccount": author.get("privateAccount"),
+ "fans": stats.get("followerCount"),
+ "following": stats.get("followingCount"),
+ "heart": stats.get("heartCount"),
+ "video": stats.get("videoCount"),
+ }
+
+
+def parse_author(user_info: dict[str, Any]) -> dict[str, Any]:
+ """Map a ``webapp.user-detail`` ``userInfo`` (``{user, stats}``) to authorMeta."""
+ return build_author_meta(user_info.get("user") or {}, user_info.get("stats") or {})
From ef66063bfc7e43bd329450fadca7e969df5aaf13 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:40:50 +0200
Subject: [PATCH 021/160] feat(tiktok): video normalizer and parser tests
---
.../platforms/tiktok/extraction/__init__.py | 3 +-
.../platforms/tiktok/extraction/timestamps.py | 18 ++++
.../platforms/tiktok/extraction/video.py | 73 +++++++++++++
.../unit/platforms/tiktok/test_parsers.py | 102 ++++++++++++++++++
4 files changed, 195 insertions(+), 1 deletion(-)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
index fa4df8731..9b4f71c68 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
@@ -4,5 +4,6 @@ from __future__ import annotations
from .author import parse_author
from .hydration import extract_rehydration_data
+from .video import parse_video
-__all__ = ["extract_rehydration_data", "parse_author"]
+__all__ = ["extract_rehydration_data", "parse_author", "parse_video"]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py
new file mode 100644
index 000000000..4b155c137
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py
@@ -0,0 +1,18 @@
+"""Millisecond-ISO timestamps matching the actor's output shape."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+
+def epoch_to_iso(seconds: int | None) -> str | None:
+ """Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``."""
+ if not seconds:
+ return None
+ stamp = datetime.fromtimestamp(seconds, tz=UTC)
+ return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def now_iso() -> str:
+ """Current UTC time in the millisecond-ISO output shape."""
+ return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py
new file mode 100644
index 000000000..1bfe66452
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py
@@ -0,0 +1,73 @@
+"""Normalize a raw TikTok ``itemStruct`` into a :class:`TikTokVideoItem` dict."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..schemas.items import TikTokVideoItem
+from .author import build_author_meta
+from .timestamps import epoch_to_iso
+
+_VIDEO_URL = "https://www.tiktok.com/@{username}/video/{video_id}"
+
+
+def _music_meta(music: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "musicId": music.get("id"),
+ "musicName": music.get("title"),
+ "musicAuthor": music.get("authorName"),
+ "musicOriginal": music.get("original"),
+ "playUrl": music.get("playUrl"),
+ }
+
+
+def _video_meta(video: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "height": video.get("height"),
+ "width": video.get("width"),
+ "duration": video.get("duration"),
+ "coverUrl": video.get("cover"),
+ "format": video.get("format"),
+ "definition": video.get("definition"),
+ }
+
+
+def _hashtags(challenges: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ return [
+ {"id": c.get("id"), "name": c.get("title")}
+ for c in challenges
+ if isinstance(c, dict)
+ ]
+
+
+def parse_video(item: dict[str, Any]) -> dict[str, Any]:
+ """Map an ``itemStruct`` to the flat output contract, filling known fields."""
+ author = item.get("author") or {}
+ author_stats = item.get("authorStats") or {}
+ stats = item.get("stats") or {}
+ username = author.get("uniqueId")
+ video_id = item.get("id")
+
+ web_url = (
+ _VIDEO_URL.format(username=username, video_id=video_id)
+ if username and video_id
+ else None
+ )
+ create_time = item.get("createTime")
+
+ return TikTokVideoItem(
+ id=video_id,
+ text=item.get("desc"),
+ createTime=create_time,
+ createTimeISO=epoch_to_iso(create_time),
+ authorMeta=build_author_meta(author, author_stats),
+ musicMeta=_music_meta(item.get("music") or {}),
+ videoMeta=_video_meta(item.get("video") or {}),
+ webVideoUrl=web_url,
+ diggCount=stats.get("diggCount"),
+ shareCount=stats.get("shareCount"),
+ playCount=stats.get("playCount"),
+ collectCount=stats.get("collectCount"),
+ commentCount=stats.get("commentCount"),
+ hashtags=_hashtags(item.get("challenges") or []),
+ ).to_output()
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py
new file mode 100644
index 000000000..03fa63133
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py
@@ -0,0 +1,102 @@
+"""Raw TikTok payload -> normalized item mapping (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import parse_author, parse_video
+
+_ITEM_STRUCT = {
+ "id": "7534061113365859586",
+ "desc": "haha #comeramabanana",
+ "createTime": 1_700_000_000,
+ "author": {
+ "id": "6733",
+ "uniqueId": "bruniela_",
+ "nickname": "Bruni",
+ "verified": False,
+ "signature": "bio here",
+ "avatarLarger": "https://cdn/avatar.jpg",
+ },
+ "authorStats": {
+ "followerCount": 51200,
+ "followingCount": 269,
+ "heartCount": 3_000_000,
+ "videoCount": 259,
+ },
+ "stats": {
+ "diggCount": 5344,
+ "shareCount": 701,
+ "playCount": 55700,
+ "commentCount": 24,
+ "collectCount": 291,
+ },
+ "music": {
+ "id": "7529",
+ "title": "som original",
+ "authorName": "fox_rus0",
+ "original": True,
+ "playUrl": "https://cdn/music.mp3",
+ },
+ "video": {
+ "height": 1024,
+ "width": 576,
+ "duration": 16,
+ "cover": "https://cdn/cover.jpg",
+ "format": "mp4",
+ "definition": "540p",
+ },
+ "challenges": [{"id": "4982299", "title": "comeramabanana"}],
+}
+
+
+def test_parse_video_maps_core_and_derived_fields():
+ item = parse_video(_ITEM_STRUCT)
+
+ assert item["id"] == "7534061113365859586"
+ assert item["text"] == "haha #comeramabanana"
+ assert item["createTimeISO"] == "2023-11-14T22:13:20.000Z"
+
+ assert item["authorMeta"]["name"] == "bruniela_"
+ assert item["authorMeta"]["nickName"] == "Bruni"
+ assert item["authorMeta"]["profileUrl"] == "https://www.tiktok.com/@bruniela_"
+ assert item["authorMeta"]["fans"] == 51200
+
+ assert item["musicMeta"]["musicName"] == "som original"
+ assert item["videoMeta"]["duration"] == 16
+
+ assert item["diggCount"] == 5344
+ assert item["playCount"] == 55700
+
+ assert item["hashtags"] == [{"id": "4982299", "name": "comeramabanana"}]
+ assert (
+ item["webVideoUrl"]
+ == "https://www.tiktok.com/@bruniela_/video/7534061113365859586"
+ )
+
+
+_USER_INFO = {
+ "user": {
+ "id": "6733",
+ "uniqueId": "bruniela_",
+ "nickname": "Bruni",
+ "verified": True,
+ "signature": "bio here",
+ "avatarLarger": "https://cdn/avatar.jpg",
+ "privateAccount": False,
+ },
+ "stats": {
+ "followerCount": 51200,
+ "followingCount": 269,
+ "heartCount": 3_000_000,
+ "videoCount": 259,
+ },
+}
+
+
+def test_parse_author_maps_user_and_stats():
+ author = parse_author(_USER_INFO)
+ assert author["name"] == "bruniela_"
+ assert author["nickName"] == "Bruni"
+ assert author["verified"] is True
+ assert author["profileUrl"] == "https://www.tiktok.com/@bruniela_"
+ assert author["fans"] == 51200
+ assert author["video"] == 259
From 5688ab0678b6571bd57ab98f20470270b2c8d8a2 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:51:17 +0200
Subject: [PATCH 022/160] feat(tiktok): scrape input schema
---
.../platforms/tiktok/schemas/__init__.py | 3 +
.../platforms/tiktok/schemas/input.py | 60 +++++++++++++++++++
.../tests/unit/platforms/tiktok/test_input.py | 26 ++++++++
3 files changed, 89 insertions(+)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_input.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py
index 28326a544..3593f0e28 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from .input import StartUrl, TikTokScrapeInput
from .items import (
AuthorMeta,
CommentItem,
@@ -16,6 +17,8 @@ __all__ = [
"CommentItem",
"ErrorItem",
"MusicMeta",
+ "StartUrl",
+ "TikTokScrapeInput",
"TikTokVideoItem",
"VideoMeta",
]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py
new file mode 100644
index 000000000..fa87772fe
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py
@@ -0,0 +1,60 @@
+# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API
+"""Input surface for the TikTok scraper, shaped to the Clockworks actor.
+
+Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first
+path does not yet act on (media downloads, follower add-ons) are still accepted
+via ``extra="allow"`` for contract parity and land inert.
+
+Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is
+caller policy applied by the collector, never baked into the flows.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+ProfileSorting = Literal["latest", "popular", "oldest"]
+ProfileSection = Literal["videos", "reposts"]
+SearchSection = Literal["", "/video", "/user"]
+
+
+class StartUrl(BaseModel):
+ """A single direct URL entry (``{"url": ...}``; extra keys ignored)."""
+
+ model_config = ConfigDict(extra="allow")
+
+ url: str
+
+
+class TikTokScrapeInput(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ # Discovery
+ startUrls: list[StartUrl] = Field(default_factory=list)
+ hashtags: list[str] = Field(default_factory=list)
+ profiles: list[str] = Field(default_factory=list)
+ searchQueries: list[str] = Field(default_factory=list)
+ postURLs: list[str] = Field(default_factory=list)
+
+ # Per-target count
+ resultsPerPage: int = Field(default=1, ge=1)
+
+ # Profile options
+ profileScrapeSections: list[ProfileSection] = Field(
+ default_factory=lambda: ["videos"]
+ )
+ profileSorting: ProfileSorting = "latest"
+ excludePinnedPosts: bool = False
+
+ # Search options
+ searchSection: SearchSection = ""
+ maxProfilesPerQuery: int = Field(default=10, ge=1)
+
+ # Incremental filters (ISO date or relative " days" per the actor)
+ oldestPostDateUnified: str | None = None
+ newestPostDate: str | None = None
+
+ # Proxy
+ proxyCountryCode: str = "None"
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_input.py b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py
new file mode 100644
index 000000000..c080ffec3
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py
@@ -0,0 +1,26 @@
+"""Input surface for the TikTok scraper (anonymous, Apify-shaped)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput
+
+
+def test_input_has_no_auth_fields():
+ forbidden = {"username", "password", "token", "login", "auth", "credentials"}
+ assert forbidden.isdisjoint(TikTokScrapeInput.model_fields)
+
+
+def test_input_defaults():
+ model = TikTokScrapeInput()
+ assert model.resultsPerPage == 1
+ assert model.profileSorting == "latest"
+ assert model.proxyCountryCode == "None"
+ assert model.hashtags == []
+ assert model.profiles == []
+ assert model.searchQueries == []
+ assert model.postURLs == []
+
+
+def test_input_allows_extra_inert_fields():
+ model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x")
+ assert model.model_dump().get("shouldDownloadVideos") is True
From 44b3e640d36c72423433881925f77c1abea92569 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 15:51:17 +0200
Subject: [PATCH 023/160] feat(tiktok): rotate-on-block fetch session
---
.../platforms/tiktok/session/__init__.py | 15 ++
.../platforms/tiktok/session/client.py | 130 +++++++++++++++++
.../platforms/tiktok/session/errors.py | 11 ++
.../platforms/tiktok/session/proxy.py | 113 +++++++++++++++
.../platforms/tiktok/test_fetch_resilience.py | 135 ++++++++++++++++++
5 files changed, 404 insertions(+)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/client.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py
new file mode 100644
index 000000000..659305055
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py
@@ -0,0 +1,15 @@
+"""Cookie-warmed, rotate-on-block proxy session and page-fetch seam."""
+
+from __future__ import annotations
+
+from .client import fetch_html
+from .errors import TikTokAccessBlockedError
+from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session
+
+__all__ = [
+ "TikTokAccessBlockedError",
+ "bind_proxy_holder",
+ "fetch_html",
+ "open_proxy_holder",
+ "proxy_session",
+]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py
new file mode 100644
index 000000000..e5be1e419
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py
@@ -0,0 +1,130 @@
+"""GET TikTok page HTML through a cookie-warmed, sticky-IP proxy session.
+
+The warm-up mints TikTok's anonymous device cookie (``ttwid``) on the first
+homepage hit; the target page then server-renders its rehydration blob. Rotates
+the residential IP and re-warms on 403, backs off on 429, and raises
+:class:`TikTokAccessBlockedError` only when every rotated IP refuses access.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from contextlib import suppress
+from typing import Any
+
+from scrapling.fetchers import AsyncFetcher
+
+from app.utils.proxy import get_proxy_url
+
+from .errors import TikTokAccessBlockedError
+from .proxy import _REQUEST_TIMEOUT_S, _current_session, proxy_session
+
+logger = logging.getLogger(__name__)
+
+# 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP.
+_ROTATE_STATUS = 403
+_BACKOFF_STATUS = 429
+_MAX_ROTATIONS = 3
+_MAX_BACKOFFS = 4
+_BACKOFF_BASE_S = 5.0
+
+_HOME_URL = "https://www.tiktok.com/"
+_TTWID_COOKIE = "ttwid"
+_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
+
+
+def _response_cookie_names(page: Any) -> set[str]:
+ cookies = getattr(page, "cookies", None)
+ return set(cookies.keys()) if isinstance(cookies, dict) else set()
+
+
+def _page_html(page: Any) -> str | None:
+ for attr in ("text", "body"):
+ val = getattr(page, attr, None)
+ if isinstance(val, bytes):
+ val = val.decode("utf-8", "replace")
+ if isinstance(val, str) and val.strip():
+ return val
+ return None
+
+
+async def warm_session(session: Any) -> bool:
+ """Mint an anonymous ``ttwid`` cookie; ``True`` if the session can now fetch."""
+ with suppress(Exception):
+ page = await session.get(_HOME_URL, headers=_HEADERS)
+ if _TTWID_COOKIE in _response_cookie_names(page):
+ return True
+ return False
+
+
+async def _get_page(session: Any, url: str) -> Any:
+ if session is not None:
+ return await session.get(url, headers=_HEADERS)
+ return await AsyncFetcher.get(
+ url,
+ headers=_HEADERS,
+ proxy=get_proxy_url(),
+ stealthy_headers=True,
+ timeout=_REQUEST_TIMEOUT_S,
+ )
+
+
+async def fetch_html(url: str) -> str | None:
+ """Return page HTML, or ``None`` on 404 / non-block failure."""
+ holder = _current_session.get()
+ if holder is None:
+ async with proxy_session():
+ return await fetch_html(url)
+
+ attempt = 0
+ backoffs = 0
+ while True:
+ session = holder.session
+ try:
+ if session is not None and not holder.warmed:
+ warmed_ok = await warm_session(session)
+ holder.warmed = True
+ if not warmed_ok:
+ if attempt < _MAX_ROTATIONS:
+ attempt += 1
+ await holder.rotate()
+ continue
+ raise TikTokAccessBlockedError(
+ f"could not warm session after {attempt} IP rotations: {url}"
+ )
+
+ await holder.pace()
+ page = await _get_page(session, url)
+ status = page.status
+
+ if status == 200:
+ return _page_html(page)
+ if status == 404:
+ return None
+ if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS:
+ backoffs += 1
+ delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1))
+ logger.warning("[tiktok] 429 on %s; backing off %.1fs", url, delay)
+ await asyncio.sleep(delay + random.uniform(0, 1))
+ continue
+ if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS:
+ attempt += 1
+ await holder.rotate()
+ continue
+ if status == _ROTATE_STATUS:
+ raise TikTokAccessBlockedError(
+ f"TikTok refused {url} on {attempt} rotated IPs (403)"
+ )
+ logger.warning("[tiktok] GET %s returned %s", url, status)
+ return None
+ except TikTokAccessBlockedError:
+ raise
+ except Exception as e:
+ logger.warning("[tiktok] GET %s failed: %s", url, e)
+ if attempt < _MAX_ROTATIONS:
+ attempt += 1
+ await holder.rotate()
+ continue
+ return None
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py
new file mode 100644
index 000000000..4d9fc712d
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py
@@ -0,0 +1,11 @@
+"""Fetch-seam errors surfaced to the capability layer."""
+
+from __future__ import annotations
+
+
+class TikTokAccessBlockedError(RuntimeError):
+ """Raised when every rotated IP is refused anonymous access.
+
+ Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather
+ than returning empty data. The route maps it to a 403.
+ """
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py
new file mode 100644
index 000000000..dca6c776b
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py
@@ -0,0 +1,113 @@
+"""Rotate-on-block sticky proxy session, bound per-flow via a ContextVar.
+
+Reusing one keep-alive connection pins a single residential exit IP so the
+warmed cookie jar (``ttwid``/``msToken``, bound to that IP) stays valid across
+the warm-up and every subsequent fetch. Ported from the Reddit sibling; the
+TikTok-specific warm-up lives in :mod:`client`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+from contextlib import asynccontextmanager, suppress
+from contextvars import ContextVar
+from typing import Any
+
+from scrapling.fetchers import FetcherSession
+
+from app.utils.proxy import get_proxy_url
+
+logger = logging.getLogger(__name__)
+
+# Pace each sticky IP so a fast exit can't burst past TikTok's per-IP threshold.
+_MIN_INTERVAL_S = 0.5
+_PACE_JITTER_S = 0.25
+# A healthy fetch lands in ~1-2s; cap a dead IP at one bounded wait before it
+# falls through to a rotation.
+_REQUEST_TIMEOUT_S = 15.0
+
+_current_session: ContextVar[_RotatingSession | None] = ContextVar(
+ "tiktok_proxy_session", default=None
+)
+
+
+class _RotatingSession:
+ """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP.
+
+ Used sequentially within a single flow (never shared across concurrent
+ tasks), so no locking is needed. ``session`` is ``None`` only when no proxy
+ is configured.
+ """
+
+ def __init__(self) -> None:
+ self._cm: Any | None = None
+ self.session: Any | None = None
+ self.rotations = 0
+ self.warmed = False
+ self._last_at = 0.0
+
+ async def _open(self) -> None:
+ proxy = get_proxy_url()
+ self.warmed = False
+ if proxy is None:
+ self._cm = self.session = None
+ return
+ self._cm = FetcherSession(
+ proxy=proxy,
+ stealthy_headers=True,
+ impersonate="chrome",
+ timeout=_REQUEST_TIMEOUT_S,
+ )
+ self.session = await self._cm.__aenter__()
+
+ async def close(self) -> None:
+ if self._cm is not None:
+ with suppress(Exception):
+ await self._cm.__aexit__(None, None, None)
+ self._cm = self.session = None
+
+ async def rotate(self) -> Any | None:
+ """Drop the current IP and connect through a fresh one."""
+ await self.close()
+ self.rotations += 1
+ await self._open()
+ logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations)
+ return self.session
+
+ async def pace(self) -> None:
+ """Sleep to hold this sticky IP under TikTok's per-IP rate threshold."""
+ wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at)
+ if wait > 0:
+ await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S))
+ self._last_at = time.monotonic()
+
+
+async def open_proxy_holder() -> _RotatingSession:
+ """Open a warm rotate-on-block session holder (caller owns ``close()``)."""
+ holder = _RotatingSession()
+ await holder._open()
+ return holder
+
+
+@asynccontextmanager
+async def bind_proxy_holder(holder: _RotatingSession):
+ """Route this task's fetches through ``holder`` for the enclosed block."""
+ token = _current_session.set(holder)
+ try:
+ yield holder
+ finally:
+ _current_session.reset(token)
+
+
+@asynccontextmanager
+async def proxy_session():
+ """Open one reused, rotate-on-block proxy session for a continuation chain."""
+ holder = await open_proxy_holder()
+ try:
+ async with bind_proxy_holder(holder):
+ yield holder
+ finally:
+ await holder.close()
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py
new file mode 100644
index 000000000..484157964
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py
@@ -0,0 +1,135 @@
+"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions).
+
+Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches
+deterministically; a live first IP normally warms and returns 200s.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.session import (
+ TikTokAccessBlockedError,
+ client,
+)
+from app.proprietary.platforms.tiktok.session.proxy import _current_session
+
+_HTML = "ok"
+
+
+class _FakePage:
+ def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML):
+ self.status = status
+ self.cookies = cookies or {}
+ self.body = body
+
+ @property
+ def text(self) -> str:
+ return self.body
+
+
+class _FakeSession:
+ """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``."""
+
+ def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML):
+ self.status = status
+ self.warms = warms
+ self.body = body
+ self.page_calls = 0
+ self.warm_calls = 0
+
+ async def get(self, url, headers=None, cookies=None):
+ if url.rstrip("/") == "https://www.tiktok.com":
+ self.warm_calls += 1
+ return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {})
+ self.page_calls += 1
+ return _FakePage(self.status, body=self.body)
+
+
+class _FakeHolder:
+ def __init__(self, sessions: list[_FakeSession]) -> None:
+ self._sessions = sessions
+ self.session = sessions[0]
+ self.rotations = 0
+ self.warmed = False
+
+ async def rotate(self):
+ self.rotations += 1
+ self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
+ self.warmed = False
+ return self.session
+
+ async def pace(self) -> None:
+ return None
+
+ async def close(self) -> None:
+ return None
+
+
+def _no_sleep(monkeypatch) -> None:
+ async def _noop(_seconds):
+ return None
+
+ monkeypatch.setattr(client.asyncio, "sleep", _noop)
+
+
+async def test_warms_then_returns_html():
+ holder = _FakeHolder([_FakeSession(200, warms=True)])
+ token = _current_session.set(holder)
+ try:
+ result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+ finally:
+ _current_session.reset(token)
+ assert result == _HTML
+ assert holder.rotations == 0
+ assert holder.session.warm_calls == 1
+
+
+async def test_rotates_when_warm_fails_then_succeeds():
+ holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)])
+ token = _current_session.set(holder)
+ try:
+ result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+ finally:
+ _current_session.reset(token)
+ assert result == _HTML
+ assert holder.rotations == 1
+
+
+async def test_404_returns_none_without_rotating():
+ holder = _FakeHolder([_FakeSession(404), _FakeSession(200)])
+ token = _current_session.set(holder)
+ try:
+ result = await client.fetch_html("https://www.tiktok.com/@missing")
+ finally:
+ _current_session.reset(token)
+ assert result is None
+ assert holder.rotations == 0
+
+
+async def test_rotates_and_rewarms_on_403():
+ holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)])
+ token = _current_session.set(holder)
+ try:
+ result = await client.fetch_html("https://www.tiktok.com/@scout2015")
+ finally:
+ _current_session.reset(token)
+ assert result == _HTML
+ assert holder.rotations == 1
+ assert holder.session.warm_calls == 1
+
+
+async def test_persistent_403_raises_blocked(monkeypatch):
+ _no_sleep(monkeypatch)
+ holder = _FakeHolder(
+ [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)]
+ )
+ token = _current_session.set(holder)
+ try:
+ raised = False
+ try:
+ await client.fetch_html("https://www.tiktok.com/@scout2015")
+ except TikTokAccessBlockedError:
+ raised = True
+ finally:
+ _current_session.reset(token)
+ assert raised
+ assert holder.rotations == client._MAX_ROTATIONS
From daa0856c443e547739b25cf82e50b80a42964237 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 16:07:07 +0200
Subject: [PATCH 024/160] feat(tiktok): blob-first orchestrator + video flow
---
.../proprietary/platforms/tiktok/__init__.py | 19 ++++
.../platforms/tiktok/extraction/__init__.py | 9 +-
.../platforms/tiktok/extraction/scopes.py | 30 +++++++
.../platforms/tiktok/flows/__init__.py | 8 ++
.../platforms/tiktok/flows/video.py | 26 ++++++
.../platforms/tiktok/orchestrator.py | 90 +++++++++++++++++++
.../platforms/tiktok/schemas/input.py | 9 +-
.../platforms/tiktok/session/errors.py | 3 +-
.../platforms/tiktok/test_orchestrator.py | 73 +++++++++++++++
.../unit/platforms/tiktok/test_scopes.py | 30 +++++++
10 files changed, 289 insertions(+), 8 deletions(-)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py
index e69de29bb..27867df73 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py
@@ -0,0 +1,19 @@
+"""Anonymous, blob-first TikTok scraper (public interface).
+
+The capability layer depends only on the names re-exported here: the input
+schema, the collector/generator, the video item shape, and the hard-block error.
+"""
+
+from __future__ import annotations
+
+from .orchestrator import iter_tiktok, scrape_tiktok
+from .schemas import TikTokScrapeInput, TikTokVideoItem
+from .session import TikTokAccessBlockedError
+
+__all__ = [
+ "TikTokAccessBlockedError",
+ "TikTokScrapeInput",
+ "TikTokVideoItem",
+ "iter_tiktok",
+ "scrape_tiktok",
+]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
index 9b4f71c68..0e6b1c3b7 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
@@ -4,6 +4,13 @@ from __future__ import annotations
from .author import parse_author
from .hydration import extract_rehydration_data
+from .scopes import user_info, video_item_struct
from .video import parse_video
-__all__ = ["extract_rehydration_data", "parse_author", "parse_video"]
+__all__ = [
+ "extract_rehydration_data",
+ "parse_author",
+ "parse_video",
+ "user_info",
+ "video_item_struct",
+]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py
new file mode 100644
index 000000000..a20a72b2b
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py
@@ -0,0 +1,30 @@
+"""Navigate the rehydration blob to the scopes the flows consume."""
+
+from __future__ import annotations
+
+from typing import Any
+
+_DEFAULT = "__DEFAULT_SCOPE__"
+
+
+def _scope(data: dict[str, Any], name: str) -> dict[str, Any] | None:
+ scope = (data.get(_DEFAULT) or {}).get(name)
+ return scope if isinstance(scope, dict) else None
+
+
+def video_item_struct(data: dict[str, Any]) -> dict[str, Any] | None:
+ """The ``itemStruct`` of a video-detail page, or ``None``."""
+ scope = _scope(data, "webapp.video-detail")
+ if not scope:
+ return None
+ item = (scope.get("itemInfo") or {}).get("itemStruct")
+ return item if isinstance(item, dict) else None
+
+
+def user_info(data: dict[str, Any]) -> dict[str, Any] | None:
+ """The ``userInfo`` (``{user, stats}``) of a profile page, or ``None``."""
+ scope = _scope(data, "webapp.user-detail")
+ if not scope:
+ return None
+ info = scope.get("userInfo")
+ return info if isinstance(info, dict) else None
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py
new file mode 100644
index 000000000..f2f4b67cc
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py
@@ -0,0 +1,8 @@
+"""Per-target scrape flows: resolved target -> normalized items."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable
+
+FetchFn = Callable[[str], Awaitable[str | None]]
+FlowResult = AsyncIterator[dict]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py
new file mode 100644
index 000000000..48e558677
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py
@@ -0,0 +1,26 @@
+"""Video-URL flow: fetch a post page, read its rehydration blob, emit one item."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import Any
+
+from ..extraction import extract_rehydration_data, parse_video, video_item_struct
+from ..extraction.timestamps import now_iso
+from ..targets.types import TikTokTarget
+from . import FetchFn
+
+
+async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
+ html = await fetch(target.url)
+ if not html:
+ return
+ data = extract_rehydration_data(html)
+ if not data:
+ return
+ item = video_item_struct(data)
+ if item is None:
+ return
+ out = parse_video(item)
+ out["scrapedAt"] = now_iso()
+ yield out
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
new file mode 100644
index 000000000..d924e3a9e
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
@@ -0,0 +1,90 @@
+"""Resolve a :class:`TikTokScrapeInput` into targets and stream their items.
+
+Targets run sequentially on one warm sticky IP; ``limit`` is collector policy
+applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to
+its flow via :func:`_dispatch`.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from typing import Any
+from urllib.parse import quote
+
+from .flows import FetchFn
+from .flows.video import iter_video
+from .schemas import TikTokScrapeInput
+from .session import fetch_html, proxy_session
+from .targets import resolve_target
+from .targets.types import TikTokTarget
+
+logger = logging.getLogger(__name__)
+
+_PROFILE_URL = "https://www.tiktok.com/@{name}"
+_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
+_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
+
+
+async def _empty() -> AsyncIterator[dict[str, Any]]:
+ for _ in ():
+ yield {}
+
+
+def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
+ """Build the target list from every input source, dropping unresolved URLs."""
+ targets: list[TikTokTarget] = []
+ for entry in input_model.startUrls:
+ resolved = resolve_target(entry.url)
+ if resolved is not None:
+ targets.append(resolved)
+ for url in input_model.postURLs:
+ resolved = resolve_target(url)
+ if resolved is not None:
+ targets.append(resolved)
+ for profile in input_model.profiles:
+ name = profile.lstrip("@")
+ targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
+ for tag in input_model.hashtags:
+ targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
+ for query in input_model.searchQueries:
+ targets.append(
+ TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query)))
+ )
+ return targets
+
+
+def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
+ if target.kind == "video":
+ return iter_video(target, fetch=fetch)
+ # Listings come from the signed item_list API, not the blob.
+ logger.debug("[tiktok] no blob flow for %s target", target.kind)
+ return _empty()
+
+
+async def iter_tiktok(
+ input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html
+) -> AsyncIterator[dict[str, Any]]:
+ """Yield normalized items for every resolved target, in order."""
+ async with proxy_session():
+ for target in _resolve_targets(input_model):
+ async for item in _dispatch(target, fetch=fetch):
+ yield item
+
+
+async def scrape_tiktok(
+ input_model: TikTokScrapeInput,
+ *,
+ limit: int | None = None,
+ fetch: FetchFn = fetch_html,
+) -> list[dict[str, Any]]:
+ """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``."""
+ from app.capabilities.core.progress import emit_progress
+
+ results: list[dict[str, Any]] = []
+ async for item in iter_tiktok(input_model, fetch=fetch):
+ results.append(item)
+ emit_progress("scraping", current=len(results), total=limit, unit="item")
+ if limit is not None and len(results) >= limit:
+ break
+ return results
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py
index fa87772fe..f710eed72 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py
@@ -1,12 +1,11 @@
# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API
"""Input surface for the TikTok scraper, shaped to the Clockworks actor.
-Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first
-path does not yet act on (media downloads, follower add-ons) are still accepted
-via ``extra="allow"`` for contract parity and land inert.
+Anonymous only: no auth-shaped field exists here. ``extra="allow"`` keeps the
+contract in parity with the actor for fields the scraper does not read.
-Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is
-caller policy applied by the collector, never baked into the flows.
+``resultsPerPage`` is a per-target count; the cross-target ceiling is caller
+policy applied by the collector, never baked into a flow.
"""
from __future__ import annotations
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py
index 4d9fc712d..708cb0212 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py
@@ -6,6 +6,5 @@ from __future__ import annotations
class TikTokAccessBlockedError(RuntimeError):
"""Raised when every rotated IP is refused anonymous access.
- Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather
- than returning empty data. The route maps it to a 403.
+ Distinguishes a hard block from an empty result; the route maps it to 403.
"""
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py
new file mode 100644
index 000000000..85c601047
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py
@@ -0,0 +1,73 @@
+"""End-to-end orchestration over a fake fetch (no network).
+
+Drives the public collector: input -> target -> blob-first flow -> items.
+"""
+
+from __future__ import annotations
+
+import json
+
+from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+
+def _video_page(video_id: str, username: str) -> str:
+ blob = {
+ "__DEFAULT_SCOPE__": {
+ "webapp.video-detail": {
+ "itemInfo": {
+ "itemStruct": {
+ "id": video_id,
+ "desc": "hello",
+ "author": {"uniqueId": username},
+ "stats": {"diggCount": 5},
+ }
+ }
+ }
+ }
+ }
+ return (
+ ''
+ )
+
+
+async def test_scrape_video_url_returns_parsed_item():
+ url = "https://www.tiktok.com/@scout2015/video/123"
+
+ async def fake_fetch(_url: str) -> str:
+ return _video_page("123", "scout2015")
+
+ items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch)
+
+ assert len(items) == 1
+ assert items[0]["id"] == "123"
+ assert items[0]["diggCount"] == 5
+ assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123"
+ assert items[0]["scrapedAt"] is not None
+
+
+async def test_scrape_honors_limit_across_targets():
+ urls = [
+ "https://www.tiktok.com/@a/video/1",
+ "https://www.tiktok.com/@b/video/2",
+ ]
+
+ async def fake_fetch(url: str) -> str:
+ vid = url.rsplit("/", 1)[1]
+ user = url.split("@")[1].split("/")[0]
+ return _video_page(vid, user)
+
+ items = await scrape_tiktok(
+ TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch
+ )
+ assert len(items) == 1
+
+
+async def test_scrape_skips_unrecognized_urls():
+ async def fake_fetch(_url: str) -> str:
+ return ""
+
+ items = await scrape_tiktok(
+ TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch
+ )
+ assert items == []
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py
new file mode 100644
index 000000000..7a52dd60d
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py
@@ -0,0 +1,30 @@
+"""Navigating the rehydration blob to its useful scopes (pure, no network)."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct
+
+
+def test_video_item_struct_navigates_video_detail_scope():
+ data = {
+ "__DEFAULT_SCOPE__": {
+ "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}}
+ }
+ }
+ item = video_item_struct(data)
+ assert item == {"id": "123"}
+
+
+def test_user_info_navigates_user_detail_scope():
+ data = {
+ "__DEFAULT_SCOPE__": {
+ "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}}
+ }
+ }
+ info = user_info(data)
+ assert info == {"user": {"uniqueId": "scout2015"}}
+
+
+def test_scopes_return_none_when_absent():
+ assert video_item_struct({}) is None
+ assert user_info({"__DEFAULT_SCOPE__": {}}) is None
From ad8b73477a281a175cd579c12105f5ed02245207 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 16:37:36 +0200
Subject: [PATCH 025/160] feat(tiktok): browser-driven signed listings
---
.../platforms/tiktok/extraction/__init__.py | 2 +
.../platforms/tiktok/extraction/item_list.py | 30 ++++++
.../platforms/tiktok/flows/__init__.py | 5 +
.../platforms/tiktok/flows/listing.py | 37 ++++++++
.../platforms/tiktok/orchestrator.py | 41 +++++----
.../platforms/tiktok/session/__init__.py | 2 +
.../platforms/tiktok/session/listing.py | 92 +++++++++++++++++++
.../unit/platforms/tiktok/test_item_list.py | 25 +++++
.../platforms/tiktok/test_orchestrator.py | 24 +++++
9 files changed, 240 insertions(+), 18 deletions(-)
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py
create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
index 0e6b1c3b7..d70a494a5 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py
@@ -4,11 +4,13 @@ from __future__ import annotations
from .author import parse_author
from .hydration import extract_rehydration_data
+from .item_list import items_from_response
from .scopes import user_info, video_item_struct
from .video import parse_video
__all__ = [
"extract_rehydration_data",
+ "items_from_response",
"parse_author",
"parse_video",
"user_info",
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py
new file mode 100644
index 000000000..7dddf891f
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py
@@ -0,0 +1,30 @@
+"""Item structs from a captured ``item_list`` / search API response body.
+
+Profile and hashtag listings return ``{"itemList": [...]}``; search returns
+``{"data": [{"item": {...}}]}``. Both element shapes are the same itemStruct
+:func:`parse_video` already consumes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def items_from_response(body: Any) -> list[dict[str, Any]]:
+ """Return the itemStructs carried by one API response, or ``[]``."""
+ if not isinstance(body, dict):
+ return []
+
+ item_list = body.get("itemList")
+ if isinstance(item_list, list):
+ return [i for i in item_list if isinstance(i, dict)]
+
+ data = body.get("data")
+ if isinstance(data, list):
+ return [
+ entry["item"]
+ for entry in data
+ if isinstance(entry, dict) and isinstance(entry.get("item"), dict)
+ ]
+
+ return []
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py
index f2f4b67cc..a7809567d 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py
@@ -5,4 +5,9 @@ from __future__ import annotations
from collections.abc import AsyncIterator, Awaitable, Callable
FetchFn = Callable[[str], Awaitable[str | None]]
+"""Fetch a page's HTML by URL (blob-first video flow)."""
+
+FetchListingFn = Callable[[str, int], Awaitable[list[dict]]]
+"""Load a listing page and return up to ``count`` captured itemStructs."""
+
FlowResult = AsyncIterator[dict]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py
new file mode 100644
index 000000000..44fcc33e2
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py
@@ -0,0 +1,37 @@
+"""Listing flow shared by profile, hashtag, and search targets.
+
+The browser seam returns raw itemStructs captured from the signed ``item_list``
+XHRs; this maps each to the output contract, drops duplicate video ids, and
+stops at the per-target ``cap``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import Any
+
+from ..extraction import parse_video
+from ..extraction.timestamps import now_iso
+from ..targets.types import TikTokTarget
+from . import FetchListingFn
+
+
+async def iter_listing(
+ target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn
+) -> AsyncIterator[dict[str, Any]]:
+ if cap <= 0:
+ return
+ seen: set[str] = set()
+ emitted = 0
+ for item in await fetch_listing(target.url, cap):
+ out = parse_video(item)
+ video_id = out.get("id")
+ if video_id is not None:
+ if video_id in seen:
+ continue
+ seen.add(video_id)
+ out["scrapedAt"] = now_iso()
+ yield out
+ emitted += 1
+ if emitted >= cap:
+ return
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
index d924e3a9e..c7f2e9008 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
@@ -2,35 +2,29 @@
Targets run sequentially on one warm sticky IP; ``limit`` is collector policy
applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to
-its flow via :func:`_dispatch`.
+its flow via :func:`_dispatch`: video URLs read the rehydration blob over HTTP,
+listings capture signed item_list XHRs through the stealth browser.
"""
from __future__ import annotations
-import logging
from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import quote
-from .flows import FetchFn
+from .flows import FetchFn, FetchListingFn
+from .flows.listing import iter_listing
from .flows.video import iter_video
from .schemas import TikTokScrapeInput
-from .session import fetch_html, proxy_session
+from .session import fetch_html, fetch_item_list, proxy_session
from .targets import resolve_target
from .targets.types import TikTokTarget
-logger = logging.getLogger(__name__)
-
_PROFILE_URL = "https://www.tiktok.com/@{name}"
_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
-async def _empty() -> AsyncIterator[dict[str, Any]]:
- for _ in ():
- yield {}
-
-
def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
"""Build the target list from every input source, dropping unresolved URLs."""
targets: list[TikTokTarget] = []
@@ -54,21 +48,31 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
return targets
-def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]:
+def _dispatch(
+ target: TikTokTarget,
+ *,
+ cap: int,
+ fetch: FetchFn,
+ fetch_listing: FetchListingFn,
+) -> AsyncIterator[dict[str, Any]]:
if target.kind == "video":
return iter_video(target, fetch=fetch)
- # Listings come from the signed item_list API, not the blob.
- logger.debug("[tiktok] no blob flow for %s target", target.kind)
- return _empty()
+ return iter_listing(target, cap=cap, fetch_listing=fetch_listing)
async def iter_tiktok(
- input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html
+ input_model: TikTokScrapeInput,
+ *,
+ fetch: FetchFn = fetch_html,
+ fetch_listing: FetchListingFn = fetch_item_list,
) -> AsyncIterator[dict[str, Any]]:
"""Yield normalized items for every resolved target, in order."""
+ cap = input_model.resultsPerPage
async with proxy_session():
for target in _resolve_targets(input_model):
- async for item in _dispatch(target, fetch=fetch):
+ async for item in _dispatch(
+ target, cap=cap, fetch=fetch, fetch_listing=fetch_listing
+ ):
yield item
@@ -77,12 +81,13 @@ async def scrape_tiktok(
*,
limit: int | None = None,
fetch: FetchFn = fetch_html,
+ fetch_listing: FetchListingFn = fetch_item_list,
) -> list[dict[str, Any]]:
"""Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``."""
from app.capabilities.core.progress import emit_progress
results: list[dict[str, Any]] = []
- async for item in iter_tiktok(input_model, fetch=fetch):
+ async for item in iter_tiktok(input_model, fetch=fetch, fetch_listing=fetch_listing):
results.append(item)
emit_progress("scraping", current=len(results), total=limit, unit="item")
if limit is not None and len(results) >= limit:
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py
index 659305055..409aa5987 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py
@@ -4,12 +4,14 @@ from __future__ import annotations
from .client import fetch_html
from .errors import TikTokAccessBlockedError
+from .listing import fetch_item_list
from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session
__all__ = [
"TikTokAccessBlockedError",
"bind_proxy_holder",
"fetch_html",
+ "fetch_item_list",
"open_proxy_holder",
"proxy_session",
]
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
new file mode 100644
index 000000000..0d19281a2
--- /dev/null
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
@@ -0,0 +1,92 @@
+"""Browser-driven listing fetch: let TikTok sign its own ``item_list`` XHRs.
+
+Profile/hashtag/search listings need signed requests (``X-Gnarly``) whose
+algorithm rev's monthly and reads a browser canvas fingerprint. Rather than port
+and chase that signer, we load the page in the stealth browser we already run
+(patchright-Chromium, via the web-crawler tier) and capture the itemStruct JSON
+the page's own scripts fetch while scrolling. The browser is the client, so it
+signs correctly for whatever version TikTok ships.
+
+The pure response-shape parsing lives in :func:`items_from_response`; this module
+is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from scrapling.fetchers import StealthyFetcher
+
+from app.proprietary.web_crawler.stealth import (
+ build_stealthy_kwargs,
+ get_stealth_config,
+)
+from app.utils.proxy import get_proxy_url
+
+from ..extraction import items_from_response
+
+logger = logging.getLogger(__name__)
+
+# XHR paths that carry itemStructs for the three listing kinds.
+_ITEM_LIST_MARKERS = (
+ "/api/post/item_list",
+ "/api/challenge/item_list",
+ "/api/search/",
+)
+# Bounded scroll: a dead page can't loop forever, and a live one stops early
+# once enough items are captured.
+_SCROLL_MAX_ROUNDS = 20
+_SCROLL_SETTLE_MS = 1200
+
+
+def _build_page_action(collected: list[dict[str, Any]], target_count: int):
+ """A sync ``page_action`` that captures item_list XHRs while scrolling."""
+
+ def _on_response(response: Any) -> None:
+ try:
+ if not any(marker in response.url for marker in _ITEM_LIST_MARKERS):
+ return
+ body = response.json()
+ except Exception:
+ return
+ collected.extend(items_from_response(body))
+
+ def page_action(page: Any) -> Any:
+ page.on("response", _on_response)
+ try:
+ last_height = 0
+ for _ in range(_SCROLL_MAX_ROUNDS):
+ if len(collected) >= target_count:
+ break
+ page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+ page.wait_for_timeout(_SCROLL_SETTLE_MS)
+ height = page.evaluate("document.body.scrollHeight")
+ if not height or height <= last_height:
+ break
+ last_height = height
+ except Exception as exc:
+ logger.debug("[tiktok] listing scroll aborted: %s", exc)
+ return page
+
+ return page_action
+
+
+def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]:
+ collected: list[dict[str, Any]] = []
+ kwargs = build_stealthy_kwargs(get_stealth_config())
+ StealthyFetcher.fetch(
+ url,
+ headless=True,
+ network_idle=False,
+ proxy=get_proxy_url(),
+ page_action=_build_page_action(collected, target_count),
+ **kwargs,
+ )
+ return collected[:target_count]
+
+
+async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]:
+ """Return up to ``target_count`` itemStructs from a listing page's XHRs."""
+ return await asyncio.to_thread(_fetch_sync, page_url, target_count)
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py
new file mode 100644
index 000000000..00ab89b02
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py
@@ -0,0 +1,25 @@
+"""Pulling item structs out of captured item_list / search API responses."""
+
+from __future__ import annotations
+
+from app.proprietary.platforms.tiktok.extraction import items_from_response
+
+
+def test_reads_item_list_shape():
+ body = {"itemList": [{"id": "1"}, {"id": "2"}], "hasMore": True}
+ assert items_from_response(body) == [{"id": "1"}, {"id": "2"}]
+
+
+def test_reads_search_data_shape():
+ body = {"data": [{"type": 1, "item": {"id": "9"}}, {"type": 4, "item": {}}]}
+ assert items_from_response(body) == [{"id": "9"}, {}]
+
+
+def test_skips_malformed_entries():
+ body = {"data": [{"type": 1}, "junk", {"item": {"id": "7"}}]}
+ assert items_from_response(body) == [{"id": "7"}]
+
+
+def test_returns_empty_for_unrelated_json():
+ assert items_from_response({"statusCode": 0}) == []
+ assert items_from_response("nope") == []
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py
index 85c601047..8520a1d85 100644
--- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py
@@ -71,3 +71,27 @@ async def test_scrape_skips_unrecognized_urls():
TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch
)
assert items == []
+
+
+async def test_scrape_profile_returns_listing_items():
+ async def fake_listing(_url: str, _count: int) -> list[dict]:
+ return [
+ {"id": "1", "author": {"uniqueId": "a"}},
+ {"id": "2", "author": {"uniqueId": "a"}},
+ ]
+
+ items = await scrape_tiktok(
+ TikTokScrapeInput(profiles=["a"], resultsPerPage=5), fetch_listing=fake_listing
+ )
+ assert [i["id"] for i in items] == ["1", "2"]
+ assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1"
+
+
+async def test_listing_dedupes_then_caps_per_target():
+ async def fake_listing(_url: str, _count: int) -> list[dict]:
+ return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}]
+
+ items = await scrape_tiktok(
+ TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing
+ )
+ assert [i["id"] for i in items] == ["1", "2"]
From 5a326f3818158e40dea0fc8847b4f563ba1d4486 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 17:14:13 +0200
Subject: [PATCH 026/160] fix(tiktok): warm browser listing + drop
ctx-var-in-generator
---
.../platforms/tiktok/orchestrator.py | 20 ++++---
.../platforms/tiktok/session/listing.py | 60 ++++++++++++++++---
2 files changed, 65 insertions(+), 15 deletions(-)
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
index c7f2e9008..95bc10fb9 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
@@ -16,7 +16,7 @@ from .flows import FetchFn, FetchListingFn
from .flows.listing import iter_listing
from .flows.video import iter_video
from .schemas import TikTokScrapeInput
-from .session import fetch_html, fetch_item_list, proxy_session
+from .session import fetch_html, fetch_item_list
from .targets import resolve_target
from .targets.types import TikTokTarget
@@ -66,14 +66,18 @@ async def iter_tiktok(
fetch: FetchFn = fetch_html,
fetch_listing: FetchListingFn = fetch_item_list,
) -> AsyncIterator[dict[str, Any]]:
- """Yield normalized items for every resolved target, in order."""
+ """Yield normalized items for every resolved target, in order.
+
+ The video flow's ``fetch_html`` opens its own warmed proxy session per call
+ when none is bound; the listing flow drives its own browser. Neither binds a
+ ContextVar across these ``yield``s, so the generator stays context-safe.
+ """
cap = input_model.resultsPerPage
- async with proxy_session():
- for target in _resolve_targets(input_model):
- async for item in _dispatch(
- target, cap=cap, fetch=fetch, fetch_listing=fetch_listing
- ):
- yield item
+ for target in _resolve_targets(input_model):
+ async for item in _dispatch(
+ target, cap=cap, fetch=fetch, fetch_listing=fetch_listing
+ ):
+ yield item
async def scrape_tiktok(
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
index 0d19281a2..98b2e6e8b 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
@@ -9,12 +9,17 @@ signs correctly for whatever version TikTok ships.
The pure response-shape parsing lives in :func:`items_from_response`; this module
is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
+
+Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning
+empty ``item_list`` bodies (and 429s) after a few hits. Set
+``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing.
"""
from __future__ import annotations
import asyncio
import logging
+import os
from typing import Any
from scrapling.fetchers import StealthyFetcher
@@ -35,27 +40,68 @@ _ITEM_LIST_MARKERS = (
"/api/challenge/item_list",
"/api/search/",
)
+_HOME_URL = "https://www.tiktok.com/"
+_MSTOKEN_COOKIE = "msToken"
# Bounded scroll: a dead page can't loop forever, and a live one stops early
# once enough items are captured.
_SCROLL_MAX_ROUNDS = 20
-_SCROLL_SETTLE_MS = 1200
+_SCROLL_SETTLE_MS = 1500
+# Warm-up poll for the anonymous msToken cookie the item_list API requires.
+_WARM_POLLS = 8
+_WARM_POLL_MS = 500
-def _build_page_action(collected: list[dict[str, Any]], target_count: int):
- """A sync ``page_action`` that captures item_list XHRs while scrolling."""
+def _has_mstoken(page: Any) -> bool:
+ try:
+ return any(c.get("name") == _MSTOKEN_COOKIE for c in page.context.cookies())
+ except Exception:
+ return False
+
+
+def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int):
+ """A sync ``page_action`` that warms the session then captures item_list XHRs.
+
+ A cold context returns an empty ``item_list`` body, so we first mint the
+ anonymous ``msToken`` (homepage hit), then navigate to the target with the
+ listener already attached so page-one fires into it; scrolling pages the rest.
+ """
+
+ debug = bool(os.getenv("TIKTOK_LISTING_DEBUG"))
def _on_response(response: Any) -> None:
+ response_url = getattr(response, "url", "")
+ if debug and "/api/" in response_url:
+ print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}")
try:
- if not any(marker in response.url for marker in _ITEM_LIST_MARKERS):
+ if not any(marker in response_url for marker in _ITEM_LIST_MARKERS):
return
body = response.json()
- except Exception:
+ except Exception as exc:
+ if debug:
+ print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}")
return
- collected.extend(items_from_response(body))
+ items = items_from_response(body)
+ if debug:
+ print(f" [match] +{len(items)} items from {response_url[:100]}")
+ collected.extend(items)
+
+ def _warm(page: Any) -> None:
+ if _has_mstoken(page):
+ return
+ page.goto(_HOME_URL, wait_until="domcontentloaded")
+ for _ in range(_WARM_POLLS):
+ page.wait_for_timeout(_WARM_POLL_MS)
+ if _has_mstoken(page):
+ break
def page_action(page: Any) -> Any:
page.on("response", _on_response)
try:
+ _warm(page)
+ # Navigate (back) to the target with the listener attached and a
+ # token in hand, so the page-one item_list fires into the capture.
+ page.goto(url, wait_until="domcontentloaded")
+ page.wait_for_timeout(_SCROLL_SETTLE_MS)
last_height = 0
for _ in range(_SCROLL_MAX_ROUNDS):
if len(collected) >= target_count:
@@ -81,7 +127,7 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]:
headless=True,
network_idle=False,
proxy=get_proxy_url(),
- page_action=_build_page_action(collected, target_count),
+ page_action=_build_page_action(collected, url, target_count),
**kwargs,
)
return collected[:target_count]
From ea95a0529fbfba5864606bdef353df32224d8e7e Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 17:14:13 +0200
Subject: [PATCH 027/160] test(tiktok): live e2e smoke script
---
.../scripts/e2e_tiktok_scrape.py | 201 ++++++++++++++++++
1 file changed, 201 insertions(+)
create mode 100644 surfsense_backend/scripts/e2e_tiktok_scrape.py
diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py
new file mode 100644
index 000000000..29cf1edc7
--- /dev/null
+++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py
@@ -0,0 +1,201 @@
+"""Manual functional e2e for the TikTok scraper (blob + browser-listing seams).
+
+Run from the backend directory:
+ cd surfsense_backend
+ uv run python scripts/e2e_tiktok_scrape.py
+
+What it exercises (everything REAL — live network, live proxy, live browser):
+
+ Stage 1 — proxy egress proof (informational).
+ Stage 2 — profile listing via the stealth browser (captures item_list XHRs).
+ Stage 3 — blob video path over HTTP (a video URL derived from Stage 2).
+ Stage 4 — hashtag listing via the stealth browser.
+ Stage 5 — full scrape_tiktok() pipeline on a profile.
+
+On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser
+suites can pin against real-shaped data without network.
+
+This is NOT a pytest test (it needs a live stack + proxy + a real browser). It is
+the manual counterpart to the unit suites and the truth check for
+``session/listing.py``, which is otherwise unverified.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+ if _candidate.exists():
+ load_dotenv(_candidate)
+ break
+
+_FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
+
+# Evergreen public targets: the official account and a broad hashtag.
+_PROFILE = "tiktok"
+_HASHTAG = "food"
+_COUNT = 5
+
+
+def _mask(url: str | None) -> str:
+ if not url:
+ return ""
+ p = urlsplit(url)
+ creds = "***@" if p.username else ""
+ port = f":{p.port}" if p.port else ""
+ return f"{p.scheme}://{creds}{p.hostname or '?'}{port}"
+
+
+def _hr(title: str) -> None:
+ print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+ print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+ return ok
+
+
+def _dump_fixture(name: str, data: Any) -> None:
+ _FIXTURES.mkdir(parents=True, exist_ok=True)
+ path = _FIXTURES / name
+ path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+ print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}")
+
+
+async def _discover_video_url() -> str | None:
+ """Find a live video URL over plain HTTP (independent of the browser path)."""
+ import re
+
+ from app.proprietary.platforms.tiktok.session import fetch_html
+
+ html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}")
+ if not html:
+ return None
+ match = re.search(r"/@[\w.]+/video/\d+", html)
+ return f"https://www.tiktok.com{match.group(0)}" if match else None
+
+
+async def stage_proxy() -> bool:
+ _hr("STAGE 1 — proxy egress (informational)")
+ from app.utils.proxy import get_active_provider, get_proxy_url, is_pool_backed
+
+ provider = get_active_provider()
+ proxy_url = get_proxy_url()
+ print(f" active proxy provider : {provider.name}")
+ print(f" proxy url : {_mask(proxy_url)}")
+ print(f" pool-backed (rotates) : {is_pool_backed()}")
+ if not proxy_url:
+ print(" [INFO] no proxy configured — TikTok may block anonymous access")
+ return True
+
+
+async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]:
+ _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}")
+ from app.proprietary.platforms.tiktok.session import fetch_item_list
+
+ url = f"https://www.tiktok.com/@{_PROFILE}"
+ raw = await fetch_item_list(url, _COUNT)
+ ok = _check(
+ "captured itemStructs from item_list XHRs",
+ len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")),
+ f"{len(raw)} struct(s)",
+ )
+ if ok:
+ _dump_fixture("listing_item.json", raw[0])
+ return ok, raw
+
+
+async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]:
+ _hr("STAGE 3 — blob video path (HTTP)")
+ print(f" target: {video_url}")
+ from app.proprietary.platforms.tiktok.extraction import (
+ extract_rehydration_data,
+ video_item_struct,
+ )
+ from app.proprietary.platforms.tiktok.session import fetch_html
+
+ html = await fetch_html(video_url)
+ if not _check("fetched page HTML", bool(html), f"{len(html or '')} chars"):
+ return False, None
+ data = extract_rehydration_data(html or "")
+ if not _check("extracted rehydration blob", data is not None):
+ return False, None
+ raw = video_item_struct(data or {})
+ ok = _check(
+ "blob carries the video itemStruct",
+ raw is not None and bool(raw.get("id")),
+ f"id={None if raw is None else raw.get('id')}",
+ )
+ if ok and raw is not None:
+ _dump_fixture("video_item_struct.json", raw)
+ return ok, raw
+
+
+async def stage_hashtag_listing() -> bool:
+ _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}")
+ from app.proprietary.platforms.tiktok.session import fetch_item_list
+
+ url = f"https://www.tiktok.com/tag/{_HASHTAG}"
+ raw = await fetch_item_list(url, _COUNT)
+ return _check(
+ "captured itemStructs for hashtag",
+ len(raw) > 0 and bool(raw[0].get("id")),
+ f"{len(raw)} struct(s)",
+ )
+
+
+async def stage_pipeline() -> bool:
+ _hr("STAGE 5 — full scrape_tiktok() pipeline")
+ from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
+
+ items = await scrape_tiktok(
+ TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3
+ )
+ ok = _check(
+ "pipeline returns normalized video items",
+ len(items) > 0
+ and bool(items[0].get("id"))
+ and bool(items[0].get("webVideoUrl")),
+ f"{len(items)} item(s)",
+ )
+ if items:
+ print(f" sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}")
+ return ok
+
+
+async def main() -> int:
+ print("TikTok scraper functional e2e — live network + proxy + browser")
+ results: dict[str, bool] = {}
+
+ results["Stage 1 proxy"] = await stage_proxy()
+
+ # Blob path first: HTTP-only, so it's validated independently of the
+ # proxy-sensitive browser listing path.
+ video_url = await _discover_video_url()
+ if video_url:
+ ok_video, _ = await stage_blob_video(video_url)
+ results["Stage 3 blob video"] = ok_video
+ else:
+ print("\n [SKIP] Stage 3 — could not discover a video URL over HTTP")
+
+ ok_profile, _ = await stage_profile_listing()
+ results["Stage 2 profile listing"] = ok_profile
+ results["Stage 4 hashtag listing"] = await stage_hashtag_listing()
+ results["Stage 5 pipeline"] = await stage_pipeline()
+
+ _hr("SUMMARY")
+ for name, ok in results.items():
+ print(f" {'PASS' if ok else 'FAIL/SKIP'} — {name}")
+ return 0 if all(results.values()) else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(asyncio.run(main()))
From 9dd39faa50331c6df0e07790520a395d51177d93 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 17:53:29 +0200
Subject: [PATCH 028/160] test(tiktok): pin live fixtures, trim listing debug
scaffolding
---
.../platforms/tiktok/session/listing.py | 29 +-
.../scripts/e2e_tiktok_scrape.py | 52 +-
.../tests/fixtures/tiktok/listing_item.json | 559 +++++++++++++++
.../fixtures/tiktok/video_item_struct.json | 660 ++++++++++++++++++
4 files changed, 1254 insertions(+), 46 deletions(-)
create mode 100644 surfsense_backend/tests/fixtures/tiktok/listing_item.json
create mode 100644 surfsense_backend/tests/fixtures/tiktok/video_item_struct.json
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
index 98b2e6e8b..cc84a48b5 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py
@@ -10,16 +10,15 @@ signs correctly for whatever version TikTok ships.
The pure response-shape parsing lives in :func:`items_from_response`; this module
is the untested browser-I/O glue (covered by the e2e smoke, not unit tests).
-Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning
-empty ``item_list`` bodies (and 429s) after a few hits. Set
-``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing.
+Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile
+feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an
+empty 200 even to its own signed request, so profile targets yield nothing.
"""
from __future__ import annotations
import asyncio
import logging
-import os
from typing import Any
from scrapling.fetchers import StealthyFetcher
@@ -66,24 +65,16 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count:
listener already attached so page-one fires into it; scrolling pages the rest.
"""
- debug = bool(os.getenv("TIKTOK_LISTING_DEBUG"))
-
def _on_response(response: Any) -> None:
response_url = getattr(response, "url", "")
- if debug and "/api/" in response_url:
- print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}")
- try:
- if not any(marker in response_url for marker in _ITEM_LIST_MARKERS):
- return
- body = response.json()
- except Exception as exc:
- if debug:
- print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}")
+ if not any(marker in response_url for marker in _ITEM_LIST_MARKERS):
return
- items = items_from_response(body)
- if debug:
- print(f" [match] +{len(items)} items from {response_url[:100]}")
- collected.extend(items)
+ try:
+ body = response.json()
+ except Exception:
+ # An empty 200 (TikTok soft-block) or a body evicted before read.
+ return
+ collected.extend(items_from_response(body))
def _warm(page: Any) -> None:
if _has_mstoken(page):
diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py
index 29cf1edc7..22bebff96 100644
--- a/surfsense_backend/scripts/e2e_tiktok_scrape.py
+++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py
@@ -7,10 +7,11 @@ Run from the backend directory:
What it exercises (everything REAL — live network, live proxy, live browser):
Stage 1 — proxy egress proof (informational).
- Stage 2 — profile listing via the stealth browser (captures item_list XHRs).
- Stage 3 — blob video path over HTTP (a video URL derived from Stage 2).
- Stage 4 — hashtag listing via the stealth browser.
- Stage 5 — full scrape_tiktok() pipeline on a profile.
+ Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok;
+ expected empty until a stronger anti-detection path exists).
+ Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct).
+ Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs).
+ Stage 5 — full scrape_tiktok() pipeline on a hashtag.
On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser
suites can pin against real-shaped data without network.
@@ -39,8 +40,8 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
_FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok"
-# Evergreen public targets: the official account and a broad hashtag.
-_PROFILE = "tiktok"
+# Evergreen public targets: a regular high-volume creator and a broad hashtag.
+_PROFILE = "nasa"
_HASHTAG = "food"
_COUNT = 5
@@ -70,17 +71,11 @@ def _dump_fixture(name: str, data: Any) -> None:
print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}")
-async def _discover_video_url() -> str | None:
- """Find a live video URL over plain HTTP (independent of the browser path)."""
- import re
-
- from app.proprietary.platforms.tiktok.session import fetch_html
-
- html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}")
- if not html:
- return None
- match = re.search(r"/@[\w.]+/video/\d+", html)
- return f"https://www.tiktok.com{match.group(0)}" if match else None
+def _url_from_struct(struct: dict[str, Any]) -> str | None:
+ """Build a canonical video URL from a captured itemStruct."""
+ author = (struct.get("author") or {}).get("uniqueId")
+ vid = struct.get("id")
+ return f"https://www.tiktok.com/@{author}/video/{vid}" if author and vid else None
async def stage_proxy() -> bool:
@@ -108,8 +103,6 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]:
len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")),
f"{len(raw)} struct(s)",
)
- if ok:
- _dump_fixture("listing_item.json", raw[0])
return ok, raw
@@ -139,17 +132,20 @@ async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]
return ok, raw
-async def stage_hashtag_listing() -> bool:
+async def stage_hashtag_listing() -> tuple[bool, list[dict[str, Any]]]:
_hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}")
from app.proprietary.platforms.tiktok.session import fetch_item_list
url = f"https://www.tiktok.com/tag/{_HASHTAG}"
raw = await fetch_item_list(url, _COUNT)
- return _check(
+ ok = _check(
"captured itemStructs for hashtag",
len(raw) > 0 and bool(raw[0].get("id")),
f"{len(raw)} struct(s)",
)
+ if ok:
+ _dump_fixture("listing_item.json", raw[0])
+ return ok, raw
async def stage_pipeline() -> bool:
@@ -157,7 +153,7 @@ async def stage_pipeline() -> bool:
from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok
items = await scrape_tiktok(
- TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3
+ TikTokScrapeInput(hashtags=[_HASHTAG], resultsPerPage=3), limit=3
)
ok = _check(
"pipeline returns normalized video items",
@@ -177,18 +173,20 @@ async def main() -> int:
results["Stage 1 proxy"] = await stage_proxy()
- # Blob path first: HTTP-only, so it's validated independently of the
- # proxy-sensitive browser listing path.
- video_url = await _discover_video_url()
+ # Hashtag listing is the reliable browser path; use one of its captured
+ # structs to build a real video URL for the HTTP blob path.
+ ok_tag, tag_structs = await stage_hashtag_listing()
+ results["Stage 4 hashtag listing"] = ok_tag
+
+ video_url = _url_from_struct(tag_structs[0]) if tag_structs else None
if video_url:
ok_video, _ = await stage_blob_video(video_url)
results["Stage 3 blob video"] = ok_video
else:
- print("\n [SKIP] Stage 3 — could not discover a video URL over HTTP")
+ print("\n [SKIP] Stage 3 — no captured struct to build a video URL")
ok_profile, _ = await stage_profile_listing()
results["Stage 2 profile listing"] = ok_profile
- results["Stage 4 hashtag listing"] = await stage_hashtag_listing()
results["Stage 5 pipeline"] = await stage_pipeline()
_hr("SUMMARY")
diff --git a/surfsense_backend/tests/fixtures/tiktok/listing_item.json b/surfsense_backend/tests/fixtures/tiktok/listing_item.json
new file mode 100644
index 000000000..374a9bf67
--- /dev/null
+++ b/surfsense_backend/tests/fixtures/tiktok/listing_item.json
@@ -0,0 +1,559 @@
+{
+ "AIGCDescription": "",
+ "CategoryType": 111,
+ "IsHDBitrate": false,
+ "anchors": [
+ {
+ "description": "CapCut · Video Editor",
+ "extraInfo": {
+ "subtype": ""
+ },
+ "icon": {
+ "urlList": [
+ "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=1EMaRlc0Snbs9Ltwr0s95%2BDvpFk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5"
+ ]
+ },
+ "id": "0",
+ "keyword": "CapCut · Editing made easy",
+ "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}",
+ "schema": "https://www.capcut.com/editor?scenario=tiktok&utm_source=tiktok_anchor_tool&utm_medium=tiktok_anchor&utm_campaign=70361230",
+ "thumbnail": {
+ "height": 64,
+ "urlList": [
+ "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=c7mUcXdyEOFc0SMcLAkkib3v9RU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5"
+ ],
+ "width": 64
+ },
+ "type": 54
+ }
+ ],
+ "author": {
+ "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "avatarThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "commentSetting": 0,
+ "downloadSetting": 3,
+ "duetSetting": 0,
+ "ftc": false,
+ "id": "7501138183701103662",
+ "isADVirtual": false,
+ "isEmbedBanned": false,
+ "nickname": "Cookwithhali",
+ "openFavorite": false,
+ "privateAccount": false,
+ "relation": 0,
+ "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q",
+ "secret": false,
+ "shortDramaCreator": {},
+ "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com",
+ "stitchSetting": 0,
+ "uniqueId": "cookwithhali",
+ "verified": false
+ },
+ "authorStats": {
+ "diggCount": 223,
+ "followerCount": 372600,
+ "followingCount": 0,
+ "friendCount": 0,
+ "heart": 8000000,
+ "heartCount": 8000000,
+ "videoCount": 279
+ },
+ "authorStatsV2": {
+ "diggCount": "223",
+ "followerCount": "372600",
+ "followingCount": "0",
+ "friendCount": "0",
+ "heart": "8000000",
+ "heartCount": "8000000",
+ "videoCount": "279"
+ },
+ "backendSourceEventTracking": "",
+ "challenges": [
+ {
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": "",
+ "desc": "",
+ "id": "6983",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "title": "bread"
+ },
+ {
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": "",
+ "desc": "",
+ "id": "285169",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "title": "garlicbread"
+ },
+ {
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": "",
+ "desc": "",
+ "id": "1643317566954502",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "title": "breadtok"
+ },
+ {
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": "",
+ "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.",
+ "id": "7250",
+ "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "title": "baking"
+ },
+ {
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": "",
+ "desc": "",
+ "id": "10488587",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "title": "homemadebread"
+ }
+ ],
+ "collected": false,
+ "contents": [
+ {
+ "desc": "Pull Apart Garlic Bread "
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "Ingredients"
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "* 1 tsp yeast"
+ },
+ {
+ "desc": "* 1 tbsp sugar"
+ },
+ {
+ "desc": "* 270g milk (about 1 cup + 2 tbsp)"
+ },
+ {
+ "desc": "* 380g flour (about 3 cups)"
+ },
+ {
+ "desc": "* 1 tsp salt"
+ },
+ {
+ "desc": "* 43g softened butter (about 3 tbsp)"
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "Butter Mixture"
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "* 100g butter, softened"
+ },
+ {
+ "desc": "* 1 tbsp chopped cilantro"
+ },
+ {
+ "desc": "* 1 garlic clove, minced"
+ },
+ {
+ "desc": "* Shredded cheese"
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "Instructions"
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined."
+ },
+ {
+ "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated."
+ },
+ {
+ "desc": "3. Cover and let the dough rest for 2 hours or until doubled."
+ },
+ {
+ "desc": "4. Mix the softened butter with the cilantro and minced garlic."
+ },
+ {
+ "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese."
+ },
+ {
+ "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan."
+ },
+ {
+ "desc": "7. Cover and let rest for 20 minutes."
+ },
+ {
+ "desc": "8. Bake at 370°F for 30 minutes, or until golden brown."
+ },
+ {
+ "desc": "9. Brush with the remaining garlic butter mixture while still warm."
+ },
+ {
+ "desc": ""
+ },
+ {
+ "desc": "#bread #garlicbread #breadtok #baking #homemadebread ",
+ "textExtra": [
+ {
+ "awemeId": "",
+ "end": 6,
+ "hashtagName": "bread",
+ "isCommerce": false,
+ "start": 0,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 19,
+ "hashtagName": "garlicbread",
+ "isCommerce": false,
+ "start": 7,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 29,
+ "hashtagName": "breadtok",
+ "isCommerce": false,
+ "start": 20,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 37,
+ "hashtagName": "baking",
+ "isCommerce": false,
+ "start": 30,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 52,
+ "hashtagName": "homemadebread",
+ "isCommerce": false,
+ "start": 38,
+ "subType": 0,
+ "type": 1
+ }
+ ]
+ }
+ ],
+ "createTime": 1782509863,
+ "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ",
+ "digged": false,
+ "diversificationId": 10040,
+ "duetDisplay": 0,
+ "duetEnabled": true,
+ "forFriend": false,
+ "id": "7655821461772406047",
+ "isAd": false,
+ "isReviewing": false,
+ "itemCommentStatus": 0,
+ "item_control": {
+ "can_repost": true
+ },
+ "music": {
+ "authorName": "Cookwithhali",
+ "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "coverThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "duration": 62,
+ "id": "7655821492566919966",
+ "isCopyrighted": false,
+ "is_commerce_music": true,
+ "is_unlimited_music": false,
+ "original": true,
+ "playUrl": "https://v16m.tiktokcdn-us.com/ecd83422a3e7bf5450bd92ebf07e6fce/6a4ec449/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7Th2sbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=2026070815413036F7D0D0506FB569C78C&btag=e00050000",
+ "private": false,
+ "shoot_duration": 62,
+ "title": "original sound",
+ "tt2dsp": {}
+ },
+ "officalItem": false,
+ "originalItem": false,
+ "privateItem": false,
+ "secret": false,
+ "shareEnabled": true,
+ "stats": {
+ "collectCount": 396400,
+ "commentCount": 3720,
+ "diggCount": 754100,
+ "playCount": 8300000,
+ "shareCount": 144500
+ },
+ "statsV2": {
+ "collectCount": "396397",
+ "commentCount": "3720",
+ "diggCount": "754100",
+ "playCount": "8300000",
+ "repostCount": "0",
+ "shareCount": "144500"
+ },
+ "stickersOnItem": [
+ {
+ "stickerText": [
+ "Cheesy Pull Apart Garlic Bread! 🧄🧀\n",
+ "(Bakery-Style & No-Knead)"
+ ],
+ "stickerType": 4
+ }
+ ],
+ "stitchDisplay": 0,
+ "stitchEnabled": true,
+ "textExtra": [
+ {
+ "awemeId": "",
+ "end": 1030,
+ "hashtagName": "bread",
+ "isCommerce": false,
+ "start": 1024,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 1043,
+ "hashtagName": "garlicbread",
+ "isCommerce": false,
+ "start": 1031,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 1053,
+ "hashtagName": "breadtok",
+ "isCommerce": false,
+ "start": 1044,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 1061,
+ "hashtagName": "baking",
+ "isCommerce": false,
+ "start": 1054,
+ "subType": 0,
+ "type": 1
+ },
+ {
+ "awemeId": "",
+ "end": 1076,
+ "hashtagName": "homemadebread",
+ "isCommerce": false,
+ "start": 1062,
+ "subType": 0,
+ "type": 1
+ }
+ ],
+ "textLanguage": "en",
+ "textTranslatable": true,
+ "video": {
+ "PlayAddrStruct": {
+ "DataSize": 13761060,
+ "FileCs": "c:0-51456-f8d1",
+ "FileHash": "485174c8492db027adbbf6dca7c7c201",
+ "Height": 1280,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "Width": 720
+ },
+ "VQScore": "69.69",
+ "bitrate": 1773703,
+ "bitrateInfo": [
+ {
+ "Bitrate": 1836180,
+ "BitrateFPS": 29,
+ "CodecType": "h265_hvc1",
+ "Format": "mp4",
+ "GearName": "adapt_lowest_1080_1",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}",
+ "PlayAddr": {
+ "DataSize": 14245777,
+ "FileCs": "c:0-52754-ed06",
+ "FileHash": "38996a98cfeb2714fca1b94768090c37",
+ "Height": 1920,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "Width": 1080
+ },
+ "QualityType": 2,
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}"
+ },
+ {
+ "Bitrate": 1773703,
+ "BitrateFPS": 29,
+ "CodecType": "h264",
+ "Format": "mp4",
+ "GearName": "normal_720_0",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}",
+ "PlayAddr": {
+ "DataSize": 13761060,
+ "FileCs": "c:0-51456-f8d1",
+ "FileHash": "485174c8492db027adbbf6dca7c7c201",
+ "Height": 1280,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "Width": 720
+ },
+ "QualityType": 10,
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}"
+ },
+ {
+ "Bitrate": 966380,
+ "BitrateFPS": 29,
+ "CodecType": "h265_hvc1",
+ "Format": "mp4",
+ "GearName": "adapt_lower_720_1",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}",
+ "PlayAddr": {
+ "DataSize": 7497540,
+ "FileCs": "c:0-52786-7f65",
+ "FileHash": "1376df24c917669727b27273697fb826",
+ "Height": 1280,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "Width": 720
+ },
+ "QualityType": 14,
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}"
+ },
+ {
+ "Bitrate": 799620,
+ "BitrateFPS": 29,
+ "CodecType": "h265_hvc1",
+ "Format": "mp4",
+ "GearName": "adapt_540_1",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}",
+ "PlayAddr": {
+ "DataSize": 6203756,
+ "FileCs": "c:0-52786-7f6d",
+ "FileHash": "41f62f740ad13045000450f28b9ff2cb",
+ "Height": 1024,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "Width": 576
+ },
+ "QualityType": 28,
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}"
+ },
+ {
+ "Bitrate": 622546,
+ "BitrateFPS": 29,
+ "CodecType": "h264",
+ "Format": "mp4",
+ "GearName": "lowest_540_0",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}",
+ "PlayAddr": {
+ "DataSize": 4829948,
+ "FileCs": "c:0-51513-3d83",
+ "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d",
+ "Height": 1024,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "Width": 576
+ },
+ "QualityType": 25,
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}"
+ }
+ ],
+ "claInfo": {
+ "enableAutoCaption": true,
+ "hasOriginalAudio": true,
+ "noCaptionReason": 3
+ },
+ "codecType": "h264",
+ "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5",
+ "definition": "720p",
+ "duration": 62,
+ "dynamicCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=JWF%2F5g2txpFXhNhJzOwleSJgKik%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5",
+ "encodeUserTag": "",
+ "encodedType": "normal",
+ "format": "mp4",
+ "height": 1280,
+ "id": "7655821461772406047",
+ "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5",
+ "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000",
+ "ratio": "720p",
+ "size": 13761060,
+ "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "videoQuality": "normal",
+ "volumeInfo": {
+ "Loudness": -48.5,
+ "Peak": 0.18408
+ },
+ "width": 720,
+ "zoomCover": {
+ "240": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=ZONw1V50ZmupszDX0NiC%2Bf8BimU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1",
+ "480": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=LztWYGkvUC4%2BvhtQb0TO0l1ky0s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1",
+ "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1",
+ "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json
new file mode 100644
index 000000000..af3bd651b
--- /dev/null
+++ b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json
@@ -0,0 +1,660 @@
+{
+ "id": "7655821461772406047",
+ "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ",
+ "createTime": "1782509863",
+ "scheduleTime": 0,
+ "video": {
+ "id": "7655821461772406047",
+ "height": 1280,
+ "width": 720,
+ "duration": 62,
+ "ratio": "720p",
+ "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5",
+ "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5",
+ "dynamicCover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=Jz%2BPtpSXdbjTV4qFbG4H7JFD0f8%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5",
+ "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000",
+ "downloadAddr": "",
+ "shareCover": [
+ ""
+ ],
+ "reflowCover": "",
+ "bitrate": 1773703,
+ "encodedType": "normal",
+ "format": "mp4",
+ "videoQuality": "normal",
+ "encodeUserTag": "",
+ "codecType": "h264",
+ "definition": "720p",
+ "subtitleInfos": [],
+ "zoomCover": {
+ "240": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=cBXoX16WOKJyUth9fnaohYv%2B%2BBs%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1",
+ "480": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=Ff5B1aV4teSPQc6JXMoidMXqJWk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1",
+ "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1",
+ "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1"
+ },
+ "volumeInfo": {
+ "Loudness": -48.5,
+ "Peak": 0.18408
+ },
+ "bitrateInfo": [
+ {
+ "Bitrate": 1836180,
+ "QualityType": 2,
+ "BitrateFPS": 29,
+ "GearName": "adapt_lowest_1080_1",
+ "PlayAddr": {
+ "DataSize": "14245777",
+ "Width": 1080,
+ "Height": 1920,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180",
+ "FileHash": "38996a98cfeb2714fca1b94768090c37",
+ "FileCs": "c:0-52754-ed06"
+ },
+ "CodecType": "h265_hvc1",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}",
+ "Format": "mp4",
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}"
+ },
+ {
+ "Bitrate": 1773703,
+ "QualityType": 10,
+ "BitrateFPS": 29,
+ "GearName": "normal_720_0",
+ "PlayAddr": {
+ "DataSize": "13761060",
+ "Width": 720,
+ "Height": 1280,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703",
+ "FileHash": "485174c8492db027adbbf6dca7c7c201",
+ "FileCs": "c:0-51456-f8d1"
+ },
+ "CodecType": "h264",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}",
+ "Format": "mp4",
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}"
+ },
+ {
+ "Bitrate": 966380,
+ "QualityType": 14,
+ "BitrateFPS": 29,
+ "GearName": "adapt_lower_720_1",
+ "PlayAddr": {
+ "DataSize": "7497540",
+ "Width": 720,
+ "Height": 1280,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380",
+ "FileHash": "1376df24c917669727b27273697fb826",
+ "FileCs": "c:0-52786-7f65"
+ },
+ "CodecType": "h265_hvc1",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}",
+ "Format": "mp4",
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}"
+ },
+ {
+ "Bitrate": 799620,
+ "QualityType": 28,
+ "BitrateFPS": 29,
+ "GearName": "adapt_540_1",
+ "PlayAddr": {
+ "DataSize": "6203756",
+ "Width": 576,
+ "Height": 1024,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620",
+ "FileHash": "41f62f740ad13045000450f28b9ff2cb",
+ "FileCs": "c:0-52786-7f6d"
+ },
+ "CodecType": "h265_hvc1",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}",
+ "Format": "mp4",
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}"
+ },
+ {
+ "Bitrate": 622546,
+ "QualityType": 25,
+ "BitrateFPS": 29,
+ "GearName": "lowest_540_0",
+ "PlayAddr": {
+ "DataSize": "4829948",
+ "Width": 576,
+ "Height": 1024,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546",
+ "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d",
+ "FileCs": "c:0-51513-3d83"
+ },
+ "CodecType": "h264",
+ "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}",
+ "Format": "mp4",
+ "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}"
+ }
+ ],
+ "size": "13761060",
+ "VQScore": "69.69",
+ "claInfo": {
+ "hasOriginalAudio": true,
+ "enableAutoCaption": true,
+ "captionInfos": [],
+ "noCaptionReason": 3
+ },
+ "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "PlayAddrStruct": {
+ "DataSize": "13761060",
+ "Width": 720,
+ "Height": 1280,
+ "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg",
+ "UrlList": [
+ "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000",
+ "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000",
+ "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg"
+ ],
+ "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703",
+ "FileHash": "485174c8492db027adbbf6dca7c7c201",
+ "FileCs": "c:0-51456-f8d1"
+ }
+ },
+ "author": {
+ "id": "7501138183701103662",
+ "shortId": "",
+ "uniqueId": "cookwithhali",
+ "nickname": "Cookwithhali",
+ "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "avatarThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com",
+ "createTime": 1746494976,
+ "verified": false,
+ "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q",
+ "ftc": false,
+ "relation": 0,
+ "openFavorite": false,
+ "commentSetting": 0,
+ "duetSetting": 0,
+ "stitchSetting": 0,
+ "privateAccount": false,
+ "secret": false,
+ "isADVirtual": false,
+ "roomId": "",
+ "uniqueIdModifyTime": 0,
+ "ttSeller": false,
+ "downloadSetting": 3,
+ "recommendReason": "",
+ "nowInvitationCardUrl": "",
+ "nickNameModifyTime": 0,
+ "isEmbedBanned": false,
+ "canExpPlaylist": false,
+ "suggestAccountBind": false,
+ "UserStoryStatus": 0,
+ "shortDramaCreator": {}
+ },
+ "music": {
+ "id": "7655821492566919966",
+ "title": "original sound",
+ "playUrl": "https://v16m.tiktokcdn-us.com/84ccc1629d7199ad68f976d419148401/6a4ec44e/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7ThasbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=20260708154135271AF032BAC01F42EF1E&btag=e00050000",
+ "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "coverThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5",
+ "authorName": "Cookwithhali",
+ "original": true,
+ "private": false,
+ "duration": 62,
+ "scheduleSearchTime": 0,
+ "collected": false,
+ "preciseDuration": {
+ "preciseDuration": 62.093063,
+ "preciseShootDuration": 62.093063,
+ "preciseAuditionDuration": 62.093063,
+ "preciseVideoDuration": 62.093063
+ },
+ "isCopyrighted": false,
+ "tt2dsp": {
+ "tt_to_dsp_song_infos": []
+ },
+ "is_unlimited_music": false,
+ "is_commerce_music": true,
+ "shoot_duration": 62
+ },
+ "challenges": [
+ {
+ "id": "6983",
+ "title": "bread",
+ "desc": "",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": ""
+ },
+ {
+ "id": "285169",
+ "title": "garlicbread",
+ "desc": "",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": ""
+ },
+ {
+ "id": "1643317566954502",
+ "title": "breadtok",
+ "desc": "",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": ""
+ },
+ {
+ "id": "7250",
+ "title": "baking",
+ "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.",
+ "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": ""
+ },
+ {
+ "id": "10488587",
+ "title": "homemadebread",
+ "desc": "",
+ "profileLarger": "",
+ "profileMedium": "",
+ "profileThumb": "",
+ "coverLarger": "",
+ "coverMedium": "",
+ "coverThumb": ""
+ }
+ ],
+ "stats": {
+ "diggCount": 754100,
+ "shareCount": 144500,
+ "commentCount": 3720,
+ "playCount": 8300000,
+ "collectCount": "396399"
+ },
+ "statsV2": {
+ "diggCount": "754100",
+ "shareCount": "144500",
+ "commentCount": "3720",
+ "playCount": "8300000",
+ "collectCount": "396399",
+ "repostCount": "0"
+ },
+ "warnInfo": [],
+ "originalItem": false,
+ "officalItem": false,
+ "textExtra": [
+ {
+ "awemeId": "",
+ "start": 1024,
+ "end": 1030,
+ "hashtagId": "6983",
+ "hashtagName": "bread",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 1031,
+ "end": 1043,
+ "hashtagId": "285169",
+ "hashtagName": "garlicbread",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 1044,
+ "end": 1053,
+ "hashtagId": "1643317566954502",
+ "hashtagName": "breadtok",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 1054,
+ "end": 1061,
+ "hashtagId": "7250",
+ "hashtagName": "baking",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 1062,
+ "end": 1076,
+ "hashtagId": "10488587",
+ "hashtagName": "homemadebread",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ }
+ ],
+ "secret": false,
+ "forFriend": false,
+ "digged": false,
+ "itemCommentStatus": 0,
+ "takeDown": 0,
+ "effectStickers": [],
+ "authorStats": {
+ "followerCount": 372600,
+ "followingCount": 0,
+ "heart": 8000000,
+ "heartCount": 8000000,
+ "videoCount": 279,
+ "diggCount": 223,
+ "friendCount": 0
+ },
+ "privateItem": false,
+ "duetEnabled": true,
+ "stitchEnabled": true,
+ "stickersOnItem": [
+ {
+ "stickerText": [
+ "Cheesy Pull Apart Garlic Bread! 🧄🧀\n",
+ "(Bakery-Style & No-Knead)"
+ ],
+ "stickerType": 4
+ }
+ ],
+ "isAd": false,
+ "shareEnabled": true,
+ "comments": [],
+ "duetDisplay": 0,
+ "stitchDisplay": 0,
+ "indexEnabled": true,
+ "diversificationLabels": [
+ "Cooking",
+ "Food & Drink",
+ "Lifestyle"
+ ],
+ "locationCreated": "US",
+ "suggestedWords": [
+ "Garlic Bread",
+ "garlic bread recipe",
+ "pull apart bread",
+ "Sourdough Bread",
+ "How To Make Garlic Bread",
+ "Baking",
+ "food",
+ "bread",
+ "Baking Recipes",
+ "Garlic Pull Apart Bread Recipe"
+ ],
+ "contents": [
+ {
+ "desc": "Pull Apart Garlic Bread ",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "Ingredients",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "* 1 tsp yeast",
+ "textExtra": []
+ },
+ {
+ "desc": "* 1 tbsp sugar",
+ "textExtra": []
+ },
+ {
+ "desc": "* 270g milk (about 1 cup + 2 tbsp)",
+ "textExtra": []
+ },
+ {
+ "desc": "* 380g flour (about 3 cups)",
+ "textExtra": []
+ },
+ {
+ "desc": "* 1 tsp salt",
+ "textExtra": []
+ },
+ {
+ "desc": "* 43g softened butter (about 3 tbsp)",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "Butter Mixture",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "* 100g butter, softened",
+ "textExtra": []
+ },
+ {
+ "desc": "* 1 tbsp chopped cilantro",
+ "textExtra": []
+ },
+ {
+ "desc": "* 1 garlic clove, minced",
+ "textExtra": []
+ },
+ {
+ "desc": "* Shredded cheese",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "Instructions",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined.",
+ "textExtra": []
+ },
+ {
+ "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated.",
+ "textExtra": []
+ },
+ {
+ "desc": "3. Cover and let the dough rest for 2 hours or until doubled.",
+ "textExtra": []
+ },
+ {
+ "desc": "4. Mix the softened butter with the cilantro and minced garlic.",
+ "textExtra": []
+ },
+ {
+ "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese.",
+ "textExtra": []
+ },
+ {
+ "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan.",
+ "textExtra": []
+ },
+ {
+ "desc": "7. Cover and let rest for 20 minutes.",
+ "textExtra": []
+ },
+ {
+ "desc": "8. Bake at 370°F for 30 minutes, or until golden brown.",
+ "textExtra": []
+ },
+ {
+ "desc": "9. Brush with the remaining garlic butter mixture while still warm.",
+ "textExtra": []
+ },
+ {
+ "desc": "",
+ "textExtra": []
+ },
+ {
+ "desc": "#bread #garlicbread #breadtok #baking #homemadebread ",
+ "textExtra": [
+ {
+ "awemeId": "",
+ "start": 0,
+ "end": 6,
+ "hashtagId": "6983",
+ "hashtagName": "bread",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 7,
+ "end": 19,
+ "hashtagId": "285169",
+ "hashtagName": "garlicbread",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 20,
+ "end": 29,
+ "hashtagId": "1643317566954502",
+ "hashtagName": "breadtok",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 30,
+ "end": 37,
+ "hashtagId": "7250",
+ "hashtagName": "baking",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ },
+ {
+ "awemeId": "",
+ "start": 38,
+ "end": 52,
+ "hashtagId": "10488587",
+ "hashtagName": "homemadebread",
+ "type": 1,
+ "subType": 0,
+ "isCommerce": false
+ }
+ ]
+ }
+ ],
+ "diversificationId": 10040,
+ "anchors": [
+ {
+ "id": "0",
+ "type": 54,
+ "keyword": "CapCut · Editing made easy",
+ "icon": {
+ "uri": "tiktok-obj/capcut_logo_64px_bk.png",
+ "urlList": [
+ "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=lzVxquDTAbut%2BvshGEhsFsYQc2s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5"
+ ]
+ },
+ "schema": "https://www.capcut.com/tools/desktop-video-editor?channel=capcutpc_ttweb_anchor_edit1&download_channel=capcutpc_ttweb_anchor_edit1&enter_from=capcutpc_ttweb_anchor_edit1&force_dp=1&from_page=capcutpc_ttweb_anchor_edit1&type=tools",
+ "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}",
+ "description": "CapCut · Video Editor",
+ "thumbnail": {
+ "uri": "tiktok-obj/64x_Capcut3x.png",
+ "urlList": [
+ "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5",
+ "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=NBemEand1jjAoV81cQ1ANQzVqqc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5"
+ ],
+ "width": 64,
+ "height": 64
+ },
+ "extraInfo": {
+ "subtype": ""
+ }
+ }
+ ],
+ "collected": false,
+ "channelTags": [],
+ "item_control": {
+ "can_repost": true
+ },
+ "IsAigc": false,
+ "AIGCDescription": "",
+ "backendSourceEventTracking": "",
+ "CategoryType": 111,
+ "textLanguage": "en",
+ "textTranslatable": true,
+ "authorStatsV2": {
+ "followerCount": "372600",
+ "followingCount": "0",
+ "heart": "8000000",
+ "heartCount": "8000000",
+ "videoCount": "279",
+ "diggCount": "223",
+ "friendCount": "0"
+ },
+ "isReviewing": false,
+ "creatorAIComment": {
+ "hasAITopic": false,
+ "categoryList": [],
+ "eligibleVideo": false,
+ "notEligibleReason": 101
+ }
+}
\ No newline at end of file
From 2943d8b23cb1b03f5b97a07eecf9298b2fafe2d1 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 18:21:44 +0200
Subject: [PATCH 029/160] feat(tiktok): tiktok.scrape capability + billing
wire-up
---
.../app/capabilities/core/billing.py | 2 +
.../app/capabilities/core/types.py | 1 +
.../app/capabilities/tiktok/__init__.py | 5 ++
.../capabilities/tiktok/scrape/__init__.py | 1 +
.../capabilities/tiktok/scrape/definition.py | 23 +++++
.../capabilities/tiktok/scrape/executor.py | 48 +++++++++++
.../app/capabilities/tiktok/scrape/schemas.py | 86 +++++++++++++++++++
surfsense_backend/app/config/__init__.py | 3 +
surfsense_backend/app/routes/__init__.py | 1 +
.../unit/capabilities/tiktok/__init__.py | 0
.../capabilities/tiktok/scrape/__init__.py | 0
.../tiktok/scrape/test_executor.py | 79 +++++++++++++++++
.../tiktok/scrape/test_schemas.py | 45 ++++++++++
.../unit/capabilities/tiktok/test_registry.py | 23 +++++
14 files changed, 317 insertions(+)
create mode 100644 surfsense_backend/app/capabilities/tiktok/__init__.py
create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/__init__.py
create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/definition.py
create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/executor.py
create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/__init__.py
create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py
create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py
create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py
create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py
diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py
index 71aa45c58..048797473 100644
--- a/surfsense_backend/app/capabilities/core/billing.py
+++ b/surfsense_backend/app/capabilities/core/billing.py
@@ -35,6 +35,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = {
BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW",
BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO",
BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT",
+ BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO",
}
@@ -51,6 +52,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = {
BillingUnit.GOOGLE_MAPS_REVIEW: "review",
BillingUnit.YOUTUBE_VIDEO: "video",
BillingUnit.YOUTUBE_COMMENT: "comment",
+ BillingUnit.TIKTOK_VIDEO: "video",
}
diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py
index c87601832..b45641a2d 100644
--- a/surfsense_backend/app/capabilities/core/types.py
+++ b/surfsense_backend/app/capabilities/core/types.py
@@ -25,6 +25,7 @@ class BillingUnit(StrEnum):
GOOGLE_MAPS_REVIEW = "google_maps_review"
YOUTUBE_VIDEO = "youtube_video"
YOUTUBE_COMMENT = "youtube_comment"
+ TIKTOK_VIDEO = "tiktok_video"
class BillableInput(Protocol):
diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py
new file mode 100644
index 000000000..ed3f32682
--- /dev/null
+++ b/surfsense_backend/app/capabilities/tiktok/__init__.py
@@ -0,0 +1,5 @@
+"""``tiktok.*`` namespace: platform-native TikTok data verbs."""
+
+from __future__ import annotations
+
+from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401
diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py
new file mode 100644
index 000000000..9e5acc165
--- /dev/null
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py
@@ -0,0 +1 @@
+"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper."""
diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
new file mode 100644
index 000000000..9f8632f3b
--- /dev/null
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
@@ -0,0 +1,23 @@
+"""``tiktok.scrape`` capability registration (billed per video; see config
+``TIKTOK_MICROS_PER_VIDEO``)."""
+
+from __future__ import annotations
+
+from app.capabilities.core import BillingUnit, Capability, register_capability
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+
+TIKTOK_SCRAPE = Capability(
+ name="tiktok.scrape",
+ description=(
+ "Scrape public TikTok videos. Use urls, profiles, hashtags, or "
+ "search_queries."
+ ),
+ input_schema=ScrapeInput,
+ output_schema=ScrapeOutput,
+ executor=build_scrape_executor(),
+ billing_unit=BillingUnit.TIKTOK_VIDEO,
+ docs_url="/docs/connectors/native/tiktok",
+)
+
+register_capability(TIKTOK_SCRAPE)
diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/executor.py b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py
new file mode 100644
index 000000000..7ef71486d
--- /dev/null
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py
@@ -0,0 +1,48 @@
+"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from app.capabilities.core import Executor
+from app.capabilities.core.progress import emit_progress
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import (
+ TikTokAccessBlockedError,
+ TikTokScrapeInput,
+ scrape_tiktok,
+)
+
+ScrapeFn = Callable[..., Awaitable[list[dict]]]
+
+
+def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
+ """Bind the executor to a scraper fn (defaults to the proprietary actor)."""
+ scrape_fn = scrape_fn or scrape_tiktok
+
+ async def execute(payload: ScrapeInput) -> ScrapeOutput:
+ actor_input = TikTokScrapeInput(
+ startUrls=[{"url": url} for url in payload.urls],
+ profiles=payload.profiles,
+ hashtags=payload.hashtags,
+ searchQueries=payload.search_queries,
+ resultsPerPage=payload.results_per_page,
+ )
+ emit_progress(
+ "starting", "Resolving TikTok targets", total=payload.max_items, unit="item"
+ )
+ try:
+ items = await scrape_fn(actor_input, limit=payload.max_items)
+ except TikTokAccessBlockedError as exc:
+ # Anonymous-only scraper; a hard block can't be retried with creds.
+ raise ForbiddenError(
+ f"TikTok refused anonymous access: {exc}",
+ code="TIKTOK_ACCESS_BLOCKED",
+ ) from exc
+ emit_progress(
+ "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item"
+ )
+ return ScrapeOutput(items=items)
+
+ return execute
diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
new file mode 100644
index 000000000..697a94043
--- /dev/null
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
@@ -0,0 +1,86 @@
+"""``tiktok.scrape`` I/O contracts.
+
+A lean, agent-friendly surface over ``TikTokScrapeInput``
+(``app/proprietary/platforms/tiktok``). The executor maps this to the full
+scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the
+output element. Any TikTok URL kind (video, profile, hashtag, search) goes in
+``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, model_validator
+
+from app.proprietary.platforms.tiktok import TikTokVideoItem
+
+MAX_TIKTOK_SOURCES = 20
+"""Per-call cap on each source list: bounds a synchronous request's fan-out."""
+
+MAX_TIKTOK_ITEMS = 100
+"""Hard ceiling on items returned per call, regardless of the per-target count."""
+
+
+class ScrapeInput(BaseModel):
+ urls: list[str] = Field(
+ default_factory=list,
+ max_length=MAX_TIKTOK_SOURCES,
+ description=(
+ "TikTok URLs to scrape: a video, a profile (/@), a hashtag "
+ "(/tag/), or a search URL. Provide these OR profiles/hashtags/"
+ "search_queries (at least one source is required)."
+ ),
+ )
+ profiles: list[str] = Field(
+ default_factory=list,
+ max_length=MAX_TIKTOK_SOURCES,
+ description="Profile usernames (with or without a leading '@').",
+ )
+ hashtags: list[str] = Field(
+ default_factory=list,
+ max_length=MAX_TIKTOK_SOURCES,
+ description="Hashtag names to scrape, without the leading '#'.",
+ )
+ search_queries: list[str] = Field(
+ default_factory=list,
+ max_length=MAX_TIKTOK_SOURCES,
+ description="Search terms to run on TikTok.",
+ )
+ results_per_page: int = Field(
+ default=10,
+ ge=1,
+ le=MAX_TIKTOK_ITEMS,
+ description="Max videos to pull per profile/hashtag/search target.",
+ )
+ max_items: int = Field(
+ default=10,
+ ge=1,
+ le=MAX_TIKTOK_ITEMS,
+ description="Max total items to return across all sources.",
+ )
+
+ @model_validator(mode="after")
+ def _require_a_source(self) -> ScrapeInput:
+ if not any((self.urls, self.profiles, self.hashtags, self.search_queries)):
+ raise ValueError(
+ "Provide at least one of 'urls', 'profiles', 'hashtags', or "
+ "'search_queries'."
+ )
+ return self
+
+ @property
+ def estimated_units(self) -> int:
+ """Worst-case billable items for the pre-flight gate: ``max_items`` is a
+ hard cross-source ceiling (le=100), so no call can exceed it."""
+ return self.max_items
+
+
+class ScrapeOutput(BaseModel):
+ items: list[TikTokVideoItem] = Field(
+ default_factory=list,
+ description="One item per video returned, in emission order.",
+ )
+
+ @property
+ def billable_units(self) -> int:
+ """One returned item = one billable unit."""
+ return len(self.items)
diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py
index 5d8e96881..c03ac3598 100644
--- a/surfsense_backend/app/config/__init__.py
+++ b/surfsense_backend/app/config/__init__.py
@@ -714,6 +714,9 @@ class Config:
# Kept separate from the video rate so comments can be re-tuned toward the
# cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.
YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500"))
+ # Browser-driven listings make TikTok heavier per item than the API-backed
+ # video meter, so it sits a touch above YouTube's video rate.
+ TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500"))
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
# so the UI can nudge the user to top up / enable auto-reload. $0.50.
diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py
index 1a5b182b8..3c28fd65c 100644
--- a/surfsense_backend/app/routes/__init__.py
+++ b/surfsense_backend/app/routes/__init__.py
@@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends
import app.capabilities.google_maps
import app.capabilities.google_search
import app.capabilities.reddit
+import app.capabilities.tiktok
import app.capabilities.web
import app.capabilities.youtube # noqa: F401
from app.automations.api import router as automations_router
diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py
new file mode 100644
index 000000000..1bf07b0c3
--- /dev/null
+++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py
@@ -0,0 +1,79 @@
+"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items.
+
+Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
+own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities.tiktok.scrape.executor import build_scrape_executor
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+from app.exceptions import ForbiddenError
+from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput
+
+pytestmark = pytest.mark.unit
+
+
+class _FakeScraper:
+ """Records the actor input + limit it was called with; returns canned items."""
+
+ def __init__(self, items: list[dict]):
+ self._items = items
+ self.calls: list[tuple[TikTokScrapeInput, int | None]] = []
+
+ async def __call__(
+ self, actor_input: TikTokScrapeInput, *, limit: int | None = None
+ ) -> list[dict]:
+ self.calls.append((actor_input, limit))
+ return self._items
+
+
+async def test_maps_urls_to_start_urls_and_wraps_items():
+ scraper = _FakeScraper([{"id": "123", "text": "hello"}])
+ execute = build_scrape_executor(scrape_fn=scraper)
+
+ out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"]))
+
+ assert isinstance(out, ScrapeOutput)
+ assert len(out.items) == 1
+ assert out.items[0].id == "123"
+
+ (actor_input, _limit) = scraper.calls[0]
+ assert [u.url for u in actor_input.startUrls] == [
+ "https://www.tiktok.com/@nasa/video/123"
+ ]
+
+
+async def test_forwards_typed_sources_and_limit():
+ scraper = _FakeScraper([])
+ execute = build_scrape_executor(scrape_fn=scraper)
+
+ await execute(
+ ScrapeInput(
+ profiles=["nasa"],
+ hashtags=["food"],
+ search_queries=["cats"],
+ results_per_page=7,
+ max_items=25,
+ )
+ )
+
+ (actor_input, limit) = scraper.calls[0]
+ assert actor_input.profiles == ["nasa"]
+ assert actor_input.hashtags == ["food"]
+ assert actor_input.searchQueries == ["cats"]
+ assert actor_input.resultsPerPage == 7
+ # The outer collection limit is the caller's total-item cap.
+ assert limit == 25
+
+
+async def test_access_blocked_maps_to_forbidden():
+ async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None):
+ raise TikTokAccessBlockedError("all IPs refused")
+
+ execute = build_scrape_executor(scrape_fn=_blocked)
+
+ with pytest.raises(ForbiddenError):
+ await execute(ScrapeInput(hashtags=["food"]))
diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py
new file mode 100644
index 000000000..e7c978bcc
--- /dev/null
+++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py
@@ -0,0 +1,45 @@
+"""``tiktok.scrape`` input guards: a source is required and the batch is bounded."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from app.capabilities.tiktok.scrape.schemas import (
+ MAX_TIKTOK_ITEMS,
+ MAX_TIKTOK_SOURCES,
+ ScrapeInput,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def test_rejects_input_with_no_source():
+ with pytest.raises(ValidationError):
+ ScrapeInput()
+
+
+def test_accepts_urls_only():
+ payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"])
+ assert payload.profiles == []
+
+
+def test_accepts_hashtags_only():
+ payload = ScrapeInput(hashtags=["food"])
+ assert payload.hashtags == ["food"]
+
+
+def test_defaults_and_bounds():
+ payload = ScrapeInput(hashtags=["food"])
+ assert payload.max_items == 10
+ assert payload.results_per_page == 10
+ with pytest.raises(ValidationError):
+ ScrapeInput(hashtags=["food"], max_items=0)
+ with pytest.raises(ValidationError):
+ ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1)
+
+
+def test_rejects_more_sources_than_the_cap():
+ too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)]
+ with pytest.raises(ValidationError):
+ ScrapeInput(hashtags=too_many)
diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py
new file mode 100644
index 000000000..3a85fe87f
--- /dev/null
+++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py
@@ -0,0 +1,23 @@
+"""The tiktok namespace registers its verb as one Capability the doors/agent read."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.capabilities import (
+ tiktok, # noqa: F401 — importing the namespace registers its verbs
+)
+from app.capabilities.core import BillingUnit
+from app.capabilities.core.store import get_capability
+from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput
+
+pytestmark = pytest.mark.unit
+
+
+def test_tiktok_scrape_is_registered_and_billed_per_video():
+ cap = get_capability("tiktok.scrape")
+
+ assert cap.name == "tiktok.scrape"
+ assert cap.input_schema is ScrapeInput
+ assert cap.output_schema is ScrapeOutput
+ assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO
From ed1c3a1f3d43cb77e214c88d87e1a793080ac9d8 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 20:20:20 +0200
Subject: [PATCH 030/160] feat(tiktok): expose tiktok.scrape on MCP, chat
subagent, playground
---
.../agents/chat/multi_agent_chat/constants.py | 1 +
.../subagents/builtins/tiktok/__init__.py | 1 +
.../subagents/builtins/tiktok/agent.py | 43 +++++++++
.../subagents/builtins/tiktok/description.md | 2 +
.../builtins/tiktok/system_prompt.md | 64 ++++++++++++++
.../builtins/tiktok/tools/__init__.py | 0
.../subagents/builtins/tiktok/tools/index.py | 27 ++++++
.../multi_agent_chat/subagents/registry.py | 4 +
.../test_subagent_composition.py | 1 +
.../mcp_server/features/scrapers/__init__.py | 4 +-
.../features/scrapers/platforms/tiktok.py | 88 +++++++++++++++++++
surfsense_mcp/mcp_server/selfcheck.py | 1 +
surfsense_web/lib/playground/catalog.ts | 7 ++
.../lib/playground/platform-icons.tsx | 1 +
surfsense_web/public/connectors/tiktok.svg | 6 ++
15 files changed, 248 insertions(+), 2 deletions(-)
create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py
create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py
create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md
create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py
create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py
create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
create mode 100644 surfsense_web/public/connectors/tiktok.svg
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py
index b132618d7..c396fceaf 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py
@@ -36,6 +36,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
"google_maps": frozenset(),
"google_search": frozenset(),
"reddit": frozenset(),
+ "tiktok": frozenset(),
"mcp_discovery": frozenset(
{
"SLACK_CONNECTOR",
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py
new file mode 100644
index 000000000..ec7d5955f
--- /dev/null
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py
@@ -0,0 +1 @@
+"""``tiktok`` builtin subagent: structured public TikTok videos and listings."""
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py
new file mode 100644
index 000000000..2c7dd014b
--- /dev/null
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py
@@ -0,0 +1,43 @@
+"""``tiktok`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
+ read_md_file,
+)
+from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
+from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
+ pack_subagent,
+)
+
+from .tools.index import NAME, RULESET, load_tools
+
+
+def build_subagent(
+ *,
+ dependencies: dict[str, Any],
+ model: BaseChatModel | None = None,
+ middleware_stack: dict[str, Any] | None = None,
+ mcp_tools: list[BaseTool] | None = None,
+) -> SurfSenseSubagentSpec:
+ tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
+ description = (
+ read_md_file(__package__, "description").strip()
+ or "Pulls structured data from public TikTok videos, hashtags, and searches."
+ )
+ system_prompt = read_md_file(__package__, "system_prompt").strip()
+ return pack_subagent(
+ name=NAME,
+ description=description,
+ system_prompt=system_prompt,
+ tools=tools,
+ ruleset=RULESET,
+ dependencies=dependencies,
+ model=model,
+ middleware_stack=middleware_stack,
+ )
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md
new file mode 100644
index 000000000..fc13d864d
--- /dev/null
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md
@@ -0,0 +1,2 @@
+TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL. Also compares fresh TikTok results against earlier findings in this chat.
+Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, or scrape a specific video URL. Triggers include "search TikTok for X", "trending TikTok videos about X", "videos with #X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist).
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
new file mode 100644
index 000000000..49c20164e
--- /dev/null
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
@@ -0,0 +1,64 @@
+You are the SurfSense TikTok sub-agent.
+You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
+
+
+Answer the delegated question from live TikTok data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it.
+
+
+
+- `tiktok_scrape`
+- `read_run` / `search_run` (free readers for stored scrape output)
+
+
+
+- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`.
+- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`.
+- Profiles: a creator's `profiles` feed can come back empty — TikTok restricts the profile video endpoint. Prefer `hashtags`, `search_queries`, or a direct video URL, and treat an empty profile result as a known limit, not a failure to retry endlessly.
+- Controlling volume: use `max_items` for the total cap and `results_per_page` per target.
+- Requested counts: `max_items` defaults to only 10 — when the task asks for N videos, set `max_items` and `results_per_page` above N. A call that caps below the target can never satisfy it.
+- Batch multiple hashtags or search terms into one call rather than many single-term calls.
+
+- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes).
+
+
+
+- Use only tools in ``.
+- Report only results present in the tool output. Never invent captions, URLs, authors, or counts.
+
+
+
+- Do not read arbitrary web pages — that belongs to the web crawling specialist.
+- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
+- Reddit belongs to the Reddit specialist; YouTube belongs to the YouTube specialist; Google results belong to the Google Search specialist.
+
+
+
+- Report uncertainty explicitly when evidence is incomplete or conflicting.
+- Never present unverified claims as facts.
+
+
+
+- Underspecified request — no usable hashtag, query, or URL — return `status=blocked` with the missing fields.
+- Tool failure: return `status=error` with a concise recovery `next_step`.
+- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
+
+
+
+Return **only** one JSON object (no markdown/prose):
+{
+ "status": "success" | "partial" | "blocked" | "error",
+ "action_summary": string,
+ "evidence": {
+ "findings": string[],
+ "sources": string[],
+ "confidence": "high" | "medium" | "low"
+ },
+ "next_step": string | null,
+ "missing_fields": string[] | null,
+ "assumptions": string[] | null
+}
+
+Route-specific rules:
+- `evidence.findings`: one entry per distinct video or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded).
+- `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once.
+
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py
new file mode 100644
index 000000000..e790d44f0
--- /dev/null
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py
@@ -0,0 +1,27 @@
+"""``tiktok`` sub-agent tools: the TikTok scrape capability verb."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.tools import BaseTool
+
+from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
+from app.capabilities.core.access.agent import build_capability_tools
+from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE
+
+NAME = "tiktok"
+
+RULESET = Ruleset(origin=NAME, rules=[])
+
+_CI_VERBS = [TIKTOK_SCRAPE]
+
+
+def load_tools(
+ *, dependencies: dict[str, Any] | None = None, **kwargs: Any
+) -> list[BaseTool]:
+ d = {**(dependencies or {}), **kwargs}
+ return build_capability_tools(
+ workspace_id=d.get("workspace_id"),
+ capabilities=_CI_VERBS,
+ )
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py
index 34895a514..71c2e6f3d 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py
@@ -33,6 +33,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import (
from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import (
build_subagent as build_reddit_subagent,
)
+from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import (
+ build_subagent as build_tiktok_subagent,
+)
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
build_subagent as build_web_crawler_subagent,
)
@@ -84,6 +87,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
"memory": build_memory_subagent,
"onedrive": build_onedrive_subagent,
"reddit": build_reddit_subagent,
+ "tiktok": build_tiktok_subagent,
"web_crawler": build_web_crawler_subagent,
"youtube": build_youtube_subagent,
}
diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py
index c3ad04250..66e8f28db 100644
--- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py
+++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py
@@ -37,6 +37,7 @@ _EXPECTED_SUBAGENTS = frozenset(
"memory",
"onedrive",
"reddit",
+ "tiktok",
"web_crawler",
"youtube",
}
diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py
index dfa2f3ab2..c0e7dea07 100644
--- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py
@@ -13,9 +13,9 @@ from mcp.server.fastmcp import FastMCP
from ...core.client import SurfSenseClient
from ...core.workspace_context import WorkspaceContext
from . import run_history
-from .platforms import google_maps, google_search, reddit, web, youtube
+from .platforms import google_maps, google_search, reddit, tiktok, web, youtube
-_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history)
+_REGISTRARS = (web, google_search, reddit, youtube, tiktok, google_maps, run_history)
def register(
diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
new file mode 100644
index 000000000..1e5f472ee
--- /dev/null
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
@@ -0,0 +1,88 @@
+"""TikTok scraper tool."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import Field
+
+from ....core.client import SurfSenseClient
+from ....core.rendering import ResponseFormatParam
+from ....core.workspace_context import WorkspaceContext, WorkspaceParam
+from ..annotations import SCRAPE
+from ..capability import run_scraper
+
+
+def register(
+ mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
+) -> None:
+ """Register the TikTok tool."""
+
+ @mcp.tool(
+ name="surfsense_tiktok_scrape",
+ title="Search or scrape TikTok",
+ annotations=SCRAPE,
+ structured_output=False,
+ )
+ async def tiktok_scrape(
+ urls: Annotated[
+ list[str] | None,
+ Field(
+ description="TikTok URLs: a video, a profile "
+ "('https://www.tiktok.com/@nasa'), a hashtag "
+ "('https://www.tiktok.com/tag/food'), or a search URL. Provide "
+ "urls OR profiles/hashtags/search_queries."
+ ),
+ ] = None,
+ profiles: Annotated[
+ list[str] | None,
+ Field(
+ description="Profile usernames to scrape, with or without a "
+ "leading '@', e.g. ['nasa']."
+ ),
+ ] = None,
+ hashtags: Annotated[
+ list[str] | None,
+ Field(
+ description="Hashtag names to scrape, without the '#', e.g. "
+ "['food']."
+ ),
+ ] = None,
+ search_queries: Annotated[
+ list[str] | None,
+ Field(description="Terms to search TikTok for, e.g. ['cooking']."),
+ ] = None,
+ results_per_page: Annotated[
+ int,
+ Field(ge=1, description="Max videos per profile/hashtag/search target."),
+ ] = 10,
+ max_items: Annotated[
+ int, Field(ge=1, description="Maximum videos to return in total.")
+ ] = 10,
+ workspace: WorkspaceParam = None,
+ response_format: ResponseFormatParam = "markdown",
+ ) -> str:
+ """Search or scrape public TikTok videos.
+
+ Use this for ANY TikTok research — a creator's videos, a hashtag feed,
+ a search, or a specific video URL — instead of a generic web search.
+ Returns videos with text, author, stats, music, and the web URL.
+ Example: hashtags=['food'], max_items=20.
+ """
+ return await run_scraper(
+ client,
+ context,
+ platform="tiktok",
+ verb="scrape",
+ payload={
+ "urls": urls,
+ "profiles": profiles,
+ "hashtags": hashtags,
+ "search_queries": search_queries,
+ "results_per_page": results_per_page,
+ "max_items": max_items,
+ },
+ workspace=workspace,
+ response_format=response_format,
+ )
diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py
index 20224c173..e5ac6bd2b 100644
--- a/surfsense_mcp/mcp_server/selfcheck.py
+++ b/surfsense_mcp/mcp_server/selfcheck.py
@@ -23,6 +23,7 @@ EXPECTED_TOOLS = {
"surfsense_reddit_scrape",
"surfsense_youtube_scrape",
"surfsense_youtube_comments",
+ "surfsense_tiktok_scrape",
"surfsense_google_maps_scrape",
"surfsense_google_maps_reviews",
"surfsense_list_scraper_runs",
diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts
index 9b508228a..4503662f8 100644
--- a/surfsense_web/lib/playground/catalog.ts
+++ b/surfsense_web/lib/playground/catalog.ts
@@ -3,6 +3,7 @@ import {
GoogleMapsIcon,
GoogleSearchIcon,
RedditIcon,
+ TikTokIcon,
WebIcon,
YouTubeIcon,
} from "./platform-icons";
@@ -48,6 +49,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
{ name: "youtube.comments", verb: "comments", label: "Comments" },
],
},
+ {
+ id: "tiktok",
+ label: "TikTok",
+ icon: TikTokIcon,
+ verbs: [{ name: "tiktok.scrape", verb: "scrape", label: "Scrape" }],
+ },
{
id: "google_maps",
label: "Google Maps",
diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx
index e7a443c2f..474bff317 100644
--- a/surfsense_web/lib/playground/platform-icons.tsx
+++ b/surfsense_web/lib/playground/platform-icons.tsx
@@ -24,6 +24,7 @@ function brandIcon(src: string, alt: string) {
export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit");
export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube");
+export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok");
export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps");
export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search");
export const WebIcon = brandIcon("/connectors/web.svg", "Web");
diff --git a/surfsense_web/public/connectors/tiktok.svg b/surfsense_web/public/connectors/tiktok.svg
new file mode 100644
index 000000000..c3c3fe06e
--- /dev/null
+++ b/surfsense_web/public/connectors/tiktok.svg
@@ -0,0 +1,6 @@
+
From ea6befdf595e91613b40d3c5f26f2f4dad116df5 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 20:21:29 +0200
Subject: [PATCH 031/160] docs(tiktok): native connector page + nav card
---
.../content/docs/connectors/native/index.mdx | 7 ++-
.../content/docs/connectors/native/meta.json | 2 +-
.../content/docs/connectors/native/tiktok.mdx | 45 +++++++++++++++++++
3 files changed, 52 insertions(+), 2 deletions(-)
create mode 100644 surfsense_web/content/docs/connectors/native/tiktok.mdx
diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx
index 5c5d24444..f915b5533 100644
--- a/surfsense_web/content/docs/connectors/native/index.mdx
+++ b/surfsense_web/content/docs/connectors/native/index.mdx
@@ -1,6 +1,6 @@
---
title: Native Connectors
-description: SurfSense's built-in scraper APIs for Reddit, YouTube, Google Maps, Google Search, and the web
+description: SurfSense's built-in scraper APIs for Reddit, YouTube, TikTok, Google Maps, Google Search, and the web
---
import { Card, Cards } from 'fumadocs-ui/components/card';
@@ -18,6 +18,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
description="Videos, channels, playlists, subtitles, and comments"
href="/docs/connectors/native/youtube"
/>
+ `), a hashtag (`/tag/`), or a search URL (max 20 sources per call) |
+| `profiles` | — | Profile usernames, with or without a leading `@` |
+| `hashtags` | — | Hashtag names, without the `#` |
+| `search_queries` | — | Search terms to run on TikTok |
+| `results_per_page` | `10` | Max videos per profile/hashtag/search target |
+| `max_items` | `10` | Max total videos returned across all sources (hard cap 100) |
+
+## Example
+
+```bash
+curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \
+ -H "Authorization: Bearer $SURFSENSE_API_KEY" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "hashtags": ["food"],
+ "max_items": 25
+ }'
+```
+
+The response is `{ "items": [...] }` — one item per video. Billing is per returned item.
+
+
+Video, hashtag, and search targets are the reliable paths. TikTok restricts the profile video endpoint for automated clients, so a `profiles` target can return no items even when the account is public — prefer hashtags, search, or a direct video URL.
+
+
+For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok → Scrape** in your workspace.
From 5f9472b122157c3b9ab8e590ae26bcdd2f0cfea9 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 20:30:38 +0200
Subject: [PATCH 032/160] feat(tiktok): advertise tiktok across agent prompts,
MCP instructions, docs
---
.../main_agent/system_prompt/prompts/identity/private.md | 6 +++---
.../main_agent/system_prompt/prompts/identity/team.md | 6 +++---
.../main_agent/system_prompt/prompts/kb_first.md | 5 +++--
.../main_agent/system_prompt/prompts/routing.md | 1 +
surfsense_mcp/README.md | 1 +
surfsense_mcp/mcp_server/server.py | 5 +++--
surfsense_web/content/docs/how-to/mcp-server.mdx | 4 ++--
7 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md
index 378c052d4..0c7e4071c 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md
@@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put
that intelligence to work alongside their own knowledge base.
You do this by dispatching **specialist subagents** via the `task` tool:
-- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the
- web crawler return structured, current platform data (posts, comments,
- transcripts, reviews, SERPs, full page content).
+- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search,
+ and the web crawler return structured, current platform data (posts,
+ comments, transcripts, videos, reviews, SERPs, full page content).
- **The user's own context** — their knowledge base, connected apps, and
persistent memory.
- **Deliverables** — reports, podcasts, and presentations built from what the
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md
index 2765c06b7..6c1076a89 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md
@@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put
that intelligence to work alongside the team's shared knowledge base.
You do this by dispatching **specialist subagents** via the `task` tool:
-- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the
- web crawler return structured, current platform data (posts, comments,
- transcripts, reviews, SERPs, full page content).
+- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search,
+ and the web crawler return structured, current platform data (posts,
+ comments, transcripts, videos, reviews, SERPs, full page content).
- **The team's own context** — its shared knowledge base, connected apps, and
persistent team memory.
- **Deliverables** — reports, podcasts, and presentations built from what the
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md
index c69ed400c..ef753b93c 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md
@@ -1,8 +1,9 @@
CRITICAL — ground factual answers in what you actually receive this turn:
- **live platform data** via the market specialists —
- `task(reddit, ...)`, `task(youtube, ...)`, `task(google_maps, ...)`,
- `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about
+ `task(reddit, ...)`, `task(youtube, ...)`, `task(tiktok, ...)`,
+ `task(google_maps, ...)`, `task(google_search, ...)`,
+ `task(web_crawler, ...)`. Anything about
competitors, markets, rankings, reviews, or audience sentiment is answered
from what these return **this turn**, never from your training data: your
general knowledge of companies, prices, and rankings is stale by definition,
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md
index eec860f3c..fb818cabc 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md
@@ -31,6 +31,7 @@ bounded fan-out (≤20 sites) the user already requested.
about a brand, product, or topic is answered from the platform where they
say it — `task(reddit, …)` for community discussion and threads,
`task(youtube, …)` for video content, transcripts, and comment sections,
+`task(tiktok, …)` for short-form video trends by hashtag or search,
`task(google_maps, …)` for customer reviews of physical businesses. Web
search only finds articles *about* the conversation; the platform
specialists return the conversation itself, structured and current. For
diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md
index c18a857e1..5bac5f8b2 100644
--- a/surfsense_mcp/README.md
+++ b/surfsense_mcp/README.md
@@ -21,6 +21,7 @@ Connect it two ways:
**Scrapers (all platforms)**
- `surfsense_web_crawl`, `surfsense_google_search`, `surfsense_reddit_scrape`,
`surfsense_youtube_scrape`, `surfsense_youtube_comments`,
+ `surfsense_tiktok_scrape`,
`surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`
- `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` — retrieve past
results in full (useful when a large result was truncated inline)
diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py
index 8ea9bc919..c0398ed50 100644
--- a/surfsense_mcp/mcp_server/server.py
+++ b/surfsense_mcp/mcp_server/server.py
@@ -36,8 +36,9 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]:
"SurfSense gives you live scrapers and a personal knowledge base. "
"Prefer these tools over generic/built-in web search whenever the "
"task involves Reddit (posts, comments, finding subreddits or "
- "communities), YouTube (videos, transcripts, comments), Google "
- "Maps (places, reviews), Google Search results, or reading "
+ "communities), YouTube (videos, transcripts, comments), TikTok "
+ "(videos by hashtag, search, or URL), Google Maps (places, "
+ "reviews), Google Search results, or reading "
"specific web pages. Scraper results are persisted as runs; if an "
"inline result is truncated, fetch it in full with "
"surfsense_get_scraper_run."
diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx
index f5f419219..67b218d58 100644
--- a/surfsense_web/content/docs/how-to/mcp-server.mdx
+++ b/surfsense_web/content/docs/how-to/mcp-server.mdx
@@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
# SurfSense MCP Server
-The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
+The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 19 native, typed tools: every scraper (Reddit, YouTube, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.
@@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie
| Group | Tools |
|-------|-------|
| Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` |
-| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` |
+| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` |
| Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` |
Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**.
From f731d371abd803514735b73d36b19b2205496875 Mon Sep 17 00:00:00 2001
From: CREDO23
Date: Wed, 8 Jul 2026 20:36:46 +0200
Subject: [PATCH 033/160] feat(tiktok): add TikTok SEO landing page and
site-wide marketing parity
---
surfsense_web/app/(home)/mcp-server/page.tsx | 13 +-
.../components/homepage/hero-section.tsx | 2 +-
.../components/homepage/persona-paths.tsx | 2 +-
.../components/pricing/pricing-section.tsx | 4 +-
surfsense_web/components/seo/json-ld.tsx | 4 +-
.../content/docs/connectors/index.mdx | 2 +-
.../lib/connectors-marketing/index.ts | 2 +
.../lib/connectors-marketing/reddit.tsx | 2 +-
.../lib/connectors-marketing/tiktok.tsx | 277 ++++++++++++++++++
.../lib/connectors-marketing/youtube.tsx | 2 +-
10 files changed, 297 insertions(+), 13 deletions(-)
create mode 100644 surfsense_web/lib/connectors-marketing/tiktok.tsx
diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx
index 5e4961b1e..22c867c73 100644
--- a/surfsense_web/app/(home)/mcp-server/page.tsx
+++ b/surfsense_web/app/(home)/mcp-server/page.tsx
@@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types";
const canonicalUrl = "https://www.surfsense.com/mcp-server";
const metaDescription =
- "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
+ "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
export const metadata: Metadata = {
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
@@ -93,6 +93,7 @@ const TOOL_GROUPS = [
"surfsense_reddit_scrape",
"surfsense_youtube_scrape",
"surfsense_youtube_comments",
+ "surfsense_tiktok_scrape",
"surfsense_google_maps_scrape",
"surfsense_google_maps_reviews",
"surfsense_google_search",
@@ -127,7 +128,7 @@ const FAQ: FaqItem[] = [
{
question: "What is the SurfSense MCP server?",
answer:
- "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
+ "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, TikTok, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
},
{
question: "Which MCP clients does it work with?",
@@ -215,8 +216,9 @@ export default function McpServerPage() {
The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform:
- scrape Reddit, YouTube, Google Maps, Google Search, and the open web, and search,
- read, and write your knowledge base. One API key, typed tools, pay as you go.
+ scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, and
+ search, read, and write your knowledge base. One API key, typed tools, pay as you
+ go.
-# SurfSense: Dê Inteligência Competitiva aos Seus Agentes de IA
+# SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva
-O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações.
+O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações.
> [!NOTE]
> **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM**
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 7d2ae35c3..11bace243 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -20,9 +20,9 @@
-# SurfSense:为你的 AI 智能体注入竞争情报能力
+# SurfSense:面向竞争情报研究的 NotebookLM
-SurfSense 是**面向 AI 智能体的开源竞争情报平台**。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。
+SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 NotebookLM,但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。
> [!NOTE]
> **📢 致我们的 NotebookLM 替代品用户**
From ab5970fab11939ec0e4d9b258a9e9b10ccbbe3c1 Mon Sep 17 00:00:00 2001
From: "DESKTOP-RTLN3BA\\$punk"
Date: Mon, 13 Jul 2026 14:42:11 -0700
Subject: [PATCH 156/160] feat: updating messaging of instagram and tiktok
pages
---
surfsense_web/lib/auth-utils.ts | 1 +
.../lib/connectors-marketing/instagram.tsx | 47 +++++++-----
.../lib/connectors-marketing/tiktok.tsx | 76 +++++++++++--------
3 files changed, 72 insertions(+), 52 deletions(-)
diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts
index 8c8d919c5..7d9320fb0 100644
--- a/surfsense_web/lib/auth-utils.ts
+++ b/surfsense_web/lib/auth-utils.ts
@@ -40,6 +40,7 @@ const PUBLIC_ROUTE_PREFIXES = [
"/external-mcp-connectors",
"/reddit",
"/instagram",
+ "/tiktok",
"/youtube",
"/google-maps",
"/google-search",
diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx
index d22400b7a..5422a0c90 100644
--- a/surfsense_web/lib/connectors-marketing/instagram.tsx
+++ b/surfsense_web/lib/connectors-marketing/instagram.tsx
@@ -6,28 +6,32 @@ export const instagram: ConnectorPageContent = {
name: "Instagram",
icon: IconBrandInstagram,
- metaTitle: "Instagram Scraper API for Creator Research | SurfSense",
+ metaTitle: "Instagram Scraper API for Profiles and Reels | SurfSense",
metaDescription:
- "Scrape public Instagram posts, reels, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.",
+ "Instagram scraper API for public profiles, posts, and reels. No login or Graph API review. Structured data for AI agents, plus a free tier. Start now.",
keywords: [
"instagram scraper",
"instagram scraper api",
"instagram api",
"instagram api alternative",
"scrape instagram",
+ "instagram scraping",
"instagram graph api alternative",
"instagram profile scraper",
"instagram post scraper",
+ "instagram posts scraper",
"instagram reel scraper",
+ "instagram reels scraper",
+ "instagram data scraper",
"instagram data api",
"instagram mcp server",
"creator research",
"social listening",
],
- h1: "Instagram Scraper API for Creator Research and Social Listening",
+ h1: "Instagram Scraper API for Profiles, Posts, and Reels",
heroLede:
- "The SurfSense Instagram API extracts public posts, reels, and profile details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.",
+ "The SurfSense Instagram scraper extracts public profiles, posts, and reels without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.",
transcript: {
prompt: "Pull recent reels from @competitor and summarize what they're posting",
@@ -54,48 +58,48 @@ export const instagram: ConnectorPageContent = {
},
extractIntro:
- "Every call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.",
+ "Every Instagram scraper call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.",
extractFields: [
{
- label: "Posts & Reels",
+ label: "Posts and reels",
description:
- "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.",
+ "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp from any public post or reel.",
},
{
- label: "Profiles",
+ label: "Profile data",
description:
- "Follower, following, and post counts, bio, external URL, verified and business flags.",
+ "Follower, following, and post counts, bio, external URL, verified and business flags for public Instagram profiles.",
},
{
- label: "Owner & Media",
+ label: "Owner and media",
description:
"Owner username and id on every item, plus image and video URLs, alt text, and view counts.",
},
],
- useCasesHeading: "What teams do with the Instagram API",
+ useCasesHeading: "What teams do with the Instagram scraper API",
useCases: [
{
title: "Creator and competitor monitoring",
description:
- "Track what your competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.",
+ "Scrape Instagram profiles and feeds to track what competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.",
},
{
title: "Content and format research",
description:
- "Study a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.",
+ "Pull a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.",
},
{
title: "Influencer vetting and outreach",
description:
- "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership.",
+ "Use the Instagram profile scraper to verify follower count, post cadence, and real engagement from public data before you pay for a partnership.",
},
],
comparison: {
- heading: "An Instagram API alternative built for agents",
+ heading: "An Instagram Graph API alternative built for agents",
intro:
- "The official Instagram Graph API requires a Business account, app review, and access tokens, and it can't read arbitrary public profiles. Here is how SurfSense compares.",
+ "The official Instagram Graph API requires a Business account, app review, and access tokens, and it cannot read arbitrary public profiles. SurfSense is an Instagram API alternative for public data. Here is how it compares.",
columnLabel: "Instagram Graph API",
rows: [
{
@@ -259,7 +263,12 @@ export const instagram: ConnectorPageContent = {
{
question: "Do I need an Instagram account or the Graph API?",
answer:
- "No. This is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured data back.",
+ "No. This Instagram scraper API is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call SurfSense with one key, or add the MCP server to your agent, and get structured profile, post, and reel data back.",
+ },
+ {
+ question: "What Instagram data can I scrape?",
+ answer:
+ "Public profiles, posts, and reels. Each item includes captions, hashtags, mentions, engagement counts, media URLs, and owner metadata. Point the Instagram profile scraper at a handle, or pass post and reel URLs directly. Discover creators with search queries when you do not have a URL yet.",
},
{
question: "What are the rate limits?",
@@ -274,11 +283,11 @@ export const instagram: ConnectorPageContent = {
],
related: [
- { label: "Reddit API", href: "/reddit" },
+ { label: "TikTok API", href: "/tiktok" },
{ label: "YouTube API", href: "/youtube" },
+ { label: "Reddit API", href: "/reddit" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
- { label: "Read the docs", href: "/docs" },
],
};
diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx
index d77865633..f245de92b 100644
--- a/surfsense_web/lib/connectors-marketing/tiktok.tsx
+++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx
@@ -6,30 +6,33 @@ export const tiktok: ConnectorPageContent = {
name: "TikTok",
icon: IconBrandTiktok,
- metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense",
+ metaTitle: "TikTok Scraper API for Videos and Comments | SurfSense",
metaDescription:
- "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, profile, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.",
+ "TikTok scraper API for public videos, comments, hashtags, and profiles. No Research API approval. Structured data for AI agents, plus a free tier. Start now.",
keywords: [
"tiktok scraper",
"tiktok scraper api",
"tiktok api",
"tiktok api alternative",
"scrape tiktok",
+ "tiktok scraping",
+ "tiktok research api alternative",
"tiktok data api",
- "tiktok hashtag scraper",
"tiktok comments scraper",
+ "tiktok comment scraper",
+ "tiktok hashtag scraper",
+ "tiktok profile scraper",
+ "tiktok video scraper",
"tiktok trending scraper",
"tiktok user search",
- "tiktok trend tracking",
"tiktok mcp",
"social listening",
"influencer research tool",
- "short-form video analytics",
],
- h1: "TikTok Scraper API for Trend and Creator Research",
+ h1: "TikTok Scraper API for Videos, Comments, and Trend Research",
heroLede:
- "The SurfSense TikTok API extracts public videos by hashtag, creator profile, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.",
+ "The SurfSense TikTok scraper extracts public videos by hashtag, creator profile, or URL, plus comment threads and trending feeds, without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.",
transcript: {
prompt: "Find trending TikToks about meal prep this week",
@@ -55,46 +58,47 @@ export const tiktok: ConnectorPageContent = {
},
extractIntro:
- "Every call returns structured items. Scrape videos from a hashtag, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.",
+ "Every TikTok scraper call returns structured items. Scrape videos from a hashtag, creator profile, or video URL, or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.",
extractFields: [
{
label: "Videos",
- description: "Caption text, canonical web URL, duration, and cover image for each video.",
+ description: "Caption text, canonical web URL, duration, and cover image for each TikTok video.",
},
{
label: "Engagement",
description:
- "Play, like, comment, share, and save counts — the signal for what is breaking out.",
+ "Play, like, comment, share, and save counts, the signal for what is breaking out.",
},
{
- label: "Authors",
+ label: "Authors and profiles",
description: "Creator handle, nickname, follower and heart counts, and verified status.",
},
+ {
+ label: "Comments",
+ description:
+ "Public comment threads on any video URL, so you can read audience reaction beyond vanity views.",
+ },
{
label: "Music",
- description: "Track name, artist, and whether the sound is original — the seed of a trend.",
+ description: "Track name, artist, and whether the sound is original, the seed of a trend.",
},
{
label: "Hashtags",
description: "Every hashtag on a video, so you can map a topic cluster or campaign.",
},
- {
- label: "Timestamps",
- description: "Created and scraped times so you can track a video's momentum over runs.",
- },
],
- useCasesHeading: "What teams do with the TikTok API",
+ useCasesHeading: "What teams do with the TikTok scraper API",
useCases: [
{
title: "Trend and hashtag monitoring",
description:
- "Track a hashtag and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.",
+ "Use the TikTok hashtag scraper to track a topic and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.",
},
{
title: "Creator and influencer discovery",
description:
- "Surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.",
+ "Scrape TikTok profiles and search users by keyword to surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.",
},
{
title: "Competitor content analysis",
@@ -102,16 +106,16 @@ export const tiktok: ConnectorPageContent = {
"Watch what your category posts and what actually lands. Turn a competitor's best-performing formats and hooks into your own content brief.",
},
{
- title: "Campaign and sentiment tracking",
+ title: "Campaign and comment sentiment",
description:
- "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — then pull the comments on top videos to read how the audience actually reacts, not just a vanity view count.",
+ "Measure how a launch or branded hashtag spreads across TikTok, then use the TikTok comments scraper on top videos to read how the audience actually reacts, not just a vanity view count.",
},
],
comparison: {
- heading: "A TikTok API alternative built for agents",
+ heading: "A TikTok Research API alternative built for agents",
intro:
- "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need it for commercial research, here is how SurfSense compares.",
+ "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need commercial TikTok scraping, SurfSense is a TikTok API alternative. Here is how it compares.",
columnLabel: "Official TikTok API",
rows: [
{
@@ -180,7 +184,7 @@ export const tiktok: ConnectorPageContent = {
type: "string[]",
defaultValue: "[]",
description:
- "Keyword search terms. Keyword video search is login-walled and returns no videos — use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.",
+ "Keyword search terms. Keyword video search is login-walled and returns no videos; use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.",
},
{
name: "results_per_page",
@@ -253,9 +257,19 @@ export const tiktok: ConnectorPageContent = {
"SurfSense reads only public TikTok data, the same videos any logged-out visitor can see. It never logs in and cannot access private or deleted content. As always, review TikTok's terms and your own compliance needs before you run at scale.",
},
{
- question: "Does this need the official TikTok API?",
+ question: "Does this need the official TikTok Research API?",
answer:
- "No. It is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured videos back.",
+ "No. This TikTok scraper API is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call SurfSense with one key, or add the MCP server to your agent, and get structured videos, comments, and profile data back.",
+ },
+ {
+ question: "What TikTok data can I scrape?",
+ answer:
+ "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword), and trending (the current Explore feed). Each returns structured items and is billed per item returned.",
+ },
+ {
+ question: "Can I scrape TikTok comments and hashtags?",
+ answer:
+ "Yes. Pass a video URL to the comments endpoint for the public comment thread. Pass hashtag names or /tag/ URLs to the TikTok hashtag scraper to pull videos under that tag. Keyword video search is login-walled, so hashtags and direct URLs are the reliable discovery paths.",
},
{
question: "What are the rate limits?",
@@ -265,18 +279,14 @@ export const tiktok: ConnectorPageContent = {
{
question: "Can I scrape a specific creator's videos?",
answer:
- "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag or by a direct video URL.",
- },
- {
- question: "What TikTok data can I scrape?",
- answer:
- "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.",
+ "Pass a profile or profile URL and you always get the account's metadata: name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account. For reliable video results, scrape by hashtag or by a direct video URL.",
},
],
related: [
- { label: "Reddit API", href: "/reddit" },
+ { label: "Instagram API", href: "/instagram" },
{ label: "YouTube API", href: "/youtube" },
+ { label: "Reddit API", href: "/reddit" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
From 1bc7d9f51c7015917a0a3af9e584a6eb03a6faaa Mon Sep 17 00:00:00 2001
From: "DESKTOP-RTLN3BA\\$punk"
Date: Mon, 13 Jul 2026 15:03:24 -0700
Subject: [PATCH 157/160] Update README files in multiple languages to include
Instagram and TikTok as new data sources for live scraping, enhancing the
platform's competitive intelligence capabilities.
---
.github/workflows/desktop-release.yml | 15 +++++++++++++++
README.es.md | 6 ++++--
README.hi.md | 6 ++++--
README.md | 6 ++++--
README.pt-BR.md | 6 ++++--
README.zh-CN.md | 6 ++++--
6 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml
index 3ad529671..7955adcbe 100644
--- a/.github/workflows/desktop-release.yml
+++ b/.github/workflows/desktop-release.yml
@@ -75,6 +75,21 @@ jobs:
echo "Windows signing: skipped"
fi
+ - name: Verify Apple notarization credentials
+ if: matrix.os == 'macos-latest'
+ shell: bash
+ # Fails in seconds with the same 401 notarytool would throw after the
+ # full build, instead of wasting ~15 min of build time on bad secrets.
+ run: |
+ xcrun notarytool history \
+ --apple-id "$APPLE_ID" \
+ --password "$APPLE_APP_SPECIFIC_PASSWORD" \
+ --team-id "$APPLE_TEAM_ID" > /dev/null
+ env:
+ APPLE_ID: ${{ secrets.APPLE_ID }}
+ APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
+ APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+
- name: Setup pnpm
uses: pnpm/action-setup@v5
diff --git a/README.es.md b/README.es.md
index ecdd93c41..08231b5ab 100644
--- a/README.es.md
+++ b/README.es.md
@@ -22,7 +22,7 @@
# SurfSense: NotebookLM para investigación de inteligencia competitiva
-SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas.
+SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas.
> [!NOTE]
> **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM**
@@ -103,6 +103,8 @@ Las automatizaciones ejecutan turnos completos de agente según un horario o en
|---|---|---|
| **Reddit** | Publicaciones, comentarios y flujos de subreddits sin los límites de tasa de la API oficial | [Reddit Scraper API](https://www.surfsense.com/reddit) |
| **YouTube** | Videos, transcripciones e hilos de comentarios para escuchar a tu marca y productos | [YouTube Scraper API](https://www.surfsense.com/youtube) |
+| **Instagram** | Perfiles, publicaciones y reels públicos sin la Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) |
+| **TikTok** | Videos, comentarios, hashtags y perfiles sin aprobación de la Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) |
| **Google Maps** | Lugares, calificaciones y reseñas para investigar competidores locales y prospectos | [Google Maps Scraper API](https://www.surfsense.com/google-maps) |
| **Google Search** | SERPs en vivo para seguimiento de posiciones y monitoreo de mercado | [Google Search API](https://www.surfsense.com/google-search) |
| **Web Crawl** (rastreo web) | Cualquier página de la web abierta como contenido limpio y estructurado | [Web Crawling API](https://www.surfsense.com/web-crawl) |
@@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
| Característica | Google NotebookLM | SurfSense |
|---------|-------------------|-----------|
-| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Google Maps, Google Search y rastreo web vía API REST y MCP |
+| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y rastreo web vía API REST y MCP |
| **Servidor MCP** | No | Cada conector expuesto como herramienta nativa de agente, más servidores MCP propios con aplicaciones OAuth de un clic |
| **Fuentes por Notebook** | 50 (gratis) a 600 (Ultra, $249.99/mes) | Ilimitadas |
| **Número de Notebooks** | 100 (gratis) a 500 (niveles de pago) | Ilimitado |
diff --git a/README.hi.md b/README.hi.md
index 3cb769318..eb65ff489 100644
--- a/README.hi.md
+++ b/README.hi.md
@@ -22,7 +22,7 @@
# SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM
-SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है।
+SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है।
> [!NOTE]
> **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना**
@@ -103,6 +103,8 @@ SurfSense **AI एजेंट्स के लिए ओपन सोर्स
|---|---|---|
| **Reddit** | आधिकारिक API की रेट लिमिट के बिना पोस्ट, कमेंट और सबरेडिट स्ट्रीम | [Reddit Scraper API](https://www.surfsense.com/reddit) |
| **YouTube** | ब्रांड और प्रोडक्ट लिसनिंग के लिए वीडियो, ट्रांसक्रिप्ट और कमेंट थ्रेड | [YouTube Scraper API](https://www.surfsense.com/youtube) |
+| **Instagram** | Graph API के बिना सार्वजनिक प्रोफ़ाइल, पोस्ट और रील्स | [Instagram Scraper API](https://www.surfsense.com/instagram) |
+| **TikTok** | Research API अप्रूवल के बिना वीडियो, कमेंट, हैशटैग और प्रोफ़ाइल | [TikTok Scraper API](https://www.surfsense.com/tiktok) |
| **Google Maps** | स्थानीय प्रतिस्पर्धी और लीड रिसर्च के लिए स्थान, रेटिंग और रिव्यू | [Google Maps Scraper API](https://www.surfsense.com/google-maps) |
| **Google Search** | रैंक ट्रैकिंग और मार्केट मॉनिटरिंग के लिए लाइव SERP | [Google Search API](https://www.surfsense.com/google-search) |
| **Web Crawl** | ओपन वेब का कोई भी पेज साफ़-सुथरे, स्ट्रक्चर्ड कंटेंट के रूप में | [Web Crawling API](https://www.surfsense.com/web-crawl) |
@@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
| फ़ीचर | Google NotebookLM | SurfSense |
|---------|-------------------|-----------|
-| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Google Maps, Google Search और वेब क्रॉल कनेक्टर |
+| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और वेब क्रॉल कनेक्टर |
| **MCP सर्वर** | नहीं | हर कनेक्टर नेटिव एजेंट टूल के रूप में उपलब्ध, साथ ही वन-क्लिक OAuth ऐप्स के साथ अपने MCP सर्वर लाने की सुविधा |
| **प्रति नोटबुक स्रोत** | 50 (Free) से 600 (Ultra, $249.99/माह) | असीमित |
| **नोटबुक की संख्या** | 100 (Free) से 500 (सशुल्क टियर) | असीमित |
diff --git a/README.md b/README.md
index eac392ebd..ba289ada4 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@
# SurfSense: NotebookLM for Competitive Intelligence Research
-SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations.
+SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations.
> [!NOTE]
> **📢 A note for our NotebookLM-alternative users**
@@ -103,6 +103,8 @@ Automations run full agent turns on a schedule or in response to events, then wr
|---|---|---|
| **Reddit** | Posts, comments, and subreddit streams without the official API's rate limits | [Reddit Scraper API](https://www.surfsense.com/reddit) |
| **YouTube** | Videos, transcripts, and comment threads for brand and product listening | [YouTube Scraper API](https://www.surfsense.com/youtube) |
+| **Instagram** | Public profiles, posts, and reels without the Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) |
+| **TikTok** | Videos, comments, hashtags, and profiles without Research API approval | [TikTok Scraper API](https://www.surfsense.com/tiktok) |
| **Google Maps** | Places, ratings, and reviews for local competitor and lead research | [Google Maps Scraper API](https://www.surfsense.com/google-maps) |
| **Google Search** | Live SERPs for rank tracking and market monitoring | [Google Search API](https://www.surfsense.com/google-search) |
| **Web Crawl** | Any page on the open web as clean, structured content | [Web Crawling API](https://www.surfsense.com/web-crawl) |
@@ -243,7 +245,7 @@ Still comparing us as a NotebookLM alternative? Here is the honest breakdown.
| Feature | Google NotebookLM | SurfSense |
|---------|-------------------|-----------|
-| **Live market data for agents** | No | Reddit, YouTube, Google Maps, Google Search, and web crawl connectors via REST API and MCP |
+| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and web crawl connectors via REST API and MCP |
| **MCP server** | No | Every connector exposed as a native agent tool, plus bring-your-own MCP servers with one-click OAuth apps |
| **Sources per Notebook** | 50 (Free) to 600 (Ultra, $249.99/mo) | Unlimited |
| **Number of Notebooks** | 100 (Free) to 500 (paid tiers) | Unlimited |
diff --git a/README.pt-BR.md b/README.pt-BR.md
index e32882831..327e3373c 100644
--- a/README.pt-BR.md
+++ b/README.pt-BR.md
@@ -22,7 +22,7 @@
# SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva
-O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações.
+O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações.
> [!NOTE]
> **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM**
@@ -103,6 +103,8 @@ As automações executam turnos completos de agente de forma agendada ou em resp
|---|---|---|
| **Reddit** | Posts, comentários e fluxos de subreddits sem os limites de requisição da API oficial | [Reddit Scraper API](https://www.surfsense.com/reddit) |
| **YouTube** | Vídeos, transcrições e threads de comentários para monitoramento de marca e produto | [YouTube Scraper API](https://www.surfsense.com/youtube) |
+| **Instagram** | Perfis, posts e reels públicos sem a Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) |
+| **TikTok** | Vídeos, comentários, hashtags e perfis sem aprovação da Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) |
| **Google Maps** | Estabelecimentos, avaliações e reviews para pesquisa local de concorrentes e leads | [Google Maps Scraper API](https://www.surfsense.com/google-maps) |
| **Google Search** | SERPs ao vivo para acompanhamento de rankings e monitoramento de mercado | [Google Search API](https://www.surfsense.com/google-search) |
| **Web Crawl** (rastreamento web) | Qualquer página da web aberta como conteúdo limpo e estruturado | [Web Crawling API](https://www.surfsense.com/web-crawl) |
@@ -244,7 +246,7 @@ Ainda nos comparando como alternativa ao NotebookLM? Aqui está o comparativo ho
| Recurso | Google NotebookLM | SurfSense |
|---------|-------------------|-----------|
-| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Google Maps, Google Search e rastreamento web via API REST e MCP |
+| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e rastreamento web via API REST e MCP |
| **Servidor MCP** | Não | Cada conector exposto como ferramenta nativa de agente, além de servidores MCP próprios com apps OAuth em um clique |
| **Fontes por Notebook** | 50 (gratuito) a 600 (Ultra, US$ 249,99/mês) | Ilimitadas |
| **Número de Notebooks** | 100 (gratuito) a 500 (planos pagos) | Ilimitado |
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 11bace243..53a95968b 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -22,7 +22,7 @@
# SurfSense:面向竞争情报研究的 NotebookLM
-SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 NotebookLM,但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。
+SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 NotebookLM,但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。
> [!NOTE]
> **📢 致我们的 NotebookLM 替代品用户**
@@ -103,6 +103,8 @@ SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 Noteboo
|---|---|---|
| **Reddit** | 帖子、评论和子版块信息流,不受官方 API 速率限制 | [Reddit Scraper API](https://www.surfsense.com/reddit) |
| **YouTube** | 视频、字幕转录和评论串,用于品牌和产品舆情监听 | [YouTube Scraper API](https://www.surfsense.com/youtube) |
+| **Instagram** | 公开主页、帖子和 Reels,无需 Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) |
+| **TikTok** | 视频、评论、话题标签和主页,无需 Research API 审批 | [TikTok Scraper API](https://www.surfsense.com/tiktok) |
| **Google Maps** | 地点、评分和评论,用于本地竞争对手和潜在客户调研 | [Google Maps Scraper API](https://www.surfsense.com/google-maps) |
| **Google Search** | 实时搜索结果页,用于排名追踪和市场监控 | [Google Search API](https://www.surfsense.com/google-search) |
| **Web Crawl** | 把开放网络上的任意页面转为干净、结构化的内容 | [Web Crawling API](https://www.surfsense.com/web-crawl) |
@@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
| 功能 | Google NotebookLM | SurfSense |
|---------|-------------------|-----------|
-| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Google Maps、Google Search 和网页爬取连接器 |
+| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和网页爬取连接器 |
| **MCP 服务器** | 无 | 每个连接器都作为原生智能体工具暴露,还可自带 MCP 服务器并使用一键 OAuth 应用 |
| **每个笔记本的来源数** | 50 个(免费版)至 600 个(Ultra 版,249.99 美元/月) | 无限制 |
| **笔记本数量** | 100 个(免费版)至 500 个(付费档位) | 无限制 |
From 1131da5ed71bbffe3f729311da38fe1c3c1eaa0f Mon Sep 17 00:00:00 2001
From: "DESKTOP-RTLN3BA\\$punk"
Date: Mon, 13 Jul 2026 16:29:39 -0700
Subject: [PATCH 158/160] feat: bumped version to 0.0.32
---
VERSION | 2 +-
.../builtins/instagram/system_prompt.md | 2 +-
.../builtins/tiktok/system_prompt.md | 4 +-
.../capabilities/tiktok/scrape/definition.py | 3 +-
.../app/capabilities/tiktok/scrape/schemas.py | 7 +-
.../platforms/instagram/scraper.py | 4 +-
.../platforms/tiktok/orchestrator.py | 86 +++++++++-
surfsense_backend/pyproject.toml | 2 +-
.../unit/platforms/tiktok/test_discovery.py | 149 ++++++++++++++++++
surfsense_backend/uv.lock | 134 +++++++++++++++-
surfsense_browser_extension/package.json | 2 +-
surfsense_desktop/package.json | 2 +-
surfsense_evals/scripts/analyze_failures.py | 2 -
.../scripts/check_uploaded_status.py | 1 -
.../scripts/compute_post_retry_accuracy.py | 4 +-
.../patch_manifest_for_parallel_ingest.py | 1 -
surfsense_evals/scripts/peek_crag_run.py | 7 +-
surfsense_evals/scripts/peek_disagreements.py | 3 +-
.../scripts/retry_failed_questions.py | 6 +-
surfsense_evals/scripts/summarise_crag_run.py | 3 +-
.../scripts/summarise_parser_compare_run.py | 3 +-
.../src/surfsense_evals/core/cli.py | 7 +-
.../surfsense_evals/core/clients/documents.py | 5 +-
.../core/metrics/comparison.py | 4 +-
.../surfsense_evals/core/parse/__init__.py | 2 +-
.../surfsense_evals/core/parse/citations.py | 4 +-
.../surfsense_evals/core/parsers/azure_di.py | 15 +-
.../core/parsers/llamacloud.py | 2 +-
.../core/providers/openrouter_chat.py | 2 +-
.../core/providers/openrouter_pdf.py | 4 +-
.../src/surfsense_evals/suites/__init__.py | 2 +-
.../suites/_demo/hello/__init__.py | 1 -
.../suites/medical/cure/__init__.py | 2 +-
.../suites/medical/cure/ingest.py | 6 +-
.../suites/medical/cure/runner.py | 5 +-
.../suites/medical/mirage/__init__.py | 2 +-
.../suites/medical/mirage/ingest.py | 96 +++++------
.../suites/medical/mirage/prompt.py | 1 -
.../suites/medical/mirage/runner.py | 5 +-
.../multimodal_doc/parser_compare/ingest.py | 4 +-
.../multimodal_doc/parser_compare/runner.py | 4 +-
.../suites/research/crag/html_extract.py | 5 +-
.../suites/research/crag/prompt.py | 1 -
.../suites/research/crag/runner.py | 4 +-
.../suites/research/frames/dataset.py | 2 +-
.../suites/research/frames/ingest.py | 1 -
.../suites/research/frames/prompt.py | 1 -
.../tests/suites/test_crag_dataset.py | 2 -
.../tests/suites/test_crag_grader.py | 2 -
.../tests/suites/test_crag_html_extract.py | 3 -
.../tests/suites/test_frames_dataset.py | 3 -
.../tests/suites/test_frames_grader.py | 3 -
.../features/scrapers/platforms/instagram.py | 8 +-
.../features/scrapers/platforms/tiktok.py | 18 ++-
surfsense_web/package.json | 2 +-
55 files changed, 496 insertions(+), 159 deletions(-)
create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py
diff --git a/VERSION b/VERSION
index d788d4335..78bae5bb6 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.0.31
+0.0.32
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md
index 42713cc38..ce9182771 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md
@@ -13,7 +13,7 @@ Answer the delegated question from live Instagram data gathered with your verbs,
- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled).
-- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only).
+- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). Google-backed discovery is slow (~30-60s per query), so start with **at most 3** distinct queries per task and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
- Profile metadata (follower counts, bio, post count): call `instagram_details`.
- Batch multiple URLs (or queries) into one call rather than many single-item calls.
diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
index 03114cd12..4300e9313 100644
--- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
+++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md
@@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com
-- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`.
+- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent.
- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`.
- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos.
- Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`.
-- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL.
+- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts.
- "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many.
- Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`).
- Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it.
diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
index 9f8632f3b..e2706a2ef 100644
--- a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py
@@ -11,7 +11,8 @@ TIKTOK_SCRAPE = Capability(
name="tiktok.scrape",
description=(
"Scrape public TikTok videos. Use urls, profiles, hashtags, or "
- "search_queries."
+ "search_queries (search_queries are resolved via Google to public "
+ "videos; for accounts by keyword use tiktok.user_search)."
),
input_schema=ScrapeInput,
output_schema=ScrapeOutput,
diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
index 2f5005353..853b70bd9 100644
--- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
+++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py
@@ -43,7 +43,12 @@ class ScrapeInput(BaseModel):
search_queries: list[str] = Field(
default_factory=list,
max_length=MAX_TIKTOK_SOURCES,
- description="Search terms to run on TikTok.",
+ description=(
+ "Search terms resolved via Google (site:tiktok.com) to public TikTok "
+ "videos, since TikTok's own keyword search is login-walled. Slower "
+ "than hashtags/urls. To find accounts by keyword, use "
+ "tiktok.user_search instead."
+ ),
)
results_per_page: int = Field(
default=10,
diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
index a4c22bde7..9f4f6e2eb 100644
--- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
+++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
@@ -306,9 +306,9 @@ async def _discover_via_google(
"""
serps = await scrape_serps(
GoogleSearchScrapeInput(
- queries=query, site="instagram.com", maxPagesPerQuery=2
+ queries=query, site="instagram.com", maxPagesPerQuery=1
),
- limit=2,
+ limit=1,
)
resolved: list[ResolvedUrl] = []
seen: set[tuple[str, str]] = set()
diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
index 5d6a24545..01fae6caf 100644
--- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
+++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py
@@ -12,6 +12,9 @@ from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import quote
+from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput
+from app.proprietary.platforms.google_search.scraper import scrape_serps
+
from .extraction.timestamps import now_iso
from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn
from .flows.comments import iter_comments
@@ -35,9 +38,24 @@ _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}"
_SEARCH_URL = "https://www.tiktok.com/search?q={query}"
_EXPLORE_URL = "https://www.tiktok.com/explore"
+# A ``searchQueries`` term whose Google discovery surfaced no scrapable video
+# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract:
+# never vanish silently).
+_EMPTY_DISCOVERY_MESSAGE = (
+ "No public TikTok videos found for this query via Google discovery. Try a "
+ "narrower phrasing, a hashtag, or a direct video URL."
+)
+
def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
- """Build the target list from every input source, dropping unresolved URLs."""
+ """Build the target list from the URL/profile/hashtag sources.
+
+ ``searchQueries`` is deliberately excluded: TikTok's own keyword search is
+ login-walled for anonymous sessions, so it is routed through Google video
+ discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...``
+ URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and
+ keeps its native listing routing.
+ """
targets: list[TikTokTarget] = []
for entry in input_model.startUrls:
resolved = resolve_target(entry.url)
@@ -52,13 +70,42 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]:
targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name)))
for tag in input_model.hashtags:
targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag)))
- for query in input_model.searchQueries:
- targets.append(
- TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query)))
- )
return targets
+async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]:
+ """Discover public TikTok video targets via Google ``site:tiktok.com``.
+
+ TikTok's anonymous keyword search is login-walled, so we reuse the existing
+ ``google_search`` platform, classify each organic URL with ``resolve_target``,
+ and keep only video hits (``/@user/video/``) — the one kind that scrapes
+ reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are
+ dropped (accounts belong to the ``user_search`` verb). De-duped, capped at
+ ``limit``.
+ """
+ serps = await scrape_serps(
+ GoogleSearchScrapeInput(
+ queries=query, site="tiktok.com", maxPagesPerQuery=1
+ ),
+ limit=1,
+ )
+ resolved: list[TikTokTarget] = []
+ seen: set[str] = set()
+ for serp in serps:
+ for org in serp.get("organicResults") or []:
+ url = org.get("url", "") if isinstance(org, dict) else ""
+ target = resolve_target(url)
+ if target is None or target.kind != "video":
+ continue
+ if target.value in seen:
+ continue
+ seen.add(target.value)
+ resolved.append(target)
+ if len(resolved) >= limit:
+ return resolved
+ return resolved
+
+
def _dispatch(
target: TikTokTarget,
*,
@@ -81,9 +128,11 @@ async def iter_tiktok(
) -> AsyncIterator[dict[str, Any]]:
"""Yield normalized items for every resolved target, in order.
- The video flow's ``fetch_html`` opens its own warmed proxy session per call
- when none is bound; the listing flow drives its own browser. Neither binds a
- ContextVar across these ``yield``s, so the generator stays context-safe.
+ Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries``
+ then run through Google video discovery. The video flow's ``fetch_html``
+ opens its own warmed proxy session per call when none is bound; the listing
+ flow drives its own browser. Neither binds a ContextVar across these
+ ``yield``s, so the generator stays context-safe.
"""
cap = input_model.resultsPerPage
for target in _resolve_targets(input_model):
@@ -92,6 +141,27 @@ async def iter_tiktok(
):
yield item
+ # searchQueries -> Google-discovered public video URLs, de-duped across
+ # queries so the same video surfacing under two terms is scraped once.
+ seen_videos: set[str] = set()
+ for query in input_model.searchQueries:
+ discovered = await _discover_via_google(query, limit=cap)
+ if not discovered:
+ yield ErrorItem(
+ url=_SEARCH_URL.format(query=quote(query)),
+ input=query,
+ error=_EMPTY_DISCOVERY_MESSAGE,
+ errorCode="no_items",
+ scrapedAt=now_iso(),
+ ).to_output()
+ continue
+ for target in discovered:
+ if target.value in seen_videos:
+ continue
+ seen_videos.add(target.value)
+ async for item in iter_video(target, fetch=fetch):
+ yield item
+
async def scrape_tiktok(
input_model: TikTokScrapeInput,
diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml
index 2faa2be17..30bb7ea5c 100644
--- a/surfsense_backend/pyproject.toml
+++ b/surfsense_backend/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "surf-new-backend"
-version = "0.0.31"
+version = "0.0.32"
description = "SurfSense Backend"
requires-python = ">=3.12"
dependencies = [
diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py
new file mode 100644
index 000000000..4664f85e0
--- /dev/null
+++ b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py
@@ -0,0 +1,149 @@
+"""Offline tests for Google-backed TikTok video discovery.
+
+``searchQueries`` are login-walled on TikTok's native search, so they route
+through the ``google_search`` platform (``site:tiktok.com``): each organic URL
+is classified with ``resolve_target`` and only video hits (``/@user/video/``)
+are kept — profiles/hashtags/search/photo/non-tiktok are dropped (accounts
+belong to the user-search verb). These tests inject a fake ``scrape_serps`` so
+there is no network: they pin the classification, cross-query de-dup, the limit
+cap, the barren-query ErrorItem, and that no ``/search?q=`` listing target is
+ever built.
+"""
+
+from __future__ import annotations
+
+import json
+
+from app.proprietary.platforms.tiktok import (
+ TikTokScrapeInput,
+ orchestrator,
+ scrape_tiktok,
+)
+
+
+def _fake_serps(*organic_urls: str):
+ async def _scrape_serps(input_model, *, limit=None):
+ assert input_model.site == "tiktok.com"
+ assert input_model.maxPagesPerQuery == 1
+ return [{"organicResults": [{"url": u} for u in organic_urls]}]
+
+ return _scrape_serps
+
+
+def _video_page(url: str) -> str:
+ """Render a rehydration blob for a ``/@user/video/`` URL."""
+ video_id = url.rsplit("/", 1)[1]
+ username = url.split("@")[1].split("/")[0]
+ blob = {
+ "__DEFAULT_SCOPE__": {
+ "webapp.video-detail": {
+ "itemInfo": {
+ "itemStruct": {
+ "id": video_id,
+ "desc": "hi",
+ "author": {"uniqueId": username},
+ "stats": {"diggCount": 1},
+ }
+ }
+ }
+ }
+ }
+ return (
+ ''
+ )
+
+
+async def _fetch_video(url: str) -> str:
+ return _video_page(url)
+
+
+async def test_search_discovery_keeps_only_videos(monkeypatch):
+ # Only the video URL survives; profile / hashtag / search / photo /
+ # non-tiktok organic results are dropped.
+ monkeypatch.setattr(
+ orchestrator,
+ "scrape_serps",
+ _fake_serps(
+ "https://www.tiktok.com/@nasa/video/123",
+ "https://www.tiktok.com/@nasa",
+ "https://www.tiktok.com/tag/space",
+ "https://www.tiktok.com/search?q=space",
+ "https://www.tiktok.com/@nasa/photo/999",
+ "https://example.com/not-tiktok",
+ ),
+ )
+ items = await scrape_tiktok(
+ TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
+ fetch=_fetch_video,
+ )
+ assert [i["id"] for i in items] == ["123"]
+
+
+async def test_search_discovery_dedupes_across_queries(monkeypatch):
+ # The same video surfacing under two queries is scraped once.
+ monkeypatch.setattr(
+ orchestrator,
+ "scrape_serps",
+ _fake_serps("https://www.tiktok.com/@nasa/video/123"),
+ )
+ items = await scrape_tiktok(
+ TikTokScrapeInput(searchQueries=["space", "rockets"], resultsPerPage=10),
+ fetch=_fetch_video,
+ )
+ assert [i["id"] for i in items] == ["123"]
+
+
+async def test_search_discovery_respects_per_target_limit(monkeypatch):
+ monkeypatch.setattr(
+ orchestrator,
+ "scrape_serps",
+ _fake_serps(
+ "https://www.tiktok.com/@a/video/1",
+ "https://www.tiktok.com/@b/video/2",
+ "https://www.tiktok.com/@c/video/3",
+ ),
+ )
+ items = await scrape_tiktok(
+ TikTokScrapeInput(searchQueries=["x"], resultsPerPage=2),
+ fetch=_fetch_video,
+ )
+ assert [i["id"] for i in items] == ["1", "2"]
+
+
+async def test_search_barren_query_emits_error_item(monkeypatch):
+ # A query whose discovery finds no video URLs degrades to one ErrorItem.
+ monkeypatch.setattr(
+ orchestrator,
+ "scrape_serps",
+ _fake_serps(
+ "https://www.tiktok.com/@nasa",
+ "https://example.com/x",
+ ),
+ )
+ items = await scrape_tiktok(
+ TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
+ fetch=_fetch_video,
+ )
+ assert len(items) == 1
+ assert items[0]["errorCode"] == "no_items"
+ assert items[0]["input"] == "space"
+
+
+async def test_search_never_builds_listing_target(monkeypatch):
+ # searchQueries must never hit the (login-walled) native search listing flow.
+ monkeypatch.setattr(
+ orchestrator,
+ "scrape_serps",
+ _fake_serps("https://www.tiktok.com/@nasa/video/123"),
+ )
+
+ async def _boom_listing(_url: str, _count: int) -> list[dict]:
+ raise AssertionError("searchQueries must not build a listing target")
+
+ items = await scrape_tiktok(
+ TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10),
+ fetch=_fetch_video,
+ fetch_listing=_boom_listing,
+ )
+ assert [i["id"] for i in items] == ["123"]
diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock
index a5b492fc8..2b1272363 100644
--- a/surfsense_backend/uv.lock
+++ b/surfsense_backend/uv.lock
@@ -36,6 +36,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -70,6 +73,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -104,6 +110,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -138,6 +147,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -172,6 +184,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -206,6 +221,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -240,6 +258,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -274,6 +295,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -308,6 +332,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -342,6 +369,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -387,6 +417,10 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -421,6 +455,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -455,6 +492,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -489,6 +529,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -523,6 +566,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -557,6 +603,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -591,6 +640,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -625,6 +677,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -659,6 +714,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -693,6 +751,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -727,6 +788,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -772,6 +836,10 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -806,6 +874,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -840,6 +911,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -874,6 +948,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -908,6 +985,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -942,6 +1022,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -976,6 +1059,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1010,6 +1096,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1044,6 +1133,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1078,6 +1170,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1112,6 +1207,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1157,6 +1255,10 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1191,6 +1293,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1258,6 +1363,12 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1292,6 +1403,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1326,6 +1440,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1393,6 +1510,12 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1460,6 +1583,12 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@@ -1494,6 +1623,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
+ "python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
]
conflicts = [[
@@ -10353,7 +10485,7 @@ wheels = [
[[package]]
name = "surf-new-backend"
-version = "0.0.31"
+version = "0.0.32"
source = { editable = "." }
dependencies = [
{ name = "alembic" },
diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json
index 8dda0d82f..a2c25a602 100644
--- a/surfsense_browser_extension/package.json
+++ b/surfsense_browser_extension/package.json
@@ -1,7 +1,7 @@
{
"name": "surfsense_browser_extension",
"displayName": "Surfsense Browser Extension",
- "version": "0.0.31",
+ "version": "0.0.32",
"description": "Extension to collect Browsing History for SurfSense.",
"author": "https://github.com/MODSetter",
"engines": {
diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json
index f12b9722c..629fe801d 100644
--- a/surfsense_desktop/package.json
+++ b/surfsense_desktop/package.json
@@ -1,7 +1,7 @@
{
"name": "surfsense-desktop",
"productName": "SurfSense",
- "version": "0.0.31",
+ "version": "0.0.32",
"description": "SurfSense Desktop App",
"main": "dist/main.js",
"scripts": {
diff --git a/surfsense_evals/scripts/analyze_failures.py b/surfsense_evals/scripts/analyze_failures.py
index e7ace1e1b..ff5ec23f4 100644
--- a/surfsense_evals/scripts/analyze_failures.py
+++ b/surfsense_evals/scripts/analyze_failures.py
@@ -12,12 +12,10 @@ Outputs (printed to stdout + written to `failures_n171.json`):
from __future__ import annotations
import json
-import re
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
-
REPO = Path(__file__).resolve().parents[1]
RUN = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
RAW = RUN / "raw.jsonl"
diff --git a/surfsense_evals/scripts/check_uploaded_status.py b/surfsense_evals/scripts/check_uploaded_status.py
index 7021ba83d..1903502ed 100644
--- a/surfsense_evals/scripts/check_uploaded_status.py
+++ b/surfsense_evals/scripts/check_uploaded_status.py
@@ -15,7 +15,6 @@ from pathlib import Path
import httpx
from dotenv import load_dotenv
-
REPO = Path(__file__).resolve().parents[1]
PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"
diff --git a/surfsense_evals/scripts/compute_post_retry_accuracy.py b/surfsense_evals/scripts/compute_post_retry_accuracy.py
index 4c8c47672..f3a716219 100644
--- a/surfsense_evals/scripts/compute_post_retry_accuracy.py
+++ b/surfsense_evals/scripts/compute_post_retry_accuracy.py
@@ -47,9 +47,7 @@ def _row_key(row: dict) -> tuple[str, str]:
def _is_failure(row: dict) -> bool:
if row.get("error"):
return True
- if not (row.get("raw_text") or "").strip():
- return True
- return False
+ return bool(not (row.get("raw_text") or "").strip())
def _summarise(rows_by_arm: dict[str, list[dict]]) -> dict[str, dict]:
diff --git a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py
index e1a2edc65..8cfb13eb7 100644
--- a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py
+++ b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py
@@ -27,7 +27,6 @@ from __future__ import annotations
import json
from pathlib import Path
-
REPO = Path(__file__).resolve().parents[1]
MAP_PATH = REPO / "data" / "multimodal_doc" / "maps" / "mmlongbench_doc_map.jsonl"
PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"
diff --git a/surfsense_evals/scripts/peek_crag_run.py b/surfsense_evals/scripts/peek_crag_run.py
index 225e5ec98..3a79d76c7 100644
--- a/surfsense_evals/scripts/peek_crag_run.py
+++ b/surfsense_evals/scripts/peek_crag_run.py
@@ -10,19 +10,20 @@ from collections import defaultdict
def main() -> None:
raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
print(f"Reading: {raw_path}")
- rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+ with open(raw_path, encoding="utf-8") as fh:
+ rows = [json.loads(line) for line in fh if line.strip()]
by_q: dict[str, dict[str, dict]] = defaultdict(dict)
for r in rows:
by_q[r["qid"]][r["arm"]] = r
for qid, arms in list(by_q.items()):
b = arms.get("bare_llm", {})
- l = arms.get("long_context", {})
+ lc = arms.get("long_context", {})
s = arms.get("surfsense", {})
print(f"\n=== {qid} ({b.get('domain')}/{b.get('question_type')}) ===")
print(f" question: {b.get('extra', {}).get('question', '?')!r}")
print(f" gold: {b.get('gold')!r}")
- for arm_name, a in (("bare_llm", b), ("long_context", l), ("surfsense", s)):
+ for arm_name, a in (("bare_llm", b), ("long_context", lc), ("surfsense", s)):
grade = a.get("graded", {})
text = (a.get("raw_text") or "").strip()
tail = text[-200:] if text else ""
diff --git a/surfsense_evals/scripts/peek_disagreements.py b/surfsense_evals/scripts/peek_disagreements.py
index c0fe0acd9..b6497570e 100644
--- a/surfsense_evals/scripts/peek_disagreements.py
+++ b/surfsense_evals/scripts/peek_disagreements.py
@@ -10,7 +10,8 @@ from collections import defaultdict
def main() -> None:
raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
print(f"Reading: {raw_path}")
- rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+ with open(raw_path, encoding="utf-8") as fh:
+ rows = [json.loads(line) for line in fh if line.strip()]
by_q: dict[str, dict[str, dict]] = defaultdict(dict)
for r in rows:
by_q[r["qid"]][r["arm"]] = r
diff --git a/surfsense_evals/scripts/retry_failed_questions.py b/surfsense_evals/scripts/retry_failed_questions.py
index 7cc9478e0..25facfe60 100644
--- a/surfsense_evals/scripts/retry_failed_questions.py
+++ b/surfsense_evals/scripts/retry_failed_questions.py
@@ -106,9 +106,7 @@ def _is_failure_row(row: dict[str, Any]) -> bool:
if row.get("error"):
return True
- if not (row.get("raw_text") or "").strip():
- return True
- return False
+ return bool(not (row.get("raw_text") or "").strip())
@dataclass
@@ -428,7 +426,7 @@ async def _run(args: argparse.Namespace) -> int:
if f.arm == "native_pdf":
pdf_path = Path(map_row["pdf_path"])
- if not pdf_path.exists():
+ if not await asyncio.to_thread(pdf_path.exists):
logger.error("PDF missing on disk: %s — skipping", pdf_path)
continue
request = _build_native_request(
diff --git a/surfsense_evals/scripts/summarise_crag_run.py b/surfsense_evals/scripts/summarise_crag_run.py
index 646fb6a97..9722cd391 100644
--- a/surfsense_evals/scripts/summarise_crag_run.py
+++ b/surfsense_evals/scripts/summarise_crag_run.py
@@ -11,7 +11,8 @@ def main() -> None:
if not runs:
print("(no CRAG runs found)")
return
- m = json.load(open(runs[-1], encoding="utf-8"))
+ with open(runs[-1], encoding="utf-8") as fh:
+ m = json.load(fh)
metrics = m["metrics"]
print(f"Reading: {runs[-1]}")
diff --git a/surfsense_evals/scripts/summarise_parser_compare_run.py b/surfsense_evals/scripts/summarise_parser_compare_run.py
index c54d82784..c091043aa 100644
--- a/surfsense_evals/scripts/summarise_parser_compare_run.py
+++ b/surfsense_evals/scripts/summarise_parser_compare_run.py
@@ -16,7 +16,6 @@ import statistics
from collections import defaultdict
from pathlib import Path
-
REPO = Path(__file__).resolve().parents[1]
RUN_DIR = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
RAW = RUN_DIR / "raw.jsonl"
@@ -91,7 +90,7 @@ def main() -> None:
print()
print("by answer_format (accuracy):")
- formats = sorted({f for m in arm_metrics.values() for f in m["by_format"].keys()})
+ formats = sorted({f for m in arm_metrics.values() for f in m["by_format"]})
header = f"{'arm':<25} " + " ".join(f"{f:>10}" for f in formats)
print(header)
print("-" * len(header))
diff --git a/surfsense_evals/src/surfsense_evals/core/cli.py b/surfsense_evals/src/surfsense_evals/core/cli.py
index 17979fba0..2dcbe7327 100644
--- a/surfsense_evals/src/surfsense_evals/core/cli.py
+++ b/surfsense_evals/src/surfsense_evals/core/cli.py
@@ -32,14 +32,13 @@ from __future__ import annotations
import argparse
import asyncio
+import contextlib
import json
import logging
import sys
from dataclasses import dataclass
from typing import Any
-import sys
-
import httpx
from rich.console import Console
from rich.table import Table
@@ -51,10 +50,8 @@ from rich.table import Table
# Terminal, PowerShell, cmd) all interpret ANSI escapes natively.
if sys.platform == "win32":
for _stream in (sys.stdout, sys.stderr):
- try:
+ with contextlib.suppress(AttributeError, ValueError):
_stream.reconfigure(encoding="utf-8", errors="replace")
- except (AttributeError, ValueError):
- pass
from . import registry
from .auth import CredentialError, acquire_token, client_with_auth
diff --git a/surfsense_evals/src/surfsense_evals/core/clients/documents.py b/surfsense_evals/src/surfsense_evals/core/clients/documents.py
index 362aae53b..b96bef6d8 100644
--- a/surfsense_evals/src/surfsense_evals/core/clients/documents.py
+++ b/surfsense_evals/src/surfsense_evals/core/clients/documents.py
@@ -18,6 +18,7 @@ Document processing is asynchronous:
from __future__ import annotations
import asyncio
+import contextlib
import logging
import mimetypes
from collections.abc import Iterable, Sequence
@@ -157,10 +158,8 @@ class DocumentsClient:
)
finally:
for _, (_, file_obj, _) in opened:
- try:
+ with contextlib.suppress(Exception):
file_obj.close()
- except Exception: # noqa: BLE001
- pass
response.raise_for_status()
return FileUploadResult.from_payload(response.json())
diff --git a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py
index 579576f4f..9e40dc5ca 100644
--- a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py
+++ b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py
@@ -71,8 +71,8 @@ def mcnemar_test(
f"Length mismatch: arm_a={len(arm_a_correct)}, arm_b={len(arm_b_correct)}"
)
n = len(arm_a_correct)
- b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct) if a and not c)
- c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct) if (not a) and cc)
+ b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct, strict=False) if a and not c)
+ c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct, strict=False) if (not a) and cc)
discordant = b + c
if discordant == 0:
return McnemarResult(
diff --git a/surfsense_evals/src/surfsense_evals/core/parse/__init__.py b/surfsense_evals/src/surfsense_evals/core/parse/__init__.py
index 208c2d374..8b90de78b 100644
--- a/surfsense_evals/src/surfsense_evals/core/parse/__init__.py
+++ b/surfsense_evals/src/surfsense_evals/core/parse/__init__.py
@@ -3,7 +3,7 @@
from __future__ import annotations
from .answer_letter import AnswerLetterResult, extract_answer_letter
-from .citations import CITATION_REGEX, CitationToken, ChunkCitation, UrlCitation, parse_citations
+from .citations import CITATION_REGEX, ChunkCitation, CitationToken, UrlCitation, parse_citations
from .freeform_answer import extract_freeform_answer
from .sse import SseEvent, iter_sse_events
diff --git a/surfsense_evals/src/surfsense_evals/core/parse/citations.py b/surfsense_evals/src/surfsense_evals/core/parse/citations.py
index 1fcd35434..c57ffeb0b 100644
--- a/surfsense_evals/src/surfsense_evals/core/parse/citations.py
+++ b/surfsense_evals/src/surfsense_evals/core/parse/citations.py
@@ -15,7 +15,7 @@ from __future__ import annotations
import re
from dataclasses import dataclass
-from typing import Any, Union
+from typing import Any
# Pattern preserves the TS source verbatim:
# /[\[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g
@@ -64,7 +64,7 @@ class UrlCitation:
return {"kind": "url", "url": self.url}
-CitationToken = Union[ChunkCitation, UrlCitation]
+CitationToken = ChunkCitation | UrlCitation
def parse_citations(text: str, *, url_map: dict[str, str] | None = None) -> list[CitationToken]:
diff --git a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py
index eebad906a..7d796ee99 100644
--- a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py
+++ b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py
@@ -25,6 +25,7 @@ import asyncio
import logging
import os
import random
+from pathlib import Path
logger = logging.getLogger(__name__)
@@ -82,7 +83,7 @@ async def parse_with_azure_di(
ServiceResponseError,
)
- file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+ file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024)
logger.info(
"Azure DI parsing %s (mode=%s, model=%s, size=%.1fMB)",
file_path, processing_mode, model_id, file_size_mb,
@@ -96,12 +97,12 @@ async def parse_with_azure_di(
credential=AzureKeyCredential(api_key),
)
async with client:
- with open(file_path, "rb") as fh:
- poller = await client.begin_analyze_document(
- model_id,
- body=fh,
- output_content_format=DocumentContentFormat.MARKDOWN,
- )
+ body = await asyncio.to_thread(Path(file_path).read_bytes)
+ poller = await client.begin_analyze_document(
+ model_id,
+ body=body,
+ output_content_format=DocumentContentFormat.MARKDOWN,
+ )
result = await poller.result()
content = (result.content or "").strip()
if not content:
diff --git a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py
index ba3d787ef..300b4ec87 100644
--- a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py
+++ b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py
@@ -98,7 +98,7 @@ async def parse_with_llamacloud(
from llama_cloud_services.parse.base import JobFailedException
from llama_cloud_services.parse.utils import ResultType
- file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+ file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024)
# Match backend's per-page timeout heuristic so big PDFs don't drop
# mid-job: 60s baseline + 30s/page (premium agent runs longer than
# basic; both fit comfortably here).
diff --git a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py
index 2494434be..208fbb865 100644
--- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py
+++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py
@@ -29,8 +29,8 @@ from typing import Any
import httpx
from .openrouter_pdf import (
- OpenRouterResponse,
_DEFAULT_HEADERS,
+ OpenRouterResponse,
_parse_chat_completion,
)
diff --git a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py
index e98590cbf..985d88a68 100644
--- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py
+++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py
@@ -34,7 +34,7 @@ import base64
import logging
import time
from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum
from pathlib import Path
from typing import Any
@@ -43,7 +43,7 @@ import httpx
logger = logging.getLogger(__name__)
-class PdfEngine(str, Enum):
+class PdfEngine(StrEnum):
NATIVE = "native"
MISTRAL_OCR = "mistral-ocr"
CLOUDFLARE_AI = "cloudflare-ai"
diff --git a/surfsense_evals/src/surfsense_evals/suites/__init__.py b/surfsense_evals/src/surfsense_evals/suites/__init__.py
index 95ed958ca..a0a01223c 100644
--- a/surfsense_evals/src/surfsense_evals/suites/__init__.py
+++ b/surfsense_evals/src/surfsense_evals/suites/__init__.py
@@ -20,7 +20,7 @@ from __future__ import annotations
import importlib
import logging
import pkgutil
-from typing import Iterable
+from collections.abc import Iterable
logger = logging.getLogger(__name__)
diff --git a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py
index 1da33926c..43dc51ac5 100644
--- a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py
+++ b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py
@@ -6,7 +6,6 @@ import argparse
from typing import Any
from ....core.registry import (
- Benchmark,
ReportSection,
RunArtifact,
RunContext,
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py
index e13224be7..7e9d9a07b 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py
@@ -12,7 +12,7 @@ Recall@k / MRR / nDCG@10 against qrels.
from __future__ import annotations
-from .runner import CureBenchmark
from ....core import registry as _registry
+from .runner import CureBenchmark
_registry.register(CureBenchmark())
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py
index 275e28ce5..0c32a38a1 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py
@@ -227,12 +227,10 @@ async def run_ingest(
def _take(it: Iterable, n: int) -> Iterable:
- yielded = 0
- for x in it:
- if yielded >= n:
+ for i, x in enumerate(it):
+ if i >= n:
return
yield x
- yielded += 1
__all__ = ["DISCIPLINES", "CorpusPassage", "PassageBatch", "run_ingest"]
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py
index 041e0e8b5..4a85b6ba5 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py
@@ -34,7 +34,6 @@ from ....core.ingest_settings import (
)
from ....core.metrics.retrieval import score_run
from ....core.registry import (
- Benchmark,
ReportSection,
RunArtifact,
RunContext,
@@ -276,7 +275,7 @@ class CureBenchmark:
)
per_query_retrieved: dict[str, list[str]] = {}
- for q, res in zip(queries, results):
+ for q, res in zip(queries, results, strict=False):
chunk_ids: list[int] = []
seen: set[int] = set()
for citation in res.citations:
@@ -311,7 +310,7 @@ class CureBenchmark:
run_dir = ctx.runs_dir(run_timestamp=run_timestamp)
raw_path = run_dir / "raw.jsonl"
with raw_path.open("w", encoding="utf-8") as fh:
- for q, res in zip(queries, results):
+ for q, res in zip(queries, results, strict=False):
fh.write(
json.dumps(
{
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py
index e527b37f4..265dd62f7 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py
@@ -11,7 +11,7 @@ document — the corpus is millions of biomedical snippets.
from __future__ import annotations
-from .runner import MirageBenchmark
from ....core import registry as _registry
+from .runner import MirageBenchmark
_registry.register(MirageBenchmark())
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py
index 59006b6c0..8e891aabf 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py
@@ -93,6 +93,25 @@ class SnippetRow:
# ---------------------------------------------------------------------------
+def _reuse_cached_dest(dest: Path, *, expect_zip: bool, label: str) -> Path | None:
+ """Return ``dest`` if a usable cache hit, else ``None`` (and delete corrupt zips)."""
+
+ if not dest.exists():
+ return None
+ if expect_zip and not _is_valid_zip(dest):
+ logger.warning(
+ "Cached %s at %s failed ZIP validation (size=%d B); deleting "
+ "and re-downloading.",
+ label,
+ dest,
+ dest.stat().st_size,
+ )
+ dest.unlink(missing_ok=True)
+ return None
+ logger.info("Using cached %s at %s", label, dest)
+ return dest
+
+
async def _fetch_to_path(
url: str,
*,
@@ -127,19 +146,9 @@ async def _fetch_to_path(
surprise-grabbing 16 GB on a slow link.
"""
- if dest.exists():
- if expect_zip and not _is_valid_zip(dest):
- logger.warning(
- "Cached %s at %s failed ZIP validation (size=%d B); deleting "
- "and re-downloading.",
- label,
- dest,
- dest.stat().st_size,
- )
- dest.unlink(missing_ok=True)
- else:
- logger.info("Using cached %s at %s", label, dest)
- return dest
+ cached = _reuse_cached_dest(dest, expect_zip=expect_zip, label=label)
+ if cached is not None:
+ return cached
dest.parent.mkdir(parents=True, exist_ok=True)
partial = dest.with_suffix(dest.suffix + ".partial")
@@ -170,39 +179,38 @@ async def _fetch_to_path(
async with httpx.AsyncClient(
timeout=httpx.Timeout(timeout_s, connect=20.0),
follow_redirects=True,
- ) as client:
- async with client.stream("GET", url, headers=headers) as response:
- if existing_bytes and response.status_code == 200:
- logger.warning(
- "Server ignored Range header for %s; restarting from 0.",
- label,
- )
- partial.unlink(missing_ok=True)
- existing_bytes = 0
- elif response.status_code == 416:
- # Range not satisfiable — the .partial is at or
- # past the end. Treat as "already downloaded";
- # validate by closing and re-opening for atomic
- # rename below.
- logger.info(
- "Server reports %s already complete (HTTP 416).",
- label,
- )
- elif response.status_code not in (200, 206):
- response.raise_for_status()
+ ) as client, client.stream("GET", url, headers=headers) as response:
+ if existing_bytes and response.status_code == 200:
+ logger.warning(
+ "Server ignored Range header for %s; restarting from 0.",
+ label,
+ )
+ partial.unlink(missing_ok=True)
+ existing_bytes = 0
+ elif response.status_code == 416:
+ # Range not satisfiable — the .partial is at or
+ # past the end. Treat as "already downloaded";
+ # validate by closing and re-opening for atomic
+ # rename below.
+ logger.info(
+ "Server reports %s already complete (HTTP 416).",
+ label,
+ )
+ elif response.status_code not in (200, 206):
+ response.raise_for_status()
- total_size = _planned_total_size(response, existing_bytes)
- if (
- total_size is not None
- and total_size > _LARGE_DOWNLOAD_BYTES
- and not allow_large_download
- ):
- raise _LargeDownloadAbort(label, total_size)
+ total_size = _planned_total_size(response, existing_bytes)
+ if (
+ total_size is not None
+ and total_size > _LARGE_DOWNLOAD_BYTES
+ and not allow_large_download
+ ):
+ raise _LargeDownloadAbort(label, total_size)
- mode = "ab" if existing_bytes else "wb"
- with partial.open(mode) as fh:
- async for chunk in response.aiter_bytes(chunk_size=1 << 18):
- fh.write(chunk)
+ mode = "ab" if existing_bytes else "wb"
+ with partial.open(mode) as fh:
+ async for chunk in response.aiter_bytes(chunk_size=1 << 18):
+ fh.write(chunk)
# Optional content sanity check before promoting to dest.
if expect_zip and not _is_valid_zip(partial):
raise zipfile.BadZipFile(
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py
index 9e5b1c618..3e5192aaa 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py
@@ -8,7 +8,6 @@ from __future__ import annotations
from collections.abc import Mapping
-
_PROMPT_TEMPLATE = """\
You are a helpful medical expert. Answer the following multiple-choice
question using the relevant medical knowledge available to you (and any
diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py
index b01b645a9..8a6b04ae0 100644
--- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py
@@ -29,7 +29,6 @@ from ....core.ingest_settings import (
)
from ....core.metrics.mc_accuracy import accuracy_with_wilson_ci, macro_accuracy
from ....core.registry import (
- Benchmark,
ReportSection,
RunArtifact,
RunContext,
@@ -229,7 +228,7 @@ class MirageBenchmark:
run_dir = ctx.runs_dir(run_timestamp=run_timestamp)
raw_path = run_dir / "raw.jsonl"
with raw_path.open("w", encoding="utf-8") as fh:
- for q, res in zip(questions, results):
+ for q, res in zip(questions, results, strict=False):
fh.write(
json.dumps(
{
@@ -246,7 +245,7 @@ class MirageBenchmark:
for task in tasks:
n_correct = 0
n_total = 0
- for q, res in zip(questions, results):
+ for q, res in zip(questions, results, strict=False):
if q.task != task:
continue
n_total += 1
diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py
index 93c8db4ab..1360ec89e 100644
--- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py
@@ -41,8 +41,6 @@ from typing import Any
from ....core.config import set_suite_state
from ....core.parsers import (
- AzureDIError,
- LlamaCloudError,
count_pdf_pages,
parse_with_azure_di,
parse_with_llamacloud,
@@ -131,7 +129,7 @@ async def _run_one_extraction(
else:
raise ValueError(f"Unknown parser {parser!r}")
out_path.parent.mkdir(parents=True, exist_ok=True)
- out_path.write_text(markdown, encoding="utf-8")
+ await asyncio.to_thread(out_path.write_text, markdown, encoding="utf-8")
return markdown, time.monotonic() - started
diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py
index 2c4a0ffe4..297bd2fa0 100644
--- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py
@@ -603,8 +603,8 @@ class ParserCompareBenchmark:
f"engine: `{extra.get('pdf_engine', 'native')}`)."
)
body.append(
- f"- Preprocess tariff: basic = $1 / 1k pages, "
- f"premium = $10 / 1k pages."
+ "- Preprocess tariff: basic = $1 / 1k pages, "
+ "premium = $10 / 1k pages."
)
body.append("")
body.append("### Per-arm summary")
diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py
index 1b00aedc2..dd618d7e3 100644
--- a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py
@@ -177,10 +177,7 @@ def extract_main_content(
# Prefer trafilatura output even if short, but only if it
# contained any prose at all — empty trafilatura fall-through
# to the stripped form.
- if body and stripped and len(stripped) > len(body) * 1.5:
- body = stripped
- method = "fallback_strip"
- elif not body and stripped:
+ if body and stripped and len(stripped) > len(body) * 1.5 or not body and stripped:
body = stripped
method = "fallback_strip"
elif body:
diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py
index 626834505..0d4327774 100644
--- a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py
@@ -28,7 +28,6 @@ in the runner, so the format is mandatory.
from __future__ import annotations
-
_BASE_INSTRUCTIONS = (
"You are a careful question-answering assistant. The question is a "
"real-world factual question that may be about finance, music, "
diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py
index 654c261a2..efb7b4474 100644
--- a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py
@@ -830,11 +830,11 @@ def _per_facet_truthfulness(
"""Bucket truthfulness scores by ``key_fn(q)``."""
buckets: dict[str, dict[str, list[CragGradeResult]]] = {}
- for q, b, l, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False):
+ for q, b, lc, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False):
key = key_fn(q)
bucket = buckets.setdefault(key, {"bare_llm": [], "long_context": [], "surfsense": []})
bucket["bare_llm"].append(b)
- bucket["long_context"].append(l)
+ bucket["long_context"].append(lc)
bucket["surfsense"].append(s)
out: dict[str, Any] = {}
for key, arms in buckets.items():
diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py
index c3b6b878e..629874102 100644
--- a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py
@@ -102,7 +102,7 @@ def _parse_wiki_links(raw: Any) -> list[str]:
except (SyntaxError, ValueError):
# Fall back: maybe it's a comma-separated string with no quotes.
return [tok.strip() for tok in text.strip("[]").split(",") if tok.strip()]
- if isinstance(parsed, (list, tuple)):
+ if isinstance(parsed, list | tuple):
return [str(x).strip() for x in parsed if str(x).strip()]
return [str(parsed).strip()]
diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py
index 98e035f28..0288e192e 100644
--- a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py
@@ -34,7 +34,6 @@ filename (without extension), so we round-trip via
from __future__ import annotations
-import asyncio
import json
import logging
from dataclasses import dataclass
diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py
index 16bb06da4..8f8b6e3b6 100644
--- a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py
+++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py
@@ -17,7 +17,6 @@ Format expectations (mirrors the FRAMES paper, section 4):
from __future__ import annotations
-
_BASE_INSTRUCTIONS = (
"You are a careful question-answering assistant. The question may "
"require combining facts from multiple sources, doing arithmetic, "
diff --git a/surfsense_evals/tests/suites/test_crag_dataset.py b/surfsense_evals/tests/suites/test_crag_dataset.py
index 36114b52e..6221467fd 100644
--- a/surfsense_evals/tests/suites/test_crag_dataset.py
+++ b/surfsense_evals/tests/suites/test_crag_dataset.py
@@ -11,8 +11,6 @@ import bz2
import json
from pathlib import Path
-import pytest
-
from surfsense_evals.suites.research.crag.dataset import (
CragPage,
CragQuestion,
diff --git a/surfsense_evals/tests/suites/test_crag_grader.py b/surfsense_evals/tests/suites/test_crag_grader.py
index 93bf6f478..74960afa6 100644
--- a/surfsense_evals/tests/suites/test_crag_grader.py
+++ b/surfsense_evals/tests/suites/test_crag_grader.py
@@ -7,8 +7,6 @@ exercise the deterministic shortcut + the special-case routing for
from __future__ import annotations
-import pytest
-
from surfsense_evals.suites.research.crag.grader import (
CragGradeResult,
_flags_false_premise,
diff --git a/surfsense_evals/tests/suites/test_crag_html_extract.py b/surfsense_evals/tests/suites/test_crag_html_extract.py
index a2b47aa45..3bf757dbd 100644
--- a/surfsense_evals/tests/suites/test_crag_html_extract.py
+++ b/surfsense_evals/tests/suites/test_crag_html_extract.py
@@ -11,13 +11,10 @@ We don't network-fetch trafilatura; we just verify the wrapper:
from __future__ import annotations
-import pytest
-
from surfsense_evals.suites.research.crag.html_extract import (
extract_main_content,
)
-
_RICH_HTML = """\
diff --git a/surfsense_evals/tests/suites/test_frames_dataset.py b/surfsense_evals/tests/suites/test_frames_dataset.py
index e79e7db89..f76dc0b6f 100644
--- a/surfsense_evals/tests/suites/test_frames_dataset.py
+++ b/surfsense_evals/tests/suites/test_frames_dataset.py
@@ -16,8 +16,6 @@ from __future__ import annotations
import textwrap
from pathlib import Path
-import pytest
-
from surfsense_evals.suites.research.frames.dataset import (
FramesQuestion,
_parse_reasoning_types,
@@ -25,7 +23,6 @@ from surfsense_evals.suites.research.frames.dataset import (
load_questions,
)
-
# ---------------------------------------------------------------------------
# Pure-function tests
# ---------------------------------------------------------------------------
diff --git a/surfsense_evals/tests/suites/test_frames_grader.py b/surfsense_evals/tests/suites/test_frames_grader.py
index e6e38ff8a..d4bbad79a 100644
--- a/surfsense_evals/tests/suites/test_frames_grader.py
+++ b/surfsense_evals/tests/suites/test_frames_grader.py
@@ -8,10 +8,7 @@ runner knows to consult the judge.
from __future__ import annotations
-import pytest
-
from surfsense_evals.suites.research.frames.grader import (
- GradeResult,
_maybe_number,
_normalise,
_whole_word_substring,
diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
index 28ad34e66..1792bf7dd 100644
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
@@ -41,7 +41,9 @@ def register(
list[str] | None,
Field(
description="Terms to discover public profiles for (resolved via "
- "Google). Provide search_queries OR urls."
+ "Google). Google-backed discovery is slow (~30-60s per query), so "
+ "start with at most 3 queries and expand only if nothing "
+ "significant is found. Provide search_queries OR urls."
),
] = None,
search_type: Annotated[
@@ -100,7 +102,9 @@ def register(
search_queries: Annotated[
list[str] | None,
Field(
- description="Terms to discover public profiles for. Provide "
+ description="Terms to discover public profiles for. Google-backed "
+ "discovery is slow (~30-60s per query), so start with at most 3 "
+ "queries and expand only if nothing significant is found. Provide "
"search_queries OR urls."
),
] = None,
diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
index c295ec3b6..7e2bff509 100644
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
@@ -52,9 +52,11 @@ def register(
search_queries: Annotated[
list[str] | None,
Field(
- description="Keyword search terms. Returns no videos — use "
- "hashtags/profiles/urls for videos, or the user-search tool for "
- "accounts."
+ description="Keyword search terms, resolved via Google to public "
+ "TikTok videos (TikTok's own keyword search is login-walled). "
+ "Slower than hashtags/urls, so start with at most 3 queries and "
+ "expand only if nothing significant is found. For accounts by "
+ "keyword use surfsense_tiktok_user_search."
),
] = None,
results_per_page: Annotated[
@@ -67,13 +69,15 @@ def register(
workspace: WorkspaceParam = None,
response_format: ResponseFormatParam = "markdown",
) -> str:
- """Scrape public TikTok videos by hashtag, profile, or URL.
+ """Scrape public TikTok videos by hashtag, profile, URL, or keyword.
Use for TikTok video research — a creator's videos, a hashtag feed, or a
specific video/profile/hashtag URL — instead of a generic web search.
- Returns videos with text, author, stats, music, and the web URL. For
- accounts by keyword use the user-search tool; keyword search returns no
- videos. Example: hashtags=['food'], max_items=20.
+ search_queries also finds videos on a topic (resolved via Google), but is
+ slower: start with at most 3 queries and expand only if nothing
+ significant is found. Returns videos with text, author, stats, music, and
+ the web URL. For accounts by keyword use surfsense_tiktok_user_search.
+ Example: hashtags=['food'], max_items=20.
"""
return await run_scraper(
client,
diff --git a/surfsense_web/package.json b/surfsense_web/package.json
index efa04b4f6..989dde480 100644
--- a/surfsense_web/package.json
+++ b/surfsense_web/package.json
@@ -1,6 +1,6 @@
{
"name": "surfsense_web",
- "version": "0.0.31",
+ "version": "0.0.32",
"private": true,
"packageManager": "pnpm@10.26.0",
"description": "SurfSense Frontend",
From e8b3692b546c234964b0fd3a32b39b535a3c796d Mon Sep 17 00:00:00 2001
From: "DESKTOP-RTLN3BA\\$punk"
Date: Mon, 13 Jul 2026 16:30:37 -0700
Subject: [PATCH 159/160] chore: linting
---
.../playground/components/output-viewer.tsx | 10 ++-------
.../components/playground-index.tsx | 3 ++-
.../playground/components/run-detail.tsx | 8 +------
.../components/run-progress-panel.tsx | 4 +++-
.../components/run-status-badge.tsx | 5 ++++-
.../playground/components/runs-table.tsx | 3 ++-
.../playground/components/schema-form.tsx | 19 +++--------------
.../components/assistant-ui/thread.tsx | 5 +----
.../components/homepage/hero-chat-demo.tsx | 3 ++-
.../components/homepage/hero-section.tsx | 8 +++----
.../ui/sidebar/NotificationsDropdown.tsx | 4 +---
.../settings/auto-reload-settings.tsx | 10 ++++++---
.../settings/workspace-api-access-control.tsx | 21 ++++++++++++++++---
.../content/docs/connectors/native/meta.json | 10 ++++++++-
surfsense_web/hooks/use-run-stream.ts | 16 +++-----------
.../lib/apis/scrapers-api.service.ts | 6 +++---
.../lib/connectors-marketing/tiktok.tsx | 3 ++-
.../lib/playground/code-snippets.selfcheck.ts | 6 +++++-
surfsense_web/lib/playground/format.ts | 4 +---
surfsense_web/lib/playground/json-schema.ts | 8 +------
20 files changed, 74 insertions(+), 82 deletions(-)
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx
index e692734d3..72262d440 100644
--- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx
+++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx
@@ -3,7 +3,6 @@
import { Check, Copy, Download } from "lucide-react";
import { useMemo, useState } from "react";
import { Button } from "@/components/ui/button";
-import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import {
Table,
TableBody,
@@ -12,6 +11,7 @@ import {
TableHeader,
TableRow,
} from "@/components/ui/table";
+import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { downloadCsv, rowsToCsv } from "@/lib/playground/csv";
const MAX_TABLE_ROWS = 200;
@@ -117,13 +117,7 @@ export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBa
- Manually run SurfSense's platform-native APIs and inspect their output. To use these APIs outside SurfSense,{" "}
+ Manually run SurfSense's platform-native APIs and inspect their output. To use these
+ APIs outside SurfSense,{" "}
parseJsonl(run?.output_text ?? null), [run?.output_text]);
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx
index 33a88ae76..a4e9f0023 100644
--- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx
+++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx
@@ -9,7 +9,9 @@ import { formatDuration } from "@/lib/playground/format";
function eventLabel(event: ScraperRunEvent): string {
const base =
event.message ||
- (event.phase ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase()) : "Working");
+ (event.phase
+ ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase())
+ : "Working");
if (event.current !== undefined && event.current !== null) {
const counter =
event.total !== undefined && event.total !== null
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx
index cfc75ca8a..9627fb287 100644
--- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx
+++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx
@@ -14,7 +14,10 @@ export function RunStatusBadge({ status }: { status: string }) {
}
if (normalized === "success") {
return (
-
+
Success
);
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx
index 6368a1f76..57b7a547c 100644
--- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx
+++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx
@@ -71,7 +71,8 @@ export function RunsTable({ workspaceId }: { workspaceId: number }) {
- View all API runs for this workspace, including runs from the playground, API keys, and agents.
+ View all API runs for this workspace, including runs from the playground, API keys, and
+ agents.
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx
index 36aee26bb..1d7eec8cf 100644
--- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx
+++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx
@@ -43,12 +43,7 @@ function FieldControl({
if (field.kind === "boolean") {
return (
-
+
);
}
@@ -141,9 +136,7 @@ function FieldRow({
{field.required ? "required" : "optional"}
- {field.description && (
-
{field.description}
- )}
+ {field.description &&
{field.description}
}
{
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx
index 0cf6ef296..4837c0b99 100644
--- a/surfsense_web/components/assistant-ui/thread.tsx
+++ b/surfsense_web/components/assistant-ui/thread.tsx
@@ -1018,10 +1018,7 @@ const ConnectedScraperIcons: FC<{ workspaceId: number }> = ({ workspaceId }) =>
return (
-
+
diff --git a/surfsense_web/components/homepage/hero-chat-demo.tsx b/surfsense_web/components/homepage/hero-chat-demo.tsx
index e617cd51e..a4a5ebc2a 100644
--- a/surfsense_web/components/homepage/hero-chat-demo.tsx
+++ b/surfsense_web/components/homepage/hero-chat-demo.tsx
@@ -27,7 +27,8 @@ export type HeroChatDemoScript = {
type Stage = "typing" | "steps" | "answer" | "done";
-const PLACEHOLDER = "Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs";
+const PLACEHOLDER =
+ "Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs";
/** Blinking caret for the typewriter (overlay only, never inside the real input). */
function Caret() {
diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx
index 05db2781f..6e31891bd 100644
--- a/surfsense_web/components/homepage/hero-section.tsx
+++ b/surfsense_web/components/homepage/hero-section.tsx
@@ -717,10 +717,10 @@ export function HeroSection() {
"relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400"
)}
>
- SurfSense is an open-source competitive intelligence platform, like NotebookLM but with
- live scraping connectors. Your AI agents monitor competitors, track rankings, and listen
- to your market with live data from platforms like Reddit, YouTube, Instagram, TikTok,
- Google Maps, Google Search, and the open web.
+ SurfSense is an open-source competitive intelligence platform, like NotebookLM but
+ with live scraping connectors. Your AI agents monitor competitors, track rankings, and
+ listen to your market with live data from platforms like Reddit, YouTube, Instagram,
+ TikTok, Google Maps, Google Search, and the open web.
diff --git a/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx b/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx
index 4fe035ec2..1e9b1a33e 100644
--- a/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx
+++ b/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx
@@ -324,9 +324,7 @@ export function NotificationsDropdown({
)}
>
{tab.label}
-
+
{formatNotificationCount(tab.count)}
diff --git a/surfsense_web/components/settings/auto-reload-settings.tsx b/surfsense_web/components/settings/auto-reload-settings.tsx
index ede826dd0..316c5975d 100644
--- a/surfsense_web/components/settings/auto-reload-settings.tsx
+++ b/surfsense_web/components/settings/auto-reload-settings.tsx
@@ -198,8 +198,8 @@ export function AutoReloadSettings() {
Last top-up failed
- Your saved card was declined and top-ups were turned off. Update your card and
- re-enable top-ups below.
+ Your saved card was declined and top-ups were turned off. Update your card and re-enable
+ top-ups below.
)}
@@ -290,7 +290,11 @@ export function AutoReloadSettings() {
-
+
{saveMutation.isPending ? (
<>
diff --git a/surfsense_web/components/settings/workspace-api-access-control.tsx b/surfsense_web/components/settings/workspace-api-access-control.tsx
index de74d103a..e5dae1bbe 100644
--- a/surfsense_web/components/settings/workspace-api-access-control.tsx
+++ b/surfsense_web/components/settings/workspace-api-access-control.tsx
@@ -59,7 +59,12 @@ export function WorkspaceApiAccessControl({
if (isLoading) {
return (
-
+
@@ -71,7 +76,12 @@ export function WorkspaceApiAccessControl({
if (isError) {
return (
-