From fb423560d5aee09e5cebbb56b83a8ff413f00901 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:57:36 -0700 Subject: [PATCH 001/160] fix: wire video_presentation fan-in as a barrier so generate_slide_scene_codes fires once --- .../app/agents/video_presentation/graph.py | 7 +- .../agents/test_video_presentation_graph.py | 144 ++++++++++++++++++ 2 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 surfsense_backend/tests/unit/agents/test_video_presentation_graph.py diff --git a/surfsense_backend/app/agents/video_presentation/graph.py b/surfsense_backend/app/agents/video_presentation/graph.py index 1d87bcd76..5aed3d3d1 100644 --- a/surfsense_backend/app/agents/video_presentation/graph.py +++ b/surfsense_backend/app/agents/video_presentation/graph.py @@ -24,9 +24,10 @@ def build_graph(): workflow.add_edge("create_presentation_slides", "create_slide_audio") workflow.add_edge("create_presentation_slides", "assign_slide_themes") - # Fan-in: scene code generation waits for both audio and themes. - workflow.add_edge("create_slide_audio", "generate_slide_scene_codes") - workflow.add_edge("assign_slide_themes", "generate_slide_scene_codes") + # Fan-in: wait for BOTH audio and themes (barrier). + workflow.add_edge( + ["create_slide_audio", "assign_slide_themes"], "generate_slide_scene_codes" + ) workflow.add_edge("generate_slide_scene_codes", "__end__") diff --git a/surfsense_backend/tests/unit/agents/test_video_presentation_graph.py b/surfsense_backend/tests/unit/agents/test_video_presentation_graph.py new file mode 100644 index 000000000..3283af782 --- /dev/null +++ b/surfsense_backend/tests/unit/agents/test_video_presentation_graph.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import importlib +import operator +import sys +import types +from typing import Annotated, Any + +import pytest +from langgraph.graph import END, START, StateGraph +from typing_extensions import TypedDict + +pytestmark = pytest.mark.unit + +_GRAPH_MODULE = "app.agents.video_presentation.graph" +_NODES_MODULE = "app.agents.video_presentation.nodes" +_TARGET_NODE = "generate_slide_scene_codes" +_BARRIER_SOURCES = ("create_slide_audio", "assign_slide_themes") + + +class _DelayedState(TypedDict, total=False): + calls: Annotated[list[str], operator.add] + audio_ready: bool + themes_ready: bool + + +def _config(thread_id: str = "video-presentation-graph-test") -> dict[str, Any]: + return { + "configurable": { + "search_space_id": 1, + "thread_id": thread_id, + "video_title": "Test deck", + } + } + + +def _input_state() -> dict[str, Any]: + return {"db_session": None, "source_content": "source material"} + + +def _build_video_graph(monkeypatch: pytest.MonkeyPatch, calls: list[str]): + nodes = types.ModuleType(_NODES_MODULE) + + async def create_presentation_slides(_state, config): + _ = config + calls.append("create_presentation_slides") + return {"slides": [{"slide_number": 1, "title": "Intro"}]} + + async def create_slide_audio(_state, config): + _ = config + calls.append("create_slide_audio") + return {"slide_audio_results": [{"slide_number": 1}]} + + async def assign_slide_themes(_state, config): + _ = config + calls.append("assign_slide_themes") + return {"slide_theme_assignments": {1: ("corporate", "light")}} + + async def generate_slide_scene_codes(state, config): + _ = config + calls.append(_TARGET_NODE) + assert state.slide_audio_results == [{"slide_number": 1}] + assert state.slide_theme_assignments == {1: ("corporate", "light")} + return {"slide_scene_codes": [{"slide_number": 1, "code": "", "title": ""}]} + + nodes.create_presentation_slides = create_presentation_slides + nodes.create_slide_audio = create_slide_audio + nodes.assign_slide_themes = assign_slide_themes + nodes.generate_slide_scene_codes = generate_slide_scene_codes + + monkeypatch.setitem(sys.modules, _NODES_MODULE, nodes) + monkeypatch.delitem(sys.modules, _GRAPH_MODULE, raising=False) + + graph_module = importlib.import_module(_GRAPH_MODULE) + return graph_module.build_graph() + + +@pytest.mark.asyncio +async def test_video_presentation_graph_generates_scene_codes_once(monkeypatch): + calls: list[str] = [] + graph = _build_video_graph(monkeypatch, calls) + + await graph.ainvoke(_input_state(), _config()) + + assert calls.count(_TARGET_NODE) == 1 + scene_index = calls.index(_TARGET_NODE) + assert calls.index("create_slide_audio") < scene_index + assert calls.index("assign_slide_themes") < scene_index + + +def test_video_presentation_graph_registers_single_barrier_trigger(monkeypatch): + graph = _build_video_graph(monkeypatch, []) + + assert graph.builder.waiting_edges == {(_BARRIER_SOURCES, _TARGET_NODE)} + assert not { + edge + for edge in graph.builder.edges + if edge[0] in _BARRIER_SOURCES and edge[1] == _TARGET_NODE + } + + join_channel = f"join:{'+'.join(_BARRIER_SOURCES)}:{_TARGET_NODE}" + assert join_channel in graph.channels + assert graph.nodes[_TARGET_NODE].triggers.count(join_channel) == 1 + + +@pytest.mark.asyncio +async def test_barrier_fires_once_when_one_branch_finishes_a_superstep_later(): + def create_presentation_slides(_state): + return {"calls": ["create_presentation_slides"]} + + def create_slide_audio(_state): + return {"calls": ["create_slide_audio"]} + + def finish_slide_audio(_state): + return {"calls": ["finish_slide_audio"], "audio_ready": True} + + def assign_slide_themes(_state): + return {"calls": ["assign_slide_themes"], "themes_ready": True} + + def generate_slide_scene_codes(state): + assert state["audio_ready"] is True + assert state["themes_ready"] is True + return {"calls": [_TARGET_NODE]} + + workflow = StateGraph(_DelayedState) + workflow.add_node("create_presentation_slides", create_presentation_slides) + workflow.add_node("create_slide_audio", create_slide_audio) + workflow.add_node("finish_slide_audio", finish_slide_audio) + workflow.add_node("assign_slide_themes", assign_slide_themes) + workflow.add_node(_TARGET_NODE, generate_slide_scene_codes) + + workflow.add_edge(START, "create_presentation_slides") + workflow.add_edge("create_presentation_slides", "create_slide_audio") + workflow.add_edge("create_presentation_slides", "assign_slide_themes") + workflow.add_edge("create_slide_audio", "finish_slide_audio") + workflow.add_edge(["finish_slide_audio", "assign_slide_themes"], _TARGET_NODE) + workflow.add_edge(_TARGET_NODE, END) + + result = await workflow.compile().ainvoke({"calls": []}) + + calls = result["calls"] + assert calls.count(_TARGET_NODE) == 1 + assert calls.index("assign_slide_themes") < calls.index(_TARGET_NODE) + assert calls.index("finish_slide_audio") < calls.index(_TARGET_NODE) From f44cb8a493f0d36f85a53a6944c44afa1872c528 Mon Sep 17 00:00:00 2001 From: pelazas Date: Sat, 4 Jul 2026 09:53:51 +0200 Subject: [PATCH 002/160] fix: dispatch periodic BookStack indexing from the scheduler task_map BOOKSTACK_CONNECTOR was missing from check_periodic_schedules' task_map, so connectors with periodic indexing enabled fell through to the "No task found" warning branch every minute and never synced. Fixes #891 Co-Authored-By: Claude Fable 5 --- .../celery_tasks/schedule_checker_task.py | 2 + .../tests/unit/tasks/celery_tasks/__init__.py | 0 .../test_schedule_checker_task.py | 129 ++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py create mode 100644 surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py diff --git a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py index e88fb58b9..5f088e4d8 100644 --- a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py @@ -48,6 +48,7 @@ async def _check_and_trigger_schedules(): # Live connectors (Linear, Slack, Jira, ClickUp, Airtable, Discord, # Teams, Gmail, Calendar, Luma) use real-time tools instead. from app.tasks.celery_tasks.connector_tasks import ( + index_bookstack_pages_task, index_confluence_pages_task, index_crawled_urls_task, index_elasticsearch_documents_task, @@ -58,6 +59,7 @@ async def _check_and_trigger_schedules(): task_map = { SearchSourceConnectorType.NOTION_CONNECTOR: index_notion_pages_task, + SearchSourceConnectorType.BOOKSTACK_CONNECTOR: index_bookstack_pages_task, SearchSourceConnectorType.GITHUB_CONNECTOR: index_github_repos_task, SearchSourceConnectorType.CONFLUENCE_CONNECTOR: index_confluence_pages_task, SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: index_elasticsearch_documents_task, diff --git a/surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py b/surfsense_backend/tests/unit/tasks/celery_tasks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py b/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py new file mode 100644 index 000000000..4bca74135 --- /dev/null +++ b/surfsense_backend/tests/unit/tasks/celery_tasks/test_schedule_checker_task.py @@ -0,0 +1,129 @@ +"""Unit tests for the periodic schedule checker's connector-to-task dispatch.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from app.db import SearchSourceConnectorType +from app.tasks.celery_tasks import schedule_checker_task + +pytestmark = pytest.mark.unit + + +class _FakeScalars: + def __init__(self, connectors): + self._connectors = connectors + + def all(self): + return self._connectors + + +class _FakeDueConnectorsResult: + def __init__(self, connectors): + self._connectors = connectors + + def scalars(self): + return _FakeScalars(self._connectors) + + +class _FakeEmptyResult: + def first(self): + return None + + +class _FakeSession: + """Session stub: first execute() returns due connectors, later ones no rows.""" + + def __init__(self, connectors): + self._results = [_FakeDueConnectorsResult(connectors)] + self.commits = 0 + + async def execute(self, _query): + if self._results: + return self._results.pop(0) + return _FakeEmptyResult() + + async def commit(self): + self.commits += 1 + + async def rollback(self): + pass + + +def _due_connector(connector_type: SearchSourceConnectorType) -> SimpleNamespace: + return SimpleNamespace( + id=42, + connector_type=connector_type, + search_space_id=7, + user_id="00000000-0000-0000-0000-000000000001", + config={}, + periodic_indexing_enabled=True, + indexing_frequency_minutes=60, + next_scheduled_at=datetime.now(UTC) - timedelta(minutes=5), + ) + + +async def _run_checker(monkeypatch: pytest.MonkeyPatch, connector: SimpleNamespace): + session = _FakeSession([connector]) + + @asynccontextmanager + async def _session_ctx(): + yield session + + monkeypatch.setattr( + schedule_checker_task, "get_celery_session_maker", lambda: _session_ctx + ) + monkeypatch.setattr( + schedule_checker_task, "is_connector_indexing_locked", lambda _id: False + ) + await schedule_checker_task._check_and_trigger_schedules() + return session + + +@pytest.mark.asyncio +async def test_due_bookstack_connector_dispatches_indexing_task(monkeypatch): + """A due BookStack connector must dispatch index_bookstack_pages_task. + + Regression test for the connector type missing from the scheduler's + task_map, which made periodic BookStack syncs silently no-op with only a + "No task found" warning. + """ + from app.tasks.celery_tasks import connector_tasks + + task_mock = MagicMock() + monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock) + + connector = _due_connector(SearchSourceConnectorType.BOOKSTACK_CONNECTOR) + session = await _run_checker(monkeypatch, connector) + + task_mock.delay.assert_called_once_with( + connector.id, + connector.search_space_id, + str(connector.user_id), + None, + None, + ) + # The next run must be rescheduled, otherwise the connector stays "due" + # and is re-examined every minute. + assert connector.next_scheduled_at > datetime.now(UTC) + assert session.commits == 1 + + +@pytest.mark.asyncio +async def test_unmapped_connector_type_does_not_dispatch(monkeypatch): + """Connector types absent from task_map are skipped without dispatching.""" + from app.tasks.celery_tasks import connector_tasks + + task_mock = MagicMock() + monkeypatch.setattr(connector_tasks, "index_bookstack_pages_task", task_mock) + + connector = _due_connector(SearchSourceConnectorType.TAVILY_API) + session = await _run_checker(monkeypatch, connector) + + task_mock.delay.assert_not_called() + assert session.commits == 0 From 6fafedca6a95f4fcfaa202d417379d3e61f2c908 Mon Sep 17 00:00:00 2001 From: Dustin Persek Date: Sun, 5 Jul 2026 22:33:09 -0400 Subject: [PATCH 003/160] Support separate embedding base URL --- docker/.env.example | 5 ++ surfsense_backend/.env.example | 4 + surfsense_backend/app/config/__init__.py | 14 ++-- .../app/config/embedding_settings.py | 48 ++++++++++++ .../unit/config/test_embedding_settings.py | 76 +++++++++++++++++++ .../docker-installation/docker-compose.mdx | 1 + .../content/docs/local-models/ollama.mdx | 17 +++++ .../content/docs/manual-installation.mdx | 1 + 8 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 surfsense_backend/app/config/embedding_settings.py create mode 100644 surfsense_backend/tests/unit/config/test_embedding_settings.py diff --git a/docker/.env.example b/docker/.env.example index 18142c614..df8fcbfba 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -48,7 +48,12 @@ ETL_SERVICE=DOCLING # Local: sentence-transformers/all-MiniLM-L6-v2 # OpenAI: openai://text-embedding-ada-002 (set OPENAI_API_KEY below) # Cohere: cohere://embed-english-light-v3.0 (set COHERE_API_KEY below) +# Ollama or OpenAI-compatible embedding endpoint: +# EMBEDDING_MODEL=litellm://ollama/nomic-embed-text +# EMBEDDING_BASE_URL=http://host.docker.internal:11434 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 +# EMBEDDING_BASE_URL= +# OLLAMA_EMBEDDING_BASE_URL= # ------------------------------------------------------------------------------ # How You Access SurfSense diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index a1d410eef..5bf888167 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -214,6 +214,10 @@ COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callb # # Get Cohere embeddings # embeddings = AutoEmbeddings.get_embeddings("cohere://embed-english-light-v3.0", api_key="...") EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 +# Optional: use a separate endpoint for Chonkie/LiteLLM embedding models, for +# example EMBEDDING_MODEL=litellm://ollama/nomic-embed-text. +# EMBEDDING_BASE_URL=http://host.docker.internal:11434 +# OLLAMA_EMBEDDING_BASE_URL=http://host.docker.internal:11434 # Rerankers Config RERANKERS_ENABLED=TRUE or FALSE(Default: FALSE) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 405718f0c..fc7c42311 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -9,6 +9,11 @@ from chonkie import AutoEmbeddings, CodeChunker, RecursiveChunker from dotenv import load_dotenv from rerankers import Reranker +from app.config.embedding_settings import ( + build_embedding_kwargs, + resolve_embedding_base_url, +) + # Get the base directory of the project BASE_DIR = Path(__file__).resolve().parent.parent.parent @@ -876,16 +881,13 @@ class Config: # Chonkie Configuration | Edit this to your needs EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL") + EMBEDDING_BASE_URL = resolve_embedding_base_url() # Azure OpenAI credentials from environment variables AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT") AZURE_OPENAI_API_KEY = os.getenv("AZURE_OPENAI_API_KEY") - # Pass Azure credentials to embeddings when using Azure OpenAI - embedding_kwargs = {} - if AZURE_OPENAI_ENDPOINT: - embedding_kwargs["azure_endpoint"] = AZURE_OPENAI_ENDPOINT - if AZURE_OPENAI_API_KEY: - embedding_kwargs["azure_api_key"] = AZURE_OPENAI_API_KEY + # Pass provider-specific settings to embeddings when supported. + embedding_kwargs = build_embedding_kwargs(embedding_model=EMBEDDING_MODEL) embedding_model_instance = AutoEmbeddings.get_embeddings( EMBEDDING_MODEL, diff --git a/surfsense_backend/app/config/embedding_settings.py b/surfsense_backend/app/config/embedding_settings.py new file mode 100644 index 000000000..571ada346 --- /dev/null +++ b/surfsense_backend/app/config/embedding_settings.py @@ -0,0 +1,48 @@ +import os +from collections.abc import Mapping + +EMBEDDING_BASE_URL_ENV = "EMBEDDING_BASE_URL" +OLLAMA_EMBEDDING_BASE_URL_ENV = "OLLAMA_EMBEDDING_BASE_URL" + + +def _clean_env_value(value: str | None) -> str | None: + if value is None: + return None + stripped = value.strip() + return stripped or None + + +def resolve_embedding_base_url(environ: Mapping[str, str] | None = None) -> str | None: + """Return the configured embedding endpoint, if any.""" + environ = os.environ if environ is None else environ + return _clean_env_value(environ.get(EMBEDDING_BASE_URL_ENV)) or _clean_env_value( + environ.get(OLLAMA_EMBEDDING_BASE_URL_ENV) + ) + + +def _supports_embedding_api_base(embedding_model: str | None) -> bool: + return (embedding_model or "").startswith("litellm://") + + +def build_embedding_kwargs( + environ: Mapping[str, str] | None = None, + *, + embedding_model: str | None = None, +) -> dict[str, str]: + """Build keyword arguments for Chonkie's embedding provider.""" + environ = os.environ if environ is None else environ + + embedding_kwargs: dict[str, str] = {} + embedding_base_url = resolve_embedding_base_url(environ) + if embedding_base_url and _supports_embedding_api_base(embedding_model): + embedding_kwargs["api_base"] = embedding_base_url + + azure_openai_endpoint = _clean_env_value(environ.get("AZURE_OPENAI_ENDPOINT")) + azure_openai_api_key = _clean_env_value(environ.get("AZURE_OPENAI_API_KEY")) + + if azure_openai_endpoint: + embedding_kwargs["azure_endpoint"] = azure_openai_endpoint + if azure_openai_api_key: + embedding_kwargs["azure_api_key"] = azure_openai_api_key + + return embedding_kwargs diff --git a/surfsense_backend/tests/unit/config/test_embedding_settings.py b/surfsense_backend/tests/unit/config/test_embedding_settings.py new file mode 100644 index 000000000..7483f9457 --- /dev/null +++ b/surfsense_backend/tests/unit/config/test_embedding_settings.py @@ -0,0 +1,76 @@ +import pytest + +from app.config.embedding_settings import ( + build_embedding_kwargs, + resolve_embedding_base_url, +) + +pytestmark = pytest.mark.unit + + +def test_resolve_embedding_base_url_prefers_generic_value() -> None: + environ = { + "EMBEDDING_BASE_URL": " http://embed-host:11434 ", + "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434", + } + + assert resolve_embedding_base_url(environ) == "http://embed-host:11434" + + +def test_resolve_embedding_base_url_falls_back_to_ollama_specific_value() -> None: + environ = { + "EMBEDDING_BASE_URL": " ", + "OLLAMA_EMBEDDING_BASE_URL": "http://ollama-embed:11434", + } + + assert resolve_embedding_base_url(environ) == "http://ollama-embed:11434" + + +def test_build_embedding_kwargs_maps_base_url_to_litellm_api_base() -> None: + kwargs = build_embedding_kwargs( + {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"}, + embedding_model="litellm://ollama/nomic-embed-text", + ) + + assert kwargs == {"api_base": "http://host.docker.internal:11435"} + + +def test_build_embedding_kwargs_does_not_leak_api_base_to_other_providers() -> None: + kwargs = build_embedding_kwargs( + {"EMBEDDING_BASE_URL": "http://host.docker.internal:11435"}, + embedding_model="cohere://embed-english-light-v3.0", + ) + + assert kwargs == {} + + +def test_build_embedding_kwargs_preserves_azure_settings() -> None: + kwargs = build_embedding_kwargs( + { + "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com", + "AZURE_OPENAI_API_KEY": "test-key", + }, + embedding_model="azure_openai://text-embedding-3-small", + ) + + assert kwargs == { + "azure_endpoint": "https://example.openai.azure.com", + "azure_api_key": "test-key", + } + + +def test_build_embedding_kwargs_combines_litellm_and_azure_env_when_set() -> None: + kwargs = build_embedding_kwargs( + { + "EMBEDDING_BASE_URL": "http://host.docker.internal:4000/v1", + "AZURE_OPENAI_ENDPOINT": "https://example.openai.azure.com", + "AZURE_OPENAI_API_KEY": "test-key", + }, + embedding_model="litellm://openai/text-embedding-3-small", + ) + + assert kwargs == { + "api_base": "http://host.docker.internal:4000/v1", + "azure_endpoint": "https://example.openai.azure.com", + "azure_api_key": "test-key", + } diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index bf71c077b..d9d713334 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -39,6 +39,7 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y | `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` | | `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` | | `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` | +| `EMBEDDING_BASE_URL` | Optional separate endpoint for Chonkie/LiteLLM embedding models. Use with values like `EMBEDDING_MODEL=litellm://ollama/nomic-embed-text`. `OLLAMA_EMBEDDING_BASE_URL` is also accepted as an Ollama-specific fallback. | *(empty)* | | `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` | | `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` | | `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` | diff --git a/surfsense_web/content/docs/local-models/ollama.mdx b/surfsense_web/content/docs/local-models/ollama.mdx index b062d98b0..ec4c50638 100644 --- a/surfsense_web/content/docs/local-models/ollama.mdx +++ b/surfsense_web/content/docs/local-models/ollama.mdx @@ -50,6 +50,23 @@ http://:11434 Replace `` with the LAN IP or domain for that machine. +## Embeddings on a Separate Ollama Server + +Search-space model connections configure chat and completion models. The global +embedding model is configured in the backend environment. + +Use Chonkie's LiteLLM embedding provider when embeddings run on Ollama: + +```dotenv +EMBEDDING_MODEL=litellm://ollama/nomic-embed-text +EMBEDDING_BASE_URL=http://host.docker.internal:11434 +``` + +If chat and embeddings run on different Ollama instances, keep the chat model +connection pointed at the chat server and set `EMBEDDING_BASE_URL` to the +embedding server. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an +Ollama-specific fallback when `EMBEDDING_BASE_URL` is not set. + ## Add the Connection 1. Open Search Space Settings. diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index ab09b4155..b115fa03d 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -85,6 +85,7 @@ Edit the `.env` file and set the following variables: | GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | | GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | | EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) | +| EMBEDDING_BASE_URL | (Optional) Separate endpoint for Chonkie/LiteLLM embedding models, such as `http://localhost:11434` with `EMBEDDING_MODEL=litellm://ollama/nomic-embed-text`. `OLLAMA_EMBEDDING_BASE_URL` is also supported as an Ollama-specific fallback. | | RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) | | RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) | | RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) (required if RERANKERS_ENABLED=TRUE) | From 97f6497e2a401e4879cc726f4ccdd5ef15ae108f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 004/160] feat(mcp): add per-request API key identity auth slice --- .../src/surfsense_mcp/core/auth/__init__.py | 7 +++ .../src/surfsense_mcp/core/auth/headers.py | 26 +++++++++ .../src/surfsense_mcp/core/auth/identity.py | 42 +++++++++++++++ .../src/surfsense_mcp/core/auth/middleware.py | 54 +++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/headers.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/identity.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py new file mode 100644 index 000000000..e837c6d61 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/__init__.py @@ -0,0 +1,7 @@ +"""Per-request caller identity: header parsing, request-scoped storage, and the +ASGI middleware that binds them together for the remote transport.""" + +from .identity import current_api_key, current_identity +from .middleware import ApiKeyIdentityMiddleware + +__all__ = ["current_api_key", "current_identity", "ApiKeyIdentityMiddleware"] diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py new file mode 100644 index 000000000..c06989909 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py @@ -0,0 +1,26 @@ +"""Extraction of a SurfSense API key from request headers. + +Pure and side-effect free: given the request headers, return the caller's key +or ``None``. Isolated from transport and state so the parsing rules stay +trivially unit-testable. +""" + +from __future__ import annotations + +from starlette.datastructures import Headers + +_BEARER_PREFIX = "bearer " + + +def extract_api_key(headers: Headers) -> str | None: + """Return the caller's key from the ``Authorization: Bearer`` slot the + backend already expects, falling back to ``X-API-Key`` for clients that can + only send custom headers.""" + authorization = headers.get("authorization", "") + if authorization[: len(_BEARER_PREFIX)].lower() == _BEARER_PREFIX: + token = authorization[len(_BEARER_PREFIX) :].strip() + if token: + return token + + fallback = headers.get("x-api-key", "").strip() + return fallback or None diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py new file mode 100644 index 000000000..5f242e94e --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py @@ -0,0 +1,42 @@ +"""Request-scoped caller identity. + +Over streamable-http one process serves many users, so the caller's key lives +in a contextvar for the life of a single request: the auth middleware binds it, +and the client reads it when building the outbound backend call. Under stdio +there is no request, the contextvar stays empty, and the env key is used. + +The contextvar is request-scoped, not stored state — it is re-derived from the +header on every request, which is what keeps the server stateless. +""" + +from __future__ import annotations + +from contextvars import ContextVar, Token + +_LOCAL_IDENTITY = "__local__" + +_api_key: ContextVar[str | None] = ContextVar("surfsense_api_key", default=None) + + +def bind_api_key(api_key: str | None) -> Token: + """Bind the caller's key to the current request; returns a reset token.""" + return _api_key.set(api_key) + + +def unbind_api_key(token: Token) -> None: + """Release the binding once the request is done.""" + _api_key.reset(token) + + +def current_api_key() -> str | None: + """The caller's key for the in-flight request, or ``None`` under stdio.""" + return _api_key.get() + + +def current_identity() -> str: + """Stable per-caller key for scoping request state. + + The token identifies the account, so state keyed on it is naturally + per-user and survives reconnects. Under stdio all calls share one identity. + """ + return _api_key.get() or _LOCAL_IDENTITY diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py new file mode 100644 index 000000000..2f2356ad7 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -0,0 +1,54 @@ +"""ASGI middleware that establishes the caller's identity for each request. + +A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``: +the latter runs the endpoint in a separate task, so a contextvar set in it does +not reach the tool handler. A pure middleware sets the key in the request's own +task, from which the SDK's per-request handling inherits it (verified). + +Requests without a key are rejected here so no tool ever runs unauthenticated. +""" + +from __future__ import annotations + +from starlette.datastructures import Headers +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from .headers import extract_api_key +from .identity import bind_api_key, unbind_api_key + + +class ApiKeyIdentityMiddleware: + """Binds the per-request API key into the identity contextvar, or 401s.""" + + def __init__(self, app: ASGIApp) -> None: + self._app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self._app(scope, receive, send) + return + + api_key = extract_api_key(Headers(scope=scope)) + if api_key is None: + await _unauthenticated()(scope, receive, send) + return + + token = bind_api_key(api_key) + try: + await self._app(scope, receive, send) + finally: + unbind_api_key(token) + + +def _unauthenticated() -> JSONResponse: + return JSONResponse( + { + "error": "unauthorized", + "message": ( + "Missing SurfSense API key. Send 'Authorization: Bearer " + "ss_pat_...' (or an X-API-Key header)." + ), + }, + status_code=401, + ) From 80e032ec3a2d43779b9bb51029b8d71804716c2f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 005/160] feat(mcp): assemble streamable-http transport app --- .../surfsense_mcp/core/transport/__init__.py | 5 ++++ .../src/surfsense_mcp/core/transport/http.py | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py create mode 100644 surfsense_mcp/src/surfsense_mcp/core/transport/http.py diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py new file mode 100644 index 000000000..53aaa6b8c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/__init__.py @@ -0,0 +1,5 @@ +"""Transport wiring for the MCP server (streamable-http app assembly).""" + +from .http import build_http_app + +__all__ = ["build_http_app"] diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py new file mode 100644 index 000000000..2f2cf8f53 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py @@ -0,0 +1,27 @@ +"""Assemble the streamable-http ASGI app for the remote transport. + +Wraps the SDK's MCP endpoint with the API-key identity middleware and CORS. +CORS sits outermost so browser preflight (which carries no key) is answered +before the identity middleware, and clients can read the ``Mcp-Session-Id`` +header the streamable-http protocol relies on. +""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP +from starlette.middleware.cors import CORSMiddleware +from starlette.types import ASGIApp + +from ..auth.middleware import ApiKeyIdentityMiddleware + + +def build_http_app(mcp: FastMCP) -> ASGIApp: + """Return the MCP streamable-http app wrapped with identity + CORS.""" + app: ASGIApp = ApiKeyIdentityMiddleware(mcp.streamable_http_app()) + return CORSMiddleware( + app, + allow_origins=["*"], + allow_methods=["GET", "POST", "DELETE", "OPTIONS"], + allow_headers=["*"], + expose_headers=["Mcp-Session-Id"], + ) From 8a54e0293eccb04ac46ef2e7dc83fd1402fb5240 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 006/160] refactor(mcp): resolve API key per request in client --- .../src/surfsense_mcp/core/client.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py index 55f9d6048..cfe1a73ee 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/client.py +++ b/surfsense_mcp/src/surfsense_mcp/core/client.py @@ -10,10 +10,11 @@ from typing import Any import httpx +from .auth.identity import current_api_key from .errors import ToolError _FAILURE_HINTS: dict[int, str] = { - 401: "Authentication failed — check that SURFSENSE_API_KEY is a valid, unexpired key.", + 401: "Authentication failed — the SurfSense API key is invalid or expired.", 402: "The workspace is out of credits for this operation.", 403: ( "Access denied — the token lacks permission, or API access is disabled " @@ -27,17 +28,31 @@ _FAILURE_HINTS: dict[int, str] = { class SurfSenseClient: """Issues authenticated requests against ``{base_url}{api_prefix}``.""" - def __init__(self, *, api_base: str, api_key: str, timeout: float) -> None: + def __init__( + self, *, api_base: str, timeout: float, fallback_api_key: str | None = None + ) -> None: self._api_base = api_base + # The key is resolved per request (one client serves many users over + # http), so none is baked into the shared client. ``fallback_api_key`` + # is the env-supplied key used under stdio, where there is no header. + self._fallback_api_key = fallback_api_key self._http = httpx.AsyncClient( base_url=api_base, - headers={ - "Authorization": f"Bearer {api_key}", - "Accept": "application/json", - }, + headers={"Accept": "application/json"}, timeout=timeout, ) + def _auth_headers(self) -> dict[str, str]: + """Resolve the caller's key: the per-request header, else the env key.""" + api_key = current_api_key() or self._fallback_api_key + if not api_key: + raise ToolError( + "No SurfSense API key supplied. Send it as an 'Authorization: " + "Bearer ss_pat_...' header (remote server), or set the " + "SURFSENSE_API_KEY environment variable (stdio)." + ) + return {"Authorization": f"Bearer {api_key}"} + async def request( self, method: str, @@ -53,9 +68,16 @@ class SurfSenseClient: # as a value (e.g. int("") on folder_id) and fail. if params is not None: params = {key: value for key, value in params.items() if value is not None} + headers = self._auth_headers() try: response = await self._http.request( - method, path, params=params, json=json, data=data, files=files + method, + path, + params=params, + json=json, + data=data, + files=files, + headers=headers, ) except httpx.RequestError as exc: raise ToolError( From 3b4678ade46df2d53be6657771007ae86f5c7f0e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 007/160] feat(mcp): make api_key optional, add http host/port config --- surfsense_mcp/src/surfsense_mcp/config.py | 25 +++++++++++++------- surfsense_mcp/src/surfsense_mcp/selfcheck.py | 2 ++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/config.py b/surfsense_mcp/src/surfsense_mcp/config.py index 5d3646545..18d0f1514 100644 --- a/surfsense_mcp/src/surfsense_mcp/config.py +++ b/surfsense_mcp/src/surfsense_mcp/config.py @@ -12,6 +12,8 @@ from dataclasses import dataclass DEFAULT_BASE_URL = "http://localhost:8000" DEFAULT_API_PREFIX = "/api/v1" DEFAULT_TIMEOUT_SECONDS = 180.0 +DEFAULT_HTTP_HOST = "127.0.0.1" +DEFAULT_HTTP_PORT = 8080 @dataclass(frozen=True) @@ -19,10 +21,12 @@ class Settings: """Resolved configuration for a server process.""" base_url: str - api_key: str + api_key: str | None api_prefix: str timeout: float default_workspace: str | None + host: str + port: int @property def api_base(self) -> str: @@ -30,13 +34,9 @@ class Settings: @classmethod def from_env(cls) -> Settings: - api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() - if not api_key: - raise SystemExit( - "SURFSENSE_API_KEY is required. Create an API key in SurfSense " - "(Settings -> API) and pass it via the SURFSENSE_API_KEY " - "environment variable." - ) + # Optional here: remote (http) callers pass their own key per request in + # a header. ``__main__`` enforces it for stdio, its only source of a key. + api_key = os.environ.get("SURFSENSE_API_KEY", "").strip() or None base_url = ( os.environ.get("SURFSENSE_BASE_URL", DEFAULT_BASE_URL).strip().rstrip("/") @@ -53,10 +53,19 @@ class Settings: default_workspace = os.environ.get("SURFSENSE_WORKSPACE", "").strip() or None + host = os.environ.get("SURFSENSE_MCP_HOST", "").strip() or DEFAULT_HTTP_HOST + raw_port = os.environ.get("SURFSENSE_MCP_PORT", "").strip() + try: + port = int(raw_port) if raw_port else DEFAULT_HTTP_PORT + except ValueError: + port = DEFAULT_HTTP_PORT + return cls( base_url=base_url, api_key=api_key, api_prefix=api_prefix, timeout=timeout, default_workspace=default_workspace, + host=host, + port=port, ) diff --git a/surfsense_mcp/src/surfsense_mcp/selfcheck.py b/surfsense_mcp/src/surfsense_mcp/selfcheck.py index e38edc909..20224c173 100644 --- a/surfsense_mcp/src/surfsense_mcp/selfcheck.py +++ b/surfsense_mcp/src/surfsense_mcp/selfcheck.py @@ -47,6 +47,8 @@ async def _collect_tools() -> dict[str, object]: api_prefix="/api/v1", timeout=5.0, default_workspace=None, + host="127.0.0.1", + port=8080, ) mcp, _client = build_server(settings) tools = await mcp.list_tools() From 93bd022c02e3d4d60c40c60b009a9a88f5f1fbb3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 008/160] feat(mcp): serve stateless streamable-http with stdio dispatch --- surfsense_mcp/src/surfsense_mcp/__main__.py | 33 +++++++++++++++++++-- surfsense_mcp/src/surfsense_mcp/server.py | 11 ++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/__main__.py b/surfsense_mcp/src/surfsense_mcp/__main__.py index 6c878ebfa..37a2a1785 100644 --- a/surfsense_mcp/src/surfsense_mcp/__main__.py +++ b/surfsense_mcp/src/surfsense_mcp/__main__.py @@ -1,7 +1,12 @@ """Entry point: load settings from the environment and run the MCP server. -Defaults to stdio (what Cursor, Claude Code, and Claude Desktop launch). stdout -is the protocol channel, so every log line goes to stderr. +Two transports share one build: +- ``stdio`` (default): Cursor/Claude launch one process per user; the key comes + from the environment, so it is required here. +- ``streamable-http``: one process serves many users, each passing their own key + per request; the key is enforced by the transport's auth middleware instead. + +For stdio, stdout is the protocol channel, so every log line goes to stderr. """ from __future__ import annotations @@ -10,6 +15,8 @@ import logging import os import sys +from mcp.server.fastmcp import FastMCP + from .config import Settings from .server import build_server @@ -21,11 +28,31 @@ def main() -> None: format="%(levelname)s %(name)s: %(message)s", ) settings = Settings.from_env() + transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio" mcp, _client = build_server(settings) - transport = os.environ.get("SURFSENSE_MCP_TRANSPORT", "stdio").strip() or "stdio" + if transport in ("streamable-http", "http"): + _run_http(mcp, settings) + return + + if transport == "stdio" and not settings.api_key: + raise SystemExit( + "SURFSENSE_API_KEY is required for stdio transport. Create an API " + "key in SurfSense (Settings -> API) and pass it via the " + "SURFSENSE_API_KEY environment variable." + ) mcp.run(transport=transport) +def _run_http(mcp: FastMCP, settings: Settings) -> None: + """Serve the streamable-http app directly, so the per-request identity + middleware wraps the SDK's MCP endpoint.""" + import uvicorn + + from .core.transport import build_http_app + + uvicorn.run(build_http_app(mcp), host=settings.host, port=settings.port) + + if __name__ == "__main__": main() diff --git a/surfsense_mcp/src/surfsense_mcp/server.py b/surfsense_mcp/src/surfsense_mcp/server.py index 88eb19437..8ea9bc919 100644 --- a/surfsense_mcp/src/surfsense_mcp/server.py +++ b/surfsense_mcp/src/surfsense_mcp/server.py @@ -17,12 +17,21 @@ from .features import knowledge_base, scrapers, workspaces def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: """Assemble a configured server and the client whose lifecycle it shares.""" client = SurfSenseClient( - api_base=settings.api_base, api_key=settings.api_key, timeout=settings.timeout + api_base=settings.api_base, + timeout=settings.timeout, + fallback_api_key=settings.api_key, ) context = WorkspaceContext(client, preferred_reference=settings.default_workspace) mcp = FastMCP( "SurfSense", + host=settings.host, + port=settings.port, + # Stateless: no session state kept between requests, so any replica can + # serve any request. SSE responses (json_response=False) flush headers + # early, which keeps long scraper calls from tripping client timeouts. + stateless_http=True, + json_response=False, instructions=( "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " From 44d0f28c5f860096f5f687639fd60faa87335e7f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 009/160] fix(mcp): scope active workspace per caller identity --- .../surfsense_mcp/core/workspace_context.py | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py index 6682d53b5..cfd90c32b 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py @@ -7,14 +7,22 @@ speaks a name, we resolve it, and remember the choice for later calls. from __future__ import annotations +from collections import OrderedDict from dataclasses import dataclass from typing import Annotated from pydantic import Field +from .auth.identity import current_identity from .client import SurfSenseClient from .errors import ToolError +# ponytail: one small entry per distinct caller (API token). Bounded so a flood +# of keys can't grow memory without limit; an evicted caller just re-resolves +# its default workspace on the next call. Upgrade path: a TTL/LRU store if +# per-caller state ever grows past this one field. +_MAX_TRACKED_IDENTITIES = 2048 + # Shared parameter type for every workspace-scoped tool. WorkspaceParam = Annotated[ str | None, @@ -44,15 +52,21 @@ class WorkspaceContext: ) -> None: self._client = client self._preferred_reference = preferred_reference - self._active: Workspace | None = None + # Active selection is per caller: one shared slot would leak one user's + # choice to every other user on a shared server. + self._active_by_identity: OrderedDict[str, Workspace] = OrderedDict() @property def active(self) -> Workspace | None: - return self._active + return self._active_by_identity.get(current_identity()) def remember(self, workspace: Workspace) -> Workspace: - """Make ``workspace`` the default for later scoped calls.""" - self._active = workspace + """Make ``workspace`` the default for the current caller's later calls.""" + identity = current_identity() + self._active_by_identity[identity] = workspace + self._active_by_identity.move_to_end(identity) + while len(self._active_by_identity) > _MAX_TRACKED_IDENTITIES: + self._active_by_identity.popitem(last=False) return workspace async def fetch_all(self) -> list[Workspace]: @@ -67,8 +81,9 @@ class WorkspaceContext: return self.remember(await self._match(reference)) async def _resolve_default(self) -> Workspace: - if self._active is not None: - return self._active + active = self.active + if active is not None: + return active if self._preferred_reference: return self.remember(await self._match(self._preferred_reference)) return self.remember(await self._only_workspace_or_prompt()) From fa7d5f83f0ea8db081199d6e2afb99154c25c827 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 010/160] test(mcp): cover per-request auth and workspace isolation --- surfsense_mcp/tests/test_auth_headers.py | 40 +++++++ surfsense_mcp/tests/test_client_errors.py | 2 +- surfsense_mcp/tests/test_client_params.py | 4 +- surfsense_mcp/tests/test_request_auth.py | 100 ++++++++++++++++++ surfsense_mcp/tests/test_workspace_context.py | 28 +++++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 surfsense_mcp/tests/test_auth_headers.py create mode 100644 surfsense_mcp/tests/test_request_auth.py diff --git a/surfsense_mcp/tests/test_auth_headers.py b/surfsense_mcp/tests/test_auth_headers.py new file mode 100644 index 000000000..efb2d85a9 --- /dev/null +++ b/surfsense_mcp/tests/test_auth_headers.py @@ -0,0 +1,40 @@ +"""API key extraction from request headers: Bearer, fallback, and rejection.""" + +from __future__ import annotations + +from starlette.datastructures import Headers + +from surfsense_mcp.core.auth.headers import extract_api_key + + +def _headers(**pairs: str) -> Headers: + return Headers(pairs) + + +def test_reads_bearer_token(): + assert extract_api_key(_headers(authorization="Bearer ss_pat_abc")) == "ss_pat_abc" + + +def test_bearer_scheme_is_case_insensitive(): + assert extract_api_key(_headers(authorization="bearer ss_pat_abc")) == "ss_pat_abc" + + +def test_falls_back_to_x_api_key(): + assert extract_api_key(Headers({"x-api-key": "ss_pat_xyz"})) == "ss_pat_xyz" + + +def test_bearer_wins_over_fallback(): + headers = Headers({"authorization": "Bearer primary", "x-api-key": "secondary"}) + assert extract_api_key(headers) == "primary" + + +def test_missing_headers_return_none(): + assert extract_api_key(_headers()) is None + + +def test_empty_bearer_is_rejected(): + assert extract_api_key(_headers(authorization="Bearer ")) is None + + +def test_non_bearer_authorization_is_ignored(): + assert extract_api_key(_headers(authorization="Basic abc123")) is None diff --git a/surfsense_mcp/tests/test_client_errors.py b/surfsense_mcp/tests/test_client_errors.py index 648bfde3e..0525fb523 100644 --- a/surfsense_mcp/tests/test_client_errors.py +++ b/surfsense_mcp/tests/test_client_errors.py @@ -15,7 +15,7 @@ def _response(status: int, **kwargs) -> httpx.Response: def test_explains_401_with_token_hint(): message = SurfSenseClient._explain_failure(_response(401, json={"detail": "bad"})) - assert "SURFSENSE_API_KEY" in message + assert "API key" in message assert "bad" in message diff --git a/surfsense_mcp/tests/test_client_params.py b/surfsense_mcp/tests/test_client_params.py index 4840ad488..e9cd33a0c 100644 --- a/surfsense_mcp/tests/test_client_params.py +++ b/surfsense_mcp/tests/test_client_params.py @@ -25,7 +25,9 @@ def _capture(client: SurfSenseClient) -> dict: def test_none_params_are_dropped(): - client = SurfSenseClient(api_base="http://test/api/v1", api_key="ss_pat_x", timeout=5) + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key="ss_pat_x" + ) seen = _capture(client) asyncio.run( client.request( diff --git a/surfsense_mcp/tests/test_request_auth.py b/surfsense_mcp/tests/test_request_auth.py new file mode 100644 index 000000000..c548dff7a --- /dev/null +++ b/surfsense_mcp/tests/test_request_auth.py @@ -0,0 +1,100 @@ +"""Per-request key resolution and the Authorization header the backend receives. + +Covers the security-critical behaviors: the per-request key wins over the env +fallback, the fallback covers stdio, a missing key is refused, and concurrent +callers never see each other's key. +""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from surfsense_mcp.core.auth import identity +from surfsense_mcp.core.client import SurfSenseClient +from surfsense_mcp.core.errors import ToolError + + +def _client_recording_auth(seen: dict, *, fallback: str | None) -> SurfSenseClient: + async def handler(request: httpx.Request) -> httpx.Response: + seen["authorization"] = request.headers.get("authorization") + return httpx.Response(200, json={"ok": True}) + + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key=fallback + ) + client._http = httpx.AsyncClient( + base_url="http://test/api/v1", transport=httpx.MockTransport(handler) + ) + return client + + +async def _get(client: SurfSenseClient) -> None: + await client.request("GET", "/workspaces") + + +def test_request_key_is_sent_as_bearer(): + seen: dict = {} + client = _client_recording_auth(seen, fallback=None) + + async def run() -> None: + token = identity.bind_api_key("ss_pat_request") + try: + await _get(client) + finally: + identity.unbind_api_key(token) + + asyncio.run(run()) + assert seen["authorization"] == "Bearer ss_pat_request" + + +def test_request_key_overrides_env_fallback(): + seen: dict = {} + client = _client_recording_auth(seen, fallback="ss_pat_env") + + async def run() -> None: + token = identity.bind_api_key("ss_pat_request") + try: + await _get(client) + finally: + identity.unbind_api_key(token) + + asyncio.run(run()) + assert seen["authorization"] == "Bearer ss_pat_request" + + +def test_env_fallback_used_without_request_key(): + seen: dict = {} + client = _client_recording_auth(seen, fallback="ss_pat_env") + asyncio.run(_get(client)) + assert seen["authorization"] == "Bearer ss_pat_env" + + +def test_missing_key_is_refused(): + client = _client_recording_auth({}, fallback=None) + with pytest.raises(ToolError): + asyncio.run(_get(client)) + + +def test_concurrent_callers_do_not_leak_keys(): + seen_by_caller: dict[str, str | None] = {} + + async def call_as(key: str) -> None: + # Each caller runs in its own task, so the contextvar is isolated. + recorded: dict = {} + client = _client_recording_auth(recorded, fallback=None) + token = identity.bind_api_key(key) + try: + await _get(client) + finally: + identity.unbind_api_key(token) + seen_by_caller[key] = recorded["authorization"] + + async def run() -> None: + await asyncio.gather(call_as("ss_pat_A"), call_as("ss_pat_B")) + + asyncio.run(run()) + assert seen_by_caller["ss_pat_A"] == "Bearer ss_pat_A" + assert seen_by_caller["ss_pat_B"] == "Bearer ss_pat_B" diff --git a/surfsense_mcp/tests/test_workspace_context.py b/surfsense_mcp/tests/test_workspace_context.py index 9ba9c2ca3..d2024e5d5 100644 --- a/surfsense_mcp/tests/test_workspace_context.py +++ b/surfsense_mcp/tests/test_workspace_context.py @@ -6,6 +6,7 @@ import asyncio import pytest +from surfsense_mcp.core.auth import identity from surfsense_mcp.core.errors import ToolError from surfsense_mcp.core.workspace_context import WorkspaceContext @@ -96,3 +97,30 @@ def test_resolution_is_remembered_as_active(): assert ctx.active is not None and ctx.active.id == 2 # a later default call reuses the active selection without re-choosing assert asyncio.run(ctx.resolve(None)).id == 2 + + +def test_active_workspace_is_isolated_per_identity(): + ctx = _context(_rows(("A", 1), ("B", 2))) + + async def select_as(key: str, reference: str) -> None: + token = identity.bind_api_key(key) + try: + await ctx.resolve(reference) + finally: + identity.unbind_api_key(token) + + async def active_for(key: str) -> int | None: + token = identity.bind_api_key(key) + try: + return ctx.active.id if ctx.active else None + finally: + identity.unbind_api_key(token) + + asyncio.run(select_as("ss_pat_A", "A")) + asyncio.run(select_as("ss_pat_B", "B")) + + # Each caller keeps its own selection; no bleed across identities. + assert asyncio.run(active_for("ss_pat_A")) == 1 + assert asyncio.run(active_for("ss_pat_B")) == 2 + # An unknown caller has no active selection. + assert asyncio.run(active_for("ss_pat_C")) is None From 0d4033682d6bda104d7a16622715bba4976ae393 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:52:30 +0200 Subject: [PATCH 011/160] build(mcp): declare starlette and uvicorn dependencies --- surfsense_mcp/pyproject.toml | 2 ++ surfsense_mcp/uv.lock | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/surfsense_mcp/pyproject.toml b/surfsense_mcp/pyproject.toml index 87db0c223..489bbf455 100644 --- a/surfsense_mcp/pyproject.toml +++ b/surfsense_mcp/pyproject.toml @@ -8,6 +8,8 @@ license = { text = "Apache-2.0" } dependencies = [ "mcp>=1.26.0", "httpx>=0.27.0", + "starlette>=0.37", + "uvicorn>=0.30", ] [project.scripts] diff --git a/surfsense_mcp/uv.lock b/surfsense_mcp/uv.lock index bc97a9b2a..d1bfab702 100644 --- a/surfsense_mcp/uv.lock +++ b/surfsense_mcp/uv.lock @@ -718,6 +718,8 @@ source = { editable = "." } dependencies = [ { name = "httpx" }, { name = "mcp" }, + { name = "starlette" }, + { name = "uvicorn" }, ] [package.dev-dependencies] @@ -729,6 +731,8 @@ dev = [ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "mcp", specifier = ">=1.26.0" }, + { name = "starlette", specifier = ">=0.37" }, + { name = "uvicorn", specifier = ">=0.30" }, ] [package.metadata.requires-dev] From a7215c09dc44a58d0a8686d74cd577deb3c6e15f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 16:55:20 +0200 Subject: [PATCH 012/160] docs(mcp): tighten comments to intent-only --- .../src/surfsense_mcp/core/auth/headers.py | 6 ++---- .../src/surfsense_mcp/core/auth/identity.py | 18 +++++------------- .../src/surfsense_mcp/core/auth/middleware.py | 4 ++-- surfsense_mcp/src/surfsense_mcp/core/client.py | 5 ++--- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py index c06989909..3bca45967 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/headers.py @@ -1,8 +1,6 @@ -"""Extraction of a SurfSense API key from request headers. +"""Extract a SurfSense API key from request headers. -Pure and side-effect free: given the request headers, return the caller's key -or ``None``. Isolated from transport and state so the parsing rules stay -trivially unit-testable. +Pure header parsing, kept separate from transport and state. """ from __future__ import annotations diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py index 5f242e94e..0c8f6b315 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/identity.py @@ -1,12 +1,9 @@ """Request-scoped caller identity. -Over streamable-http one process serves many users, so the caller's key lives -in a contextvar for the life of a single request: the auth middleware binds it, -and the client reads it when building the outbound backend call. Under stdio -there is no request, the contextvar stays empty, and the env key is used. - -The contextvar is request-scoped, not stored state — it is re-derived from the -header on every request, which is what keeps the server stateless. +Over streamable-http one process serves many users, so the caller's key lives in +a contextvar for the life of a request: the auth middleware binds it and the +client reads it when calling the backend. Under stdio there is no request, so the +contextvar is empty and the env key is used instead. """ from __future__ import annotations @@ -24,7 +21,6 @@ def bind_api_key(api_key: str | None) -> Token: def unbind_api_key(token: Token) -> None: - """Release the binding once the request is done.""" _api_key.reset(token) @@ -34,9 +30,5 @@ def current_api_key() -> str | None: def current_identity() -> str: - """Stable per-caller key for scoping request state. - - The token identifies the account, so state keyed on it is naturally - per-user and survives reconnects. Under stdio all calls share one identity. - """ + """Stable per-caller key for scoping request state; shared under stdio.""" return _api_key.get() or _LOCAL_IDENTITY diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py index 2f2356ad7..354ae0a89 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -2,8 +2,8 @@ A pure ASGI middleware, deliberately not Starlette's ``BaseHTTPMiddleware``: the latter runs the endpoint in a separate task, so a contextvar set in it does -not reach the tool handler. A pure middleware sets the key in the request's own -task, from which the SDK's per-request handling inherits it (verified). +not reach the tool handler. A pure middleware binds the key in the request's own +task, from which the SDK's per-request handling inherits it. Requests without a key are rejected here so no tool ever runs unauthenticated. """ diff --git a/surfsense_mcp/src/surfsense_mcp/core/client.py b/surfsense_mcp/src/surfsense_mcp/core/client.py index cfe1a73ee..9ae446ae2 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/client.py +++ b/surfsense_mcp/src/surfsense_mcp/core/client.py @@ -32,9 +32,8 @@ class SurfSenseClient: self, *, api_base: str, timeout: float, fallback_api_key: str | None = None ) -> None: self._api_base = api_base - # The key is resolved per request (one client serves many users over - # http), so none is baked into the shared client. ``fallback_api_key`` - # is the env-supplied key used under stdio, where there is no header. + # Resolved per request, so no key is baked into the shared client. The + # fallback is the env key used under stdio, where there is no header. self._fallback_api_key = fallback_api_key self._http = httpx.AsyncClient( base_url=api_base, From 2acc1426bf8294f0fd03c5a6e1e7f97b6aba9310 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 17:22:37 +0200 Subject: [PATCH 013/160] refactor(mcp): split scraper and knowledge-base features into SRP modules --- .../surfsense_mcp/core/workspace_context.py | 43 +- .../surfsense_mcp/core/workspace_matching.py | 51 ++ .../features/knowledge_base/__init__.py | 371 +----------- .../features/knowledge_base/annotations.py | 34 ++ .../features/knowledge_base/document_tools.py | 185 ++++++ .../features/knowledge_base/search_tools.py | 188 ++++++ .../features/scrapers/__init__.py | 562 +----------------- .../features/scrapers/annotations.py | 13 + .../features/scrapers/platforms/__init__.py | 1 + .../scrapers/platforms/google_maps.py | 151 +++++ .../scrapers/platforms/google_search.py | 79 +++ .../features/scrapers/platforms/reddit.py | 98 +++ .../features/scrapers/platforms/web.py | 89 +++ .../features/scrapers/platforms/youtube.py | 131 ++++ .../features/scrapers/run_history.py | 112 ++++ 15 files changed, 1153 insertions(+), 955 deletions(-) create mode 100644 surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py create mode 100644 surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py index cfd90c32b..0c5423e6b 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_context.py @@ -16,6 +16,7 @@ from pydantic import Field from .auth.identity import current_identity from .client import SurfSenseClient from .errors import ToolError +from .workspace_matching import as_int, match_by_name, name_list # ponytail: one small entry per distinct caller (API token). Bounded so a flood # of keys can't grow memory without limit; an evicted caller just re-resolves @@ -99,43 +100,20 @@ class WorkspaceContext: ) raise ToolError( "No workspace selected. Choose one first with surfsense_select_workspace, " - f"or pass 'workspace'. Available: {_name_list(workspaces)}." + f"or pass 'workspace'. Available: {name_list(workspaces)}." ) async def _match(self, reference: str | int) -> Workspace: workspaces = await self.fetch_all() - as_id = _as_int(reference) + as_id = as_int(reference) if as_id is not None: found = next((w for w in workspaces if w.id == as_id), None) if found is None: raise ToolError( - f"No workspace with id {as_id}. Available: {_name_list(workspaces)}." + f"No workspace with id {as_id}. Available: {name_list(workspaces)}." ) return found - return _match_by_name(str(reference), workspaces) - - -def _match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace: - """Match on name: exact, then case-insensitive, then unique substring.""" - needle = reference.strip() - exact = [w for w in workspaces if w.name == needle] - if exact: - return exact[0] - lowered = needle.casefold() - insensitive = [w for w in workspaces if w.name.casefold() == lowered] - if insensitive: - return insensitive[0] - partial = [w for w in workspaces if lowered in w.name.casefold()] - if len(partial) == 1: - return partial[0] - if len(partial) > 1: - raise ToolError( - f"'{reference}' matches several workspaces: {_name_list(partial)}. " - "Use a more specific name or the id." - ) - raise ToolError( - f"No workspace named '{reference}'. Available: {_name_list(workspaces)}." - ) + return match_by_name(str(reference), workspaces) def _to_workspace(row: dict) -> Workspace: @@ -146,14 +124,3 @@ def _to_workspace(row: dict) -> Workspace: is_owner=row.get("is_owner", False), member_count=row.get("member_count", 1), ) - - -def _as_int(reference: str | int) -> int | None: - if isinstance(reference, int): - return reference - text = reference.strip() - return int(text) if text.isdigit() else None - - -def _name_list(workspaces: list[Workspace]) -> str: - return ", ".join(f"{w.name} (id {w.id})" for w in workspaces) diff --git a/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py new file mode 100644 index 000000000..33bfc50a5 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/core/workspace_matching.py @@ -0,0 +1,51 @@ +"""Resolve a user-supplied workspace reference to a single workspace. + +Pure matching over an already-fetched list: name (exact, then case-insensitive, +then unique substring) or numeric id. Kept apart from WorkspaceContext so the +resolution rules can be read and tested without the network. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .errors import ToolError + +if TYPE_CHECKING: + from .workspace_context import Workspace + + +def match_by_name(reference: str, workspaces: list[Workspace]) -> Workspace: + """Match on name: exact, then case-insensitive, then unique substring.""" + needle = reference.strip() + exact = [w for w in workspaces if w.name == needle] + if exact: + return exact[0] + lowered = needle.casefold() + insensitive = [w for w in workspaces if w.name.casefold() == lowered] + if insensitive: + return insensitive[0] + partial = [w for w in workspaces if lowered in w.name.casefold()] + if len(partial) == 1: + return partial[0] + if len(partial) > 1: + raise ToolError( + f"'{reference}' matches several workspaces: {name_list(partial)}. " + "Use a more specific name or the id." + ) + raise ToolError( + f"No workspace named '{reference}'. Available: {name_list(workspaces)}." + ) + + +def as_int(reference: str | int) -> int | None: + """Return the reference as an id, or None when it isn't numeric.""" + if isinstance(reference, int): + return reference + text = reference.strip() + return int(text) if text.isdigit() else None + + +def name_list(workspaces: list[Workspace]) -> str: + """Render workspaces as a human-readable 'name (id N)' list.""" + return ", ".join(f"{w.name} (id {w.id})" for w in workspaces) diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py index 7fb1802ae..1a971bfe4 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/__init__.py @@ -1,379 +1,22 @@ """Knowledge-base tools: search the KB and manage its documents. Semantic search plus the document lifecycle — list, read, add text, upload a -file, update, and delete — over a workspace's knowledge base. Search and reads -default to the active workspace; document ids identify a single document across -the whole account, so id-addressed tools need no workspace. +file, update, and delete — over a workspace's knowledge base. Read tools live in +search_tools, mutations in document_tools. """ from __future__ import annotations -import mimetypes -from pathlib import Path -from typing import Annotated - from mcp.server.fastmcp import FastMCP -from mcp.types import ToolAnnotations -from pydantic import Field from ...core.client import SurfSenseClient -from ...core.errors import ToolError -from ...core.rendering import ResponseFormatParam, clip, to_json -from ...core.workspace_context import WorkspaceContext, WorkspaceParam -from .note_ingestion import build_note_document - -_READ = ToolAnnotations( - readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False -) -_WRITE = ToolAnnotations( - readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False -) -_DELETE = ToolAnnotations( - readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False -) - -_DOCUMENT_ID = Annotated[ - int, - Field( - description="Document id from surfsense_search_knowledge_base or " - "surfsense_list_documents results." - ), -] - -_DOCUMENT_TYPES = Annotated[ - list[str] | None, - Field( - description="Restrict to these document types, e.g. " - "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." - ), -] +from ...core.workspace_context import WorkspaceContext +from . import document_tools, search_tools def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the knowledge-base tools on the server.""" - - @mcp.tool( - name="surfsense_search_knowledge_base", - title="Search knowledge base", - annotations=_READ, - structured_output=False, - ) - async def search_knowledge_base( - query: Annotated[ - str, - Field( - min_length=1, - description="Natural-language search, e.g. " - "'notebooklm user complaints'.", - ), - ], - top_k: Annotated[ - int, Field(ge=1, le=20, description="Maximum documents to return.") - ] = 5, - document_types: _DOCUMENT_TYPES = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search the workspace's knowledge base by meaning and keywords. - - Use this FIRST when a question might be answered by content already - stored in SurfSense — notes, uploaded files, saved pages, past - research. Do NOT use it to fetch new data from the web; use the - scraper tools for that. Returns the most relevant documents with the - passages that matched, ranked by relevance score. - Example: query='pricing feedback', top_k=5. - """ - resolved = await context.resolve(workspace) - hits = await client.request( - "POST", - "/documents/search-semantic", - json={ - "workspace_id": resolved.id, - "query": query, - "top_k": max(1, min(top_k, 20)), - "document_types": document_types, - }, - ) - items = (hits or {}).get("items", []) - if response_format == "json": - return to_json(items) - return _render_search(query, items) - - @mcp.tool( - name="surfsense_list_documents", - title="List documents", - annotations=_READ, - structured_output=False, - ) - async def list_documents( - document_types: _DOCUMENT_TYPES = None, - folder_id: Annotated[ - int | None, - Field(description="Only documents in this folder. Omit for all."), - ] = None, - page: Annotated[ - int, Field(ge=0, description="Zero-based page number.") - ] = 0, - page_size: Annotated[ - int, Field(ge=1, description="Documents per page.") - ] = 20, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """List documents in the workspace's knowledge base, newest first. - - Use this to browse or inventory what is stored; to find documents - about a topic, prefer surfsense_search_knowledge_base. Returns each - document's title, id, type, and update time, plus a has_more flag — - request the next page by increasing page. - Example: document_types=['FILE'], page=0, page_size=20. - """ - resolved = await context.resolve(workspace) - result = await client.request( - "GET", - "/documents", - params={ - "workspace_id": resolved.id, - "page": page, - "page_size": page_size, - "document_types": _join(document_types), - "folder_id": folder_id, - }, - ) - if response_format == "json": - return to_json(result) - return _render_document_list(result) - - @mcp.tool( - name="surfsense_get_document", - title="Read one document", - annotations=_READ, - structured_output=False, - ) - async def get_document( - document_id: _DOCUMENT_ID, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Read one document's full content and metadata by id. - - Use this after surfsense_search_knowledge_base or - surfsense_list_documents to open a specific document — search results - only include the matching passages, this returns the whole text. - """ - document = await client.request("GET", f"/documents/{document_id}") - if response_format == "json": - return clip(to_json(document)) - return _render_document(document) - - @mcp.tool( - name="surfsense_add_document", - title="Add a note", - annotations=_WRITE, - structured_output=False, - ) - async def add_document( - title: Annotated[ - str, - Field(min_length=1, description="Short descriptive title for the note."), - ], - content: Annotated[ - str, - Field( - min_length=1, - description="The note's body; plain text or markdown.", - ), - ], - source_url: Annotated[ - str | None, - Field(description="Where the text came from, if anywhere."), - ] = None, - workspace: WorkspaceParam = None, - ) -> str: - """Save a text or markdown note into the workspace's knowledge base. - - Use this to store notes, summaries, or findings so they become - searchable later — e.g. after finishing a piece of research. For files - on disk use surfsense_upload_file instead. Indexing is asynchronous, - so the note may take a moment to appear in search. - Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. - """ - resolved = await context.resolve(workspace) - await client.request( - "POST", - "/documents", - json=build_note_document( - workspace_id=resolved.id, - title=title, - content=content, - source_url=source_url, - ), - ) - return ( - f"Queued '{title}' for indexing in '{resolved.name}'. " - "It will be searchable once processing completes." - ) - - @mcp.tool( - name="surfsense_upload_file", - title="Upload a file", - annotations=_WRITE, - structured_output=False, - ) - async def upload_file( - file_path: Annotated[ - str, - Field( - description="Path to a local file, e.g. " - "'C:/Users/me/report.pdf' or '~/notes/summary.md'." - ), - ], - use_vision_llm: Annotated[ - bool, - Field( - description="True reads scanned or image-heavy files with a " - "vision model (slower)." - ), - ] = False, - workspace: WorkspaceParam = None, - ) -> str: - """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. - - Use this to ingest a file from disk so its content becomes searchable; - for text you already have in hand use surfsense_add_document instead. - The file is parsed, chunked, and indexed asynchronously. Duplicate - files are detected and skipped. - Example: file_path='C:/Users/me/report.pdf'. - """ - resolved = await context.resolve(workspace) - payload = _read_upload(file_path) - result = await client.request( - "POST", - "/documents/fileupload", - data={ - "workspace_id": str(resolved.id), - "use_vision_llm": str(use_vision_llm).lower(), - "processing_mode": "basic", - }, - files=[("files", payload)], - ) - pending = (result or {}).get("pending_files", 0) - skipped = (result or {}).get("skipped_duplicates", 0) - note = " (already present, skipped)" if skipped and not pending else "" - return ( - f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. " - "It will be searchable once processing completes." - ) - - @mcp.tool( - name="surfsense_update_document", - title="Replace a document's content", - annotations=_WRITE, - structured_output=False, - ) - async def update_document( - document_id: _DOCUMENT_ID, - content: Annotated[ - str, - Field( - min_length=1, - description="New full text; replaces the existing content " - "entirely.", - ), - ], - ) -> str: - """Replace a document's stored content by id. - - Use this to correct or rewrite a document's text. The new content - REPLACES the old entirely — to append, read the document first with - surfsense_get_document and resend the combined text. Search chunks are - not re-indexed by this call. - """ - existing = await client.request("GET", f"/documents/{document_id}") - await client.request( - "PUT", - f"/documents/{document_id}", - json={ - "document_type": existing["document_type"], - "workspace_id": existing["workspace_id"], - "content": content, - }, - ) - return f"Updated document {document_id} ('{existing.get('title', '')}')." - - @mcp.tool( - name="surfsense_delete_document", - title="Delete a document", - annotations=_DELETE, - structured_output=False, - ) - async def delete_document(document_id: _DOCUMENT_ID) -> str: - """Permanently delete a document from the knowledge base by id. - - Use this only when the user explicitly asks to remove a document — - deletion cannot be undone. The document stops appearing in searches - immediately. - """ - await client.request("DELETE", f"/documents/{document_id}") - return f"Deleted document {document_id}." - - -def _read_upload(file_path: str) -> tuple[str, bytes, str]: - path = Path(file_path).expanduser() - if not path.is_file(): - raise ToolError(f"No file at '{file_path}'.") - mime, _ = mimetypes.guess_type(path.name) - return (path.name, path.read_bytes(), mime or "application/octet-stream") - - -def _join(values: list[str] | None) -> str | None: - return ",".join(values) if values else None - - -def _render_search(query: str, items: list[dict]) -> str: - if not items: - return f'No matches for "{query}".' - lines = [f'# {len(items)} result(s) for "{query}"', ""] - for hit in items: - lines.append( - f"## {hit.get('title', 'Untitled')} " - f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}" - ) - for chunk in hit.get("chunks", []): - excerpt = clip(chunk.get("content", "").strip(), 500) - lines.append(f"> {excerpt}") - lines.append("") - return "\n".join(lines).strip() - - -def _render_document_list(result: dict | None) -> str: - items = (result or {}).get("items", []) - if not items: - return "No documents found." - lines = ["# Documents", ""] - for doc in items: - lines.append( - f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · " - f"{doc.get('document_type')} · updated {doc.get('updated_at')}" - ) - total = (result or {}).get("total", len(items)) - page = (result or {}).get("page", 0) - has_more = (result or {}).get("has_more", False) - lines.append("") - lines.append( - f"_Page {page} · showing {len(items)} of {total}" - + (" · more available_" if has_more else "_") - ) - return "\n".join(lines) - - -def _render_document(document: dict) -> str: - content = clip(document.get("content", "") or "(empty)") - return ( - f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" - f"- type: {document.get('document_type')}\n" - f"- workspace: {document.get('workspace_id')}\n" - f"- updated: {document.get('updated_at')}\n\n" - f"{content}" - ) + """Register every knowledge-base tool on the server.""" + search_tools.register(mcp, client, context) + document_tools.register(mcp, client, context) diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py new file mode 100644 index 000000000..16322506c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/annotations.py @@ -0,0 +1,34 @@ +"""Tool-call policy hints and shared parameter types for knowledge-base tools.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.types import ToolAnnotations +from pydantic import Field + +READ = ToolAnnotations( + readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False +) +WRITE = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False +) +DELETE = ToolAnnotations( + readOnlyHint=False, destructiveHint=True, idempotentHint=False, openWorldHint=False +) + +DocumentId = Annotated[ + int, + Field( + description="Document id from surfsense_search_knowledge_base or " + "surfsense_list_documents results." + ), +] + +DocumentTypes = Annotated[ + list[str] | None, + Field( + description="Restrict to these document types, e.g. " + "['FILE', 'CRAWLED_URL', 'YOUTUBE_VIDEO']. Omit for all types." + ), +] diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py new file mode 100644 index 000000000..497a2526c --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/document_tools.py @@ -0,0 +1,185 @@ +"""Knowledge-base write tools: add a note, upload a file, update, and delete. + +Add and upload target the active workspace; update and delete address a document +by its account-unique id, so they need no workspace. +""" + +from __future__ import annotations + +import mimetypes +from pathlib import Path +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.errors import ToolError +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import DELETE, WRITE, DocumentId +from .note_ingestion import build_note_document + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the knowledge-base write and delete tools.""" + + @mcp.tool( + name="surfsense_add_document", + title="Add a note", + annotations=WRITE, + structured_output=False, + ) + async def add_document( + title: Annotated[ + str, + Field(min_length=1, description="Short descriptive title for the note."), + ], + content: Annotated[ + str, + Field( + min_length=1, + description="The note's body; plain text or markdown.", + ), + ], + source_url: Annotated[ + str | None, + Field(description="Where the text came from, if anywhere."), + ] = None, + workspace: WorkspaceParam = None, + ) -> str: + """Save a text or markdown note into the workspace's knowledge base. + + Use this to store notes, summaries, or findings so they become + searchable later — e.g. after finishing a piece of research. For files + on disk use surfsense_upload_file instead. Indexing is asynchronous, + so the note may take a moment to appear in search. + Example: title='NotebookLM subreddits', content='- r/notebooklm ...'. + """ + resolved = await context.resolve(workspace) + await client.request( + "POST", + "/documents", + json=build_note_document( + workspace_id=resolved.id, + title=title, + content=content, + source_url=source_url, + ), + ) + return ( + f"Queued '{title}' for indexing in '{resolved.name}'. " + "It will be searchable once processing completes." + ) + + @mcp.tool( + name="surfsense_upload_file", + title="Upload a file", + annotations=WRITE, + structured_output=False, + ) + async def upload_file( + file_path: Annotated[ + str, + Field( + description="Path to a local file, e.g. " + "'C:/Users/me/report.pdf' or '~/notes/summary.md'." + ), + ], + use_vision_llm: Annotated[ + bool, + Field( + description="True reads scanned or image-heavy files with a " + "vision model (slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + ) -> str: + """Upload a local file (PDF, docx, markdown, etc.) into the knowledge base. + + Use this to ingest a file from disk so its content becomes searchable; + for text you already have in hand use surfsense_add_document instead. + The file is parsed, chunked, and indexed asynchronously. Duplicate + files are detected and skipped. + Example: file_path='C:/Users/me/report.pdf'. + """ + resolved = await context.resolve(workspace) + payload = _read_upload(file_path) + result = await client.request( + "POST", + "/documents/fileupload", + data={ + "workspace_id": str(resolved.id), + "use_vision_llm": str(use_vision_llm).lower(), + "processing_mode": "basic", + }, + files=[("files", payload)], + ) + pending = (result or {}).get("pending_files", 0) + skipped = (result or {}).get("skipped_duplicates", 0) + note = " (already present, skipped)" if skipped and not pending else "" + return ( + f"Uploaded '{Path(file_path).name}' to '{resolved.name}'{note}. " + "It will be searchable once processing completes." + ) + + @mcp.tool( + name="surfsense_update_document", + title="Replace a document's content", + annotations=WRITE, + structured_output=False, + ) + async def update_document( + document_id: DocumentId, + content: Annotated[ + str, + Field( + min_length=1, + description="New full text; replaces the existing content " + "entirely.", + ), + ], + ) -> str: + """Replace a document's stored content by id. + + Use this to correct or rewrite a document's text. The new content + REPLACES the old entirely — to append, read the document first with + surfsense_get_document and resend the combined text. Search chunks are + not re-indexed by this call. + """ + existing = await client.request("GET", f"/documents/{document_id}") + await client.request( + "PUT", + f"/documents/{document_id}", + json={ + "document_type": existing["document_type"], + "workspace_id": existing["workspace_id"], + "content": content, + }, + ) + return f"Updated document {document_id} ('{existing.get('title', '')}')." + + @mcp.tool( + name="surfsense_delete_document", + title="Delete a document", + annotations=DELETE, + structured_output=False, + ) + async def delete_document(document_id: DocumentId) -> str: + """Permanently delete a document from the knowledge base by id. + + Use this only when the user explicitly asks to remove a document — + deletion cannot be undone. The document stops appearing in searches + immediately. + """ + await client.request("DELETE", f"/documents/{document_id}") + return f"Deleted document {document_id}." + + +def _read_upload(file_path: str) -> tuple[str, bytes, str]: + path = Path(file_path).expanduser() + if not path.is_file(): + raise ToolError(f"No file at '{file_path}'.") + mime, _ = mimetypes.guess_type(path.name) + return (path.name, path.read_bytes(), mime or "application/octet-stream") diff --git a/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py new file mode 100644 index 000000000..a9e60810d --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/knowledge_base/search_tools.py @@ -0,0 +1,188 @@ +"""Knowledge-base read tools: semantic search, list, and read one document. + +Search and list default to the active workspace; a document read is addressed by +id, which is unique across the account, so it needs no workspace. +""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import READ, DocumentId, DocumentTypes + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the knowledge-base read tools.""" + + @mcp.tool( + name="surfsense_search_knowledge_base", + title="Search knowledge base", + annotations=READ, + structured_output=False, + ) + async def search_knowledge_base( + query: Annotated[ + str, + Field( + min_length=1, + description="Natural-language search, e.g. " + "'notebooklm user complaints'.", + ), + ], + top_k: Annotated[ + int, Field(ge=1, le=20, description="Maximum documents to return.") + ] = 5, + document_types: DocumentTypes = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search the workspace's knowledge base by meaning and keywords. + + Use this FIRST when a question might be answered by content already + stored in SurfSense — notes, uploaded files, saved pages, past + research. Do NOT use it to fetch new data from the web; use the + scraper tools for that. Returns the most relevant documents with the + passages that matched, ranked by relevance score. + Example: query='pricing feedback', top_k=5. + """ + resolved = await context.resolve(workspace) + hits = await client.request( + "POST", + "/documents/search-semantic", + json={ + "workspace_id": resolved.id, + "query": query, + "top_k": max(1, min(top_k, 20)), + "document_types": document_types, + }, + ) + items = (hits or {}).get("items", []) + if response_format == "json": + return to_json(items) + return _render_search(query, items) + + @mcp.tool( + name="surfsense_list_documents", + title="List documents", + annotations=READ, + structured_output=False, + ) + async def list_documents( + document_types: DocumentTypes = None, + folder_id: Annotated[ + int | None, + Field(description="Only documents in this folder. Omit for all."), + ] = None, + page: Annotated[ + int, Field(ge=0, description="Zero-based page number.") + ] = 0, + page_size: Annotated[ + int, Field(ge=1, description="Documents per page.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """List documents in the workspace's knowledge base, newest first. + + Use this to browse or inventory what is stored; to find documents + about a topic, prefer surfsense_search_knowledge_base. Returns each + document's title, id, type, and update time, plus a has_more flag — + request the next page by increasing page. + Example: document_types=['FILE'], page=0, page_size=20. + """ + resolved = await context.resolve(workspace) + result = await client.request( + "GET", + "/documents", + params={ + "workspace_id": resolved.id, + "page": page, + "page_size": page_size, + "document_types": _join(document_types), + "folder_id": folder_id, + }, + ) + if response_format == "json": + return to_json(result) + return _render_document_list(result) + + @mcp.tool( + name="surfsense_get_document", + title="Read one document", + annotations=READ, + structured_output=False, + ) + async def get_document( + document_id: DocumentId, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Read one document's full content and metadata by id. + + Use this after surfsense_search_knowledge_base or + surfsense_list_documents to open a specific document — search results + only include the matching passages, this returns the whole text. + """ + document = await client.request("GET", f"/documents/{document_id}") + if response_format == "json": + return clip(to_json(document)) + return _render_document(document) + + +def _join(values: list[str] | None) -> str | None: + return ",".join(values) if values else None + + +def _render_search(query: str, items: list[dict]) -> str: + if not items: + return f'No matches for "{query}".' + lines = [f'# {len(items)} result(s) for "{query}"', ""] + for hit in items: + lines.append( + f"## {hit.get('title', 'Untitled')} " + f"(id {hit.get('document_id')}) — score {hit.get('score', 0):.3f}" + ) + for chunk in hit.get("chunks", []): + excerpt = clip(chunk.get("content", "").strip(), 500) + lines.append(f"> {excerpt}") + lines.append("") + return "\n".join(lines).strip() + + +def _render_document_list(result: dict | None) -> str: + items = (result or {}).get("items", []) + if not items: + return "No documents found." + lines = ["# Documents", ""] + for doc in items: + lines.append( + f"- **{doc.get('title', 'Untitled')}** (id {doc.get('id')}) · " + f"{doc.get('document_type')} · updated {doc.get('updated_at')}" + ) + total = (result or {}).get("total", len(items)) + page = (result or {}).get("page", 0) + has_more = (result or {}).get("has_more", False) + lines.append("") + lines.append( + f"_Page {page} · showing {len(items)} of {total}" + + (" · more available_" if has_more else "_") + ) + return "\n".join(lines) + + +def _render_document(document: dict) -> str: + content = clip(document.get("content", "") or "(empty)") + return ( + f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" + f"- type: {document.get('document_type')}\n" + f"- workspace: {document.get('workspace_id')}\n" + f"- updated: {document.get('updated_at')}\n\n" + f"{content}" + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py index aa8265148..dfa2f3ab2 100644 --- a/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/__init__.py @@ -1,570 +1,26 @@ """Scraper tools: one MCP surface per SurfSense platform capability. Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper door. Two more tools +maps a natural-language request to the workspace's scraper. Two run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in -full later. +full later. Each platform lives in its own module under platforms/. """ from __future__ import annotations -from typing import Annotated, Literal - from mcp.server.fastmcp import FastMCP -from mcp.types import ToolAnnotations -from pydantic import Field from ...core.client import SurfSenseClient -from ...core.rendering import ResponseFormatParam, clip, to_json -from ...core.workspace_context import WorkspaceContext, WorkspaceParam -from .capability import run_scraper +from ...core.workspace_context import WorkspaceContext +from . import run_history +from .platforms import google_maps, google_search, reddit, web, youtube -# Scrapers reach the open web and record a billable run; they are neither -# read-only nor idempotent, but they do not mutate the knowledge base. -_SCRAPE = ToolAnnotations( - readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True -) -_READ_RUNS = ToolAnnotations( - readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False -) - -RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"] -RedditTime = Literal["hour", "day", "week", "month", "year", "all"] -CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"] -ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] +_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the scraper and run-history tools on the server.""" - - @mcp.tool( - name="surfsense_web_crawl", - title="Crawl web pages", - annotations=_SCRAPE, - structured_output=False, - ) - async def web_crawl( - start_urls: Annotated[ - list[str], - Field( - min_length=1, - description="Full URLs to fetch, e.g. " - "['https://example.com/blog/post'].", - ), - ], - max_crawl_depth: Annotated[ - int, - Field( - ge=0, - description="Link-hops to follow from start_urls within the " - "same site. 0 fetches only start_urls.", - ), - ] = 0, - max_crawl_pages: Annotated[ - int, Field(ge=1, description="Stop after this many pages in total.") - ] = 10, - max_length: Annotated[ - int, Field(ge=1, description="Max characters kept per page.") - ] = 50_000, - include_url_patterns: Annotated[ - list[str] | None, - Field( - description="Regexes; only discovered links matching one are " - "followed, e.g. ['/docs/.*']." - ), - ] = None, - exclude_url_patterns: Annotated[ - list[str] | None, - Field(description="Regexes; discovered links matching one are skipped."), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch specific web pages and return their cleaned content as markdown. - - Use this to read a page the user names, or to spider a site from a - starting URL. Do NOT use it to find pages on a topic — use - surfsense_google_search for discovery. Returns one item per crawled - page: url, title, and the page text as markdown. - Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, - include_url_patterns=['/2026/']. - """ - return await run_scraper( - client, - context, - platform="web", - verb="crawl", - payload={ - "startUrls": start_urls, - "maxCrawlDepth": max_crawl_depth, - "maxCrawlPages": max_crawl_pages, - "maxLength": max_length, - "includeUrlPatterns": include_url_patterns, - "excludeUrlPatterns": exclude_url_patterns, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_search", - title="Scrape Google Search", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_search( - queries: Annotated[ - list[str], - Field( - min_length=1, - description="Search terms or full Google Search URLs, e.g. " - "['best rss readers 2026'].", - ), - ], - max_pages_per_query: Annotated[ - int, Field(ge=1, description="Result pages to fetch per query.") - ] = 1, - country_code: Annotated[ - str | None, - Field(description="Two-letter country to search from, e.g. 'us'."), - ] = None, - language_code: Annotated[ - str, Field(description="Results language, e.g. 'en'. Empty for default.") - ] = "", - site: Annotated[ - str | None, - Field( - description="Restrict results to one domain, e.g. 'example.com'." - ), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Scrape Google Search result pages for one or more queries. - - Use this to discover pages on the open web by topic; follow up with - surfsense_web_crawl to read a result in full. Do NOT use it for - Reddit, YouTube, or Google Maps research — the dedicated tools return - richer data. Returns each query's parsed results: title, url, and - snippet per organic result. - Example: queries=['notebooklm review'], site='news.ycombinator.com'. - """ - return await run_scraper( - client, - context, - platform="google_search", - verb="scrape", - payload={ - "queries": queries, - "max_pages_per_query": max_pages_per_query, - "country_code": country_code, - "language_code": language_code, - "site": site, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_reddit_scrape", - title="Search or scrape Reddit", - annotations=_SCRAPE, - structured_output=False, - ) - async def reddit_scrape( - urls: Annotated[ - list[str] | None, - Field( - description="Reddit URLs: a post, a subreddit like " - "'https://reddit.com/r/LocalLLaMA', a user page, or a search " - "URL. Provide urls OR search_queries." - ), - ] = None, - search_queries: Annotated[ - list[str] | None, - Field( - description="Terms to search Reddit for, e.g. " - "['NotebookLM alternatives']. Provide search_queries OR urls." - ), - ] = None, - community: Annotated[ - str | None, - Field( - description="Restrict a search to one subreddit, name without " - "'r/', e.g. 'ArtificialInteligence'." - ), - ] = None, - sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", - time_filter: Annotated[ - RedditTime | None, - Field(description="Time window; only valid with sort='top'."), - ] = None, - max_items: Annotated[ - int, Field(ge=1, description="Maximum posts to return.") - ] = 10, - skip_comments: Annotated[ - bool, - Field( - description="True fetches posts only (faster); False also " - "fetches each post's comment thread." - ), - ] = False, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search or scrape Reddit: posts, comments, subreddits, and users. - - Use this for ANY Reddit research — finding relevant subreddits or - communities for a topic, top posts, or discussions — instead of a - generic web search. Returns posts (title, text, score, subreddit, url) - with comment threads unless skip_comments is set. Every post carries - its subreddit, so to find communities for a topic, search posts and - aggregate their subreddits. - Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. - """ - return await run_scraper( - client, - context, - platform="reddit", - verb="scrape", - payload={ - "urls": urls, - "search_queries": search_queries, - "community": community, - "sort": sort, - "time_filter": time_filter, - "max_items": max_items, - "skip_comments": skip_comments, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_youtube_scrape", - title="Search or scrape YouTube", - annotations=_SCRAPE, - structured_output=False, - ) - async def youtube_scrape( - urls: Annotated[ - list[str] | None, - Field( - description="YouTube URLs: video, channel, playlist, shorts, " - "or hashtag pages. Provide urls OR search_queries." - ), - ] = None, - search_queries: Annotated[ - list[str] | None, - Field( - description="Terms to search YouTube for, e.g. " - "['NotebookLM tutorial']. Provide search_queries OR urls." - ), - ] = None, - max_results: Annotated[ - int, Field(ge=1, description="Maximum videos to return.") - ] = 10, - download_subtitles: Annotated[ - bool, - Field(description="True also fetches each video's transcript."), - ] = False, - subtitles_language: Annotated[ - str, Field(description="Transcript language code, e.g. 'en'.") - ] = "en", - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Search or scrape YouTube videos, optionally with transcripts. - - Use this for YouTube research: finding videos on a topic, or reading a - video's details or transcript. For a video's comment section use - surfsense_youtube_comments instead. Returns per-video metadata (title, - channel, views, description, url) and, if requested, the transcript. - Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. - """ - return await run_scraper( - client, - context, - platform="youtube", - verb="scrape", - payload={ - "urls": urls, - "search_queries": search_queries, - "max_results": max_results, - "download_subtitles": download_subtitles, - "subtitles_language": subtitles_language, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_youtube_comments", - title="Fetch YouTube comments", - annotations=_SCRAPE, - structured_output=False, - ) - async def youtube_comments( - urls: Annotated[ - list[str], - Field( - min_length=1, - description="YouTube video URLs, e.g. " - "['https://www.youtube.com/watch?v=abc123'].", - ), - ], - max_comments: Annotated[ - int, - Field( - ge=1, - description="Maximum comments per video, counting top-level " - "comments and replies together.", - ), - ] = 20, - sort_by: Annotated[ - CommentSort, Field(description="Comment ordering.") - ] = "NEWEST_FIRST", - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch the comments (and replies) on one or more YouTube videos. - - Use this when the user wants a video's discussion or audience reaction - rather than the video itself; get video URLs from - surfsense_youtube_scrape if you only have a topic. Returns comment - text, author, likes, and replies. - Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. - """ - return await run_scraper( - client, - context, - platform="youtube", - verb="comments", - payload={ - "urls": urls, - "max_comments": max_comments, - "sort_by": sort_by, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_maps_scrape", - title="Find places on Google Maps", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_maps_scrape( - search_queries: Annotated[ - list[str] | None, - Field( - description="Place searches, e.g. ['coffee shops']. Provide " - "search_queries OR urls OR place_ids." - ), - ] = None, - urls: Annotated[ - list[str] | None, - Field(description="Google Maps URLs of specific places."), - ] = None, - place_ids: Annotated[ - list[str] | None, - Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), - ] = None, - location: Annotated[ - str | None, - Field( - description="Geographic scope for a search, e.g. " - "'Seattle, USA'." - ), - ] = None, - max_places: Annotated[ - int, Field(ge=1, description="Maximum places to return.") - ] = 10, - include_details: Annotated[ - bool, - Field( - description="True adds opening hours and extra contact info " - "(slower)." - ), - ] = False, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Find places on Google Maps by search, URL, or place id. - - Use this for local-business and location research: names, addresses, - ratings, categories, coordinates, place ids. For a place's customer - reviews use surfsense_google_maps_reviews instead. - Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. - """ - return await run_scraper( - client, - context, - platform="google_maps", - verb="scrape", - payload={ - "search_queries": search_queries, - "urls": urls, - "place_ids": place_ids, - "location": location, - "max_places": max_places, - "include_details": include_details, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_google_maps_reviews", - title="Fetch Google Maps reviews", - annotations=_SCRAPE, - structured_output=False, - ) - async def google_maps_reviews( - urls: Annotated[ - list[str] | None, - Field( - description="Google Maps URLs of places. Provide urls OR " - "place_ids." - ), - ] = None, - place_ids: Annotated[ - list[str] | None, - Field( - description="Google place ids from surfsense_google_maps_scrape." - ), - ] = None, - max_reviews: Annotated[ - int, Field(ge=1, description="Maximum reviews per place.") - ] = 20, - sort_by: Annotated[ - ReviewSort, Field(description="Review ordering.") - ] = "newest", - language: Annotated[ - str, Field(description="Reviews language code, e.g. 'en'.") - ] = "en", - start_date: Annotated[ - str | None, - Field( - description="ISO date like '2026-01-01'; keeps only reviews on " - "or after that day." - ), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch customer reviews for Google Maps places by URL or place id. - - Use this to read feedback on specific places; get urls or place_ids - from surfsense_google_maps_scrape first if you only have a name. - Returns review text, rating, author, and date per review. - Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. - """ - return await run_scraper( - client, - context, - platform="google_maps", - verb="reviews", - payload={ - "urls": urls, - "place_ids": place_ids, - "max_reviews": max_reviews, - "sort_by": sort_by, - "language": language, - "start_date": start_date, - }, - workspace=workspace, - response_format=response_format, - ) - - @mcp.tool( - name="surfsense_list_scraper_runs", - title="List past scraper runs", - annotations=_READ_RUNS, - structured_output=False, - ) - async def list_scraper_runs( - limit: Annotated[ - int, Field(ge=1, description="Maximum runs to list.") - ] = 20, - capability: Annotated[ - str | None, - Field( - description="Filter by capability slug, e.g. 'web.crawl' or " - "'reddit.scrape'." - ), - ] = None, - status: Annotated[ - str | None, - Field(description="Filter by run status: 'success' or 'error'."), - ] = None, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """List recent scraper runs in the workspace, newest first. - - Use this to find the run_id of an earlier scrape — for example when an - inline result was truncated — then fetch it in full with - surfsense_get_scraper_run. Returns each run's id, capability, status, - item count, and creation time. - Example: capability='reddit.scrape', status='success'. - """ - resolved = await context.resolve(workspace) - runs = await client.request( - "GET", - f"/workspaces/{resolved.id}/scrapers/runs", - params={ - "limit": limit, - "capability": capability, - "status": status, - }, - ) - if response_format == "json": - return to_json(runs) - return _render_runs(runs) - - @mcp.tool( - name="surfsense_get_scraper_run", - title="Fetch one scraper run in full", - annotations=_READ_RUNS, - structured_output=False, - ) - async def get_scraper_run( - run_id: Annotated[ - str, - Field( - description="Run id from surfsense_list_scraper_runs or a " - "prior scrape's output." - ), - ], - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch a single scraper run in full, including its stored output. - - Use this to retrieve the complete, untruncated result of an earlier - scrape. Do NOT re-run a scraper just to recover a truncated result — - fetch the stored run instead. - """ - resolved = await context.resolve(workspace) - run = await client.request( - "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}" - ) - if response_format == "json": - return clip(to_json(run)) - return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```" - - -def _render_runs(runs: list[dict] | None) -> str: - if not runs: - return "No scraper runs found." - lines = ["# Scraper runs", ""] - for run in runs: - lines.append( - f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · " - f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}" - ) - return "\n".join(lines) + """Register every scraper and run-history tool on the server.""" + for module in _REGISTRARS: + module.register(mcp, client, context) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py new file mode 100644 index 000000000..cb872cc90 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/annotations.py @@ -0,0 +1,13 @@ +"""Tool-call policy hints shared across scraper tools.""" + +from __future__ import annotations + +from mcp.types import ToolAnnotations + +SCRAPE = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=True +) + +READ_RUNS = ToolAnnotations( + readOnlyHint=True, destructiveHint=False, idempotentHint=True, openWorldHint=False +) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py new file mode 100644 index 000000000..f61704380 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/__init__.py @@ -0,0 +1 @@ +"""One module per scraper platform; each exposes register(mcp, client, context).""" diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py new file mode 100644 index 000000000..e1613ca4e --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_maps.py @@ -0,0 +1,151 @@ +"""Google Maps scraper tools: places and reviews.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Google Maps place and review tools.""" + + @mcp.tool( + name="surfsense_google_maps_scrape", + title="Find places on Google Maps", + annotations=SCRAPE, + structured_output=False, + ) + async def google_maps_scrape( + search_queries: Annotated[ + list[str] | None, + Field( + description="Place searches, e.g. ['coffee shops']. Provide " + "search_queries OR urls OR place_ids." + ), + ] = None, + urls: Annotated[ + list[str] | None, + Field(description="Google Maps URLs of specific places."), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field(description="Google place ids, e.g. ['ChIJj61dQgK6j4AR...']."), + ] = None, + location: Annotated[ + str | None, + Field( + description="Geographic scope for a search, e.g. " + "'Seattle, USA'." + ), + ] = None, + max_places: Annotated[ + int, Field(ge=1, description="Maximum places to return.") + ] = 10, + include_details: Annotated[ + bool, + Field( + description="True adds opening hours and extra contact info " + "(slower)." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Find places on Google Maps by search, URL, or place id. + + Use this for local-business and location research: names, addresses, + ratings, categories, coordinates, place ids. For a place's customer + reviews use surfsense_google_maps_reviews instead. + Example: search_queries=['ramen'], location='Osaka, Japan', max_places=5. + """ + return await run_scraper( + client, + context, + platform="google_maps", + verb="scrape", + payload={ + "search_queries": search_queries, + "urls": urls, + "place_ids": place_ids, + "location": location, + "max_places": max_places, + "include_details": include_details, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_google_maps_reviews", + title="Fetch Google Maps reviews", + annotations=SCRAPE, + structured_output=False, + ) + async def google_maps_reviews( + urls: Annotated[ + list[str] | None, + Field( + description="Google Maps URLs of places. Provide urls OR " + "place_ids." + ), + ] = None, + place_ids: Annotated[ + list[str] | None, + Field( + description="Google place ids from surfsense_google_maps_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, Field(ge=1, description="Maximum reviews per place.") + ] = 20, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "newest", + language: Annotated[ + str, Field(description="Reviews language code, e.g. 'en'.") + ] = "en", + start_date: Annotated[ + str | None, + Field( + description="ISO date like '2026-01-01'; keeps only reviews on " + "or after that day." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch customer reviews for Google Maps places by URL or place id. + + Use this to read feedback on specific places; get urls or place_ids + from surfsense_google_maps_scrape first if you only have a name. + Returns review text, rating, author, and date per review. + Example: place_ids=['ChIJj61dQgK6j4AR...'], sort_by='newest'. + """ + return await run_scraper( + client, + context, + platform="google_maps", + verb="reviews", + payload={ + "urls": urls, + "place_ids": place_ids, + "max_reviews": max_reviews, + "sort_by": sort_by, + "language": language, + "start_date": start_date, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py new file mode 100644 index 000000000..cc1a1f8ed --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/google_search.py @@ -0,0 +1,79 @@ +"""Google Search scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Google Search tool.""" + + @mcp.tool( + name="surfsense_google_search", + title="Scrape Google Search", + annotations=SCRAPE, + structured_output=False, + ) + async def google_search( + queries: Annotated[ + list[str], + Field( + min_length=1, + description="Search terms or full Google Search URLs, e.g. " + "['best rss readers 2026'].", + ), + ], + max_pages_per_query: Annotated[ + int, Field(ge=1, description="Result pages to fetch per query.") + ] = 1, + country_code: Annotated[ + str | None, + Field(description="Two-letter country to search from, e.g. 'us'."), + ] = None, + language_code: Annotated[ + str, Field(description="Results language, e.g. 'en'. Empty for default.") + ] = "", + site: Annotated[ + str | None, + Field( + description="Restrict results to one domain, e.g. 'example.com'." + ), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape Google Search result pages for one or more queries. + + Use this to discover pages on the open web by topic; follow up with + surfsense_web_crawl to read a result in full. Do NOT use it for + Reddit, YouTube, or Google Maps research — the dedicated tools return + richer data. Returns each query's parsed results: title, url, and + snippet per organic result. + Example: queries=['notebooklm review'], site='news.ycombinator.com'. + """ + return await run_scraper( + client, + context, + platform="google_search", + verb="scrape", + payload={ + "queries": queries, + "max_pages_per_query": max_pages_per_query, + "country_code": country_code, + "language_code": language_code, + "site": site, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py new file mode 100644 index 000000000..035193ebc --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/reddit.py @@ -0,0 +1,98 @@ +"""Reddit scraper tool.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"] +RedditTime = Literal["hour", "day", "week", "month", "year", "all"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Reddit tool.""" + + @mcp.tool( + name="surfsense_reddit_scrape", + title="Search or scrape Reddit", + annotations=SCRAPE, + structured_output=False, + ) + async def reddit_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Reddit URLs: a post, a subreddit like " + "'https://reddit.com/r/LocalLLaMA', a user page, or a search " + "URL. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search Reddit for, e.g. " + "['NotebookLM alternatives']. Provide search_queries OR urls." + ), + ] = None, + community: Annotated[ + str | None, + Field( + description="Restrict a search to one subreddit, name without " + "'r/', e.g. 'ArtificialInteligence'." + ), + ] = None, + sort: Annotated[RedditSort, Field(description="Post ordering.")] = "new", + time_filter: Annotated[ + RedditTime | None, + Field(description="Time window; only valid with sort='top'."), + ] = None, + max_items: Annotated[ + int, Field(ge=1, description="Maximum posts to return.") + ] = 10, + skip_comments: Annotated[ + bool, + Field( + description="True fetches posts only (faster); False also " + "fetches each post's comment thread." + ), + ] = False, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape Reddit: posts, comments, subreddits, and users. + + Use this for ANY Reddit research — finding relevant subreddits or + communities for a topic, top posts, or discussions — instead of a + generic web search. Returns posts (title, text, score, subreddit, url) + with comment threads unless skip_comments is set. Every post carries + its subreddit, so to find communities for a topic, search posts and + aggregate their subreddits. + Example: search_queries=['NotebookLM'], sort='top', time_filter='month'. + """ + return await run_scraper( + client, + context, + platform="reddit", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "community": community, + "sort": sort, + "time_filter": time_filter, + "max_items": max_items, + "skip_comments": skip_comments, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py new file mode 100644 index 000000000..9c24a4352 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/web.py @@ -0,0 +1,89 @@ +"""Web crawl scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the web crawl tool.""" + + @mcp.tool( + name="surfsense_web_crawl", + title="Crawl web pages", + annotations=SCRAPE, + structured_output=False, + ) + async def web_crawl( + start_urls: Annotated[ + list[str], + Field( + min_length=1, + description="Full URLs to fetch, e.g. " + "['https://example.com/blog/post'].", + ), + ], + max_crawl_depth: Annotated[ + int, + Field( + ge=0, + description="Link-hops to follow from start_urls within the " + "same site. 0 fetches only start_urls.", + ), + ] = 0, + max_crawl_pages: Annotated[ + int, Field(ge=1, description="Stop after this many pages in total.") + ] = 10, + max_length: Annotated[ + int, Field(ge=1, description="Max characters kept per page.") + ] = 50_000, + include_url_patterns: Annotated[ + list[str] | None, + Field( + description="Regexes; only discovered links matching one are " + "followed, e.g. ['/docs/.*']." + ), + ] = None, + exclude_url_patterns: Annotated[ + list[str] | None, + Field(description="Regexes; discovered links matching one are skipped."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch specific web pages and return their cleaned content as markdown. + + Use this to read a page the user names, or to spider a site from a + starting URL. Do NOT use it to find pages on a topic — use + surfsense_google_search for discovery. Returns one item per crawled + page: url, title, and the page text as markdown. + Example: start_urls=['https://blog.example.com'], max_crawl_depth=1, + include_url_patterns=['/2026/']. + """ + return await run_scraper( + client, + context, + platform="web", + verb="crawl", + payload={ + "startUrls": start_urls, + "maxCrawlDepth": max_crawl_depth, + "maxCrawlPages": max_crawl_pages, + "maxLength": max_length, + "includeUrlPatterns": include_url_patterns, + "excludeUrlPatterns": exclude_url_patterns, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py new file mode 100644 index 000000000..5582c82bb --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/platforms/youtube.py @@ -0,0 +1,131 @@ +"""YouTube scraper tools: videos and comments.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the YouTube video and comment tools.""" + + @mcp.tool( + name="surfsense_youtube_scrape", + title="Search or scrape YouTube", + annotations=SCRAPE, + structured_output=False, + ) + async def youtube_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="YouTube URLs: video, channel, playlist, shorts, " + "or hashtag pages. Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to search YouTube for, e.g. " + "['NotebookLM tutorial']. Provide search_queries OR urls." + ), + ] = None, + max_results: Annotated[ + int, Field(ge=1, description="Maximum videos to return.") + ] = 10, + download_subtitles: Annotated[ + bool, + Field(description="True also fetches each video's transcript."), + ] = False, + subtitles_language: Annotated[ + str, Field(description="Transcript language code, e.g. 'en'.") + ] = "en", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape YouTube videos, optionally with transcripts. + + Use this for YouTube research: finding videos on a topic, or reading a + video's details or transcript. For a video's comment section use + surfsense_youtube_comments instead. Returns per-video metadata (title, + channel, views, description, url) and, if requested, the transcript. + Example: search_queries=['NotebookLM tutorial'], download_subtitles=True. + """ + return await run_scraper( + client, + context, + platform="youtube", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "max_results": max_results, + "download_subtitles": download_subtitles, + "subtitles_language": subtitles_language, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_youtube_comments", + title="Fetch YouTube comments", + annotations=SCRAPE, + structured_output=False, + ) + async def youtube_comments( + urls: Annotated[ + list[str], + Field( + min_length=1, + description="YouTube video URLs, e.g. " + "['https://www.youtube.com/watch?v=abc123'].", + ), + ], + max_comments: Annotated[ + int, + Field( + ge=1, + description="Maximum comments per video, counting top-level " + "comments and replies together.", + ), + ] = 20, + sort_by: Annotated[ + CommentSort, Field(description="Comment ordering.") + ] = "NEWEST_FIRST", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch the comments (and replies) on one or more YouTube videos. + + Use this when the user wants a video's discussion or audience reaction + rather than the video itself; get video URLs from + surfsense_youtube_scrape if you only have a topic. Returns comment + text, author, likes, and replies. + Example: urls=['https://www.youtube.com/watch?v=abc123'], max_comments=50. + """ + return await run_scraper( + client, + context, + platform="youtube", + verb="comments", + payload={ + "urls": urls, + "max_comments": max_comments, + "sort_by": sort_by, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py new file mode 100644 index 000000000..9274a1a69 --- /dev/null +++ b/surfsense_mcp/src/surfsense_mcp/features/scrapers/run_history.py @@ -0,0 +1,112 @@ +"""Scraper run history: list past runs and fetch one in full. + +A scrape whose inline result was truncated is retrievable here by run id, so the +model never re-runs a scraper just to recover output. +""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ...core.client import SurfSenseClient +from ...core.rendering import ResponseFormatParam, clip, to_json +from ...core.workspace_context import WorkspaceContext, WorkspaceParam +from .annotations import READ_RUNS + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the run-history tools.""" + + @mcp.tool( + name="surfsense_list_scraper_runs", + title="List past scraper runs", + annotations=READ_RUNS, + structured_output=False, + ) + async def list_scraper_runs( + limit: Annotated[ + int, Field(ge=1, description="Maximum runs to list.") + ] = 20, + capability: Annotated[ + str | None, + Field( + description="Filter by capability slug, e.g. 'web.crawl' or " + "'reddit.scrape'." + ), + ] = None, + status: Annotated[ + str | None, + Field(description="Filter by run status: 'success' or 'error'."), + ] = None, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """List recent scraper runs in the workspace, newest first. + + Use this to find the run_id of an earlier scrape — for example when an + inline result was truncated — then fetch it in full with + surfsense_get_scraper_run. Returns each run's id, capability, status, + item count, and creation time. + Example: capability='reddit.scrape', status='success'. + """ + resolved = await context.resolve(workspace) + runs = await client.request( + "GET", + f"/workspaces/{resolved.id}/scrapers/runs", + params={ + "limit": limit, + "capability": capability, + "status": status, + }, + ) + if response_format == "json": + return to_json(runs) + return _render_runs(runs) + + @mcp.tool( + name="surfsense_get_scraper_run", + title="Fetch one scraper run in full", + annotations=READ_RUNS, + structured_output=False, + ) + async def get_scraper_run( + run_id: Annotated[ + str, + Field( + description="Run id from surfsense_list_scraper_runs or a " + "prior scrape's output." + ), + ], + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch a single scraper run in full, including its stored output. + + Use this to retrieve the complete, untruncated result of an earlier + scrape. Do NOT re-run a scraper just to recover a truncated result — + fetch the stored run instead. + """ + resolved = await context.resolve(workspace) + run = await client.request( + "GET", f"/workspaces/{resolved.id}/scrapers/runs/{run_id}" + ) + if response_format == "json": + return clip(to_json(run)) + return f"# Run {run.get('id', run_id)}\n\n```json\n{clip(to_json(run))}\n```" + + +def _render_runs(runs: list[dict] | None) -> str: + if not runs: + return "No scraper runs found." + lines = ["# Scraper runs", ""] + for run in runs: + lines.append( + f"- **{run.get('id')}** — {run.get('capability')} · {run.get('status')} · " + f"{run.get('item_count', 0)} item(s) · {run.get('created_at')}" + ) + return "\n".join(lines) From 2b9daead58dcd4f98108d779edd55c1dc19765b3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 18:05:56 +0200 Subject: [PATCH 014/160] feat(mcp): add health endpoint with public-path auth exemption --- .../src/surfsense_mcp/core/auth/middleware.py | 8 +++++-- .../src/surfsense_mcp/core/transport/http.py | 21 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py index 354ae0a89..8ba6ce939 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py +++ b/surfsense_mcp/src/surfsense_mcp/core/auth/middleware.py @@ -6,10 +6,13 @@ not reach the tool handler. A pure middleware binds the key in the request's own task, from which the SDK's per-request handling inherits it. Requests without a key are rejected here so no tool ever runs unauthenticated. +Paths in ``public_paths`` (e.g. the health probe) skip the check entirely. """ from __future__ import annotations +from collections.abc import Iterable + from starlette.datastructures import Headers from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send @@ -21,11 +24,12 @@ from .identity import bind_api_key, unbind_api_key class ApiKeyIdentityMiddleware: """Binds the per-request API key into the identity contextvar, or 401s.""" - def __init__(self, app: ASGIApp) -> None: + def __init__(self, app: ASGIApp, public_paths: Iterable[str] = ()) -> None: self._app = app + self._public_paths = frozenset(public_paths) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": + if scope["type"] != "http" or scope["path"] in self._public_paths: await self._app(scope, receive, send) return diff --git a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py index 2f2cf8f53..e375e8b98 100644 --- a/surfsense_mcp/src/surfsense_mcp/core/transport/http.py +++ b/surfsense_mcp/src/surfsense_mcp/core/transport/http.py @@ -1,23 +1,32 @@ -"""Assemble the streamable-http ASGI app for the remote transport. +"""Wrap the SDK's MCP endpoint with identity + CORS for the remote transport. -Wraps the SDK's MCP endpoint with the API-key identity middleware and CORS. -CORS sits outermost so browser preflight (which carries no key) is answered -before the identity middleware, and clients can read the ``Mcp-Session-Id`` -header the streamable-http protocol relies on. +CORS is outermost so keyless browser preflight is answered before the identity +middleware. ``/health`` is a public path, exempt from the key check. """ from __future__ import annotations from mcp.server.fastmcp import FastMCP +from starlette.requests import Request +from starlette.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware from starlette.types import ASGIApp from ..auth.middleware import ApiKeyIdentityMiddleware +HEALTH_PATH = "/health" + + +async def _health(_request: Request) -> JSONResponse: + return JSONResponse({"status": "ok"}) + def build_http_app(mcp: FastMCP) -> ASGIApp: """Return the MCP streamable-http app wrapped with identity + CORS.""" - app: ASGIApp = ApiKeyIdentityMiddleware(mcp.streamable_http_app()) + mcp.custom_route(HEALTH_PATH, methods=["GET"])(_health) + app: ASGIApp = ApiKeyIdentityMiddleware( + mcp.streamable_http_app(), public_paths={HEALTH_PATH} + ) return CORSMiddleware( app, allow_origins=["*"], From 9aef0c27a28181e34275e8b629f27d7ee94877f2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 18:06:00 +0200 Subject: [PATCH 015/160] docs(mcp): document hosted /mcp remote endpoint --- README.es.md | 2 +- README.hi.md | 2 +- README.md | 2 +- README.pt-BR.md | 2 +- README.zh-CN.md | 2 +- surfsense_mcp/README.md | 68 +++++++++++++------ .../content/docs/how-to/mcp-server.mdx | 63 +++++++++-------- 7 files changed, 88 insertions(+), 53 deletions(-) diff --git a/README.es.md b/README.es.md index 82cfcae1d..70ccd5eb1 100644 --- a/README.es.md +++ b/README.es.md @@ -138,7 +138,7 @@ Agrega el servidor MCP de SurfSense a Claude, Cursor o tu propio framework de ag { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.hi.md b/README.hi.md index 1c9cfa4a7..cbf9ae8fa 100644 --- a/README.hi.md +++ b/README.hi.md @@ -138,7 +138,7 @@ SurfSense MCP सर्वर को Claude, Cursor या अपने एज { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.md b/README.md index a05548d66..8f2e65118 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Add the SurfSense MCP server to Claude, Cursor, or your own agent framework: { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.pt-BR.md b/README.pt-BR.md index 4f2b945f4..076736bcf 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -138,7 +138,7 @@ Adicione o servidor MCP do SurfSense ao Claude, ao Cursor ou ao seu próprio fra { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/README.zh-CN.md b/README.zh-CN.md index 22410a727..7d2ae35c3 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -138,7 +138,7 @@ curl -X POST "$SURFSENSE_API_URL/workspaces/$WORKSPACE_ID/scrapers/reddit/scrape { "mcpServers": { "surfsense": { - "url": "https://mcp.surfsense.com", + "url": "https://mcp.surfsense.com/mcp", "headers": { "Authorization": "Bearer ${SURFSENSE_API_KEY}" } } } diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index 47dabf55c..cc5ef19c5 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -2,9 +2,15 @@ A [Model Context Protocol](https://modelcontextprotocol.io/) server that exposes SurfSense to MCP clients like **Claude Code**, **Cursor**, and **Claude Desktop**. -It talks to a running SurfSense backend purely over its REST API using a SurfSense -API key — it imports no backend code and can point at any instance (local or -hosted) by changing two environment variables. +It talks to a SurfSense backend purely over its REST API using a SurfSense API +key — it imports no backend code. + +Connect it two ways: + +- **Hosted** (recommended) — point your client at `https://mcp.surfsense.com/mcp` + and pass your API key in a header. Nothing to install or keep running. +- **Self-host (stdio)** — run the server yourself against any backend (cloud or + your own). Best for self-hosters and clients without remote-server support. ## Tools @@ -29,15 +35,42 @@ Workspace-scoped tools default to the active workspace; pass `workspace` (a name or id) to override for a single call. Ids never need to be typed by hand — the model carries them between calls. -## Prerequisites +## Get an API key -1. A running SurfSense backend (default `http://localhost:8000`). -2. A **SurfSense API key**: SurfSense → Settings → API → create key (`ss_pat_…`). -3. **API access enabled** on the workspace(s) you want to use (workspace settings). +1. SurfSense → **API Playground → API Keys**: create a personal key (`ss_pat_…`). + It is shown only once. +2. Toggle **API key access** on for the workspace(s) you want to use. -## Setup +## Connect (hosted) -Uses [uv](https://github.com/astral-sh/uv): +Point your client at the hosted server and send the key as a Bearer token. For +clients that read an `mcpServers` map (Cursor, Claude Desktop, and others): + +```json +{ + "mcpServers": { + "surfsense": { + "url": "https://mcp.surfsense.com/mcp", + "headers": { "Authorization": "Bearer ss_pat_your_key_here" } + } + } +} +``` + +Claude Code, from a terminal: + +```bash +claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \ + --header "Authorization: Bearer ss_pat_your_key_here" +``` + +Most MCP clients accept this `url` + `headers` form; check your client's docs for +its exact remote-server field. + +## Self-host (stdio) + +Run the server yourself when you host your own backend or use a client without +remote support. It uses [uv](https://github.com/astral-sh/uv): ```bash cd surfsense_mcp @@ -45,11 +78,8 @@ uv sync uv run python -m surfsense_mcp.selfcheck # verify tools register correctly ``` -## Connect it to a client - -### Cursor - -Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`): +Then add it to your client. Cursor (`~/.cursor/mcp.json` or a project +`.cursor/mcp.json`): ```json { @@ -66,7 +96,7 @@ Add to `~/.cursor/mcp.json` (or a project `.cursor/mcp.json`): } ``` -### Claude Code +Claude Code: ```bash claude mcp add surfsense \ @@ -75,15 +105,13 @@ claude mcp add surfsense \ -- uv run --directory /absolute/path/to/SurfSense/surfsense_mcp python -m surfsense_mcp ``` -### Claude Desktop - -Add the same `mcpServers` block as Cursor to +Claude Desktop: add the same `mcpServers` block as Cursor to `claude_desktop_config.json` (Settings → Developer → Edit Config). ## Configuration -See `.env.example`. Secrets are passed as environment variables by the client; -never commit tokens. +See `.env.example`. For self-host, secrets are passed as environment variables by +the client; never commit tokens. ## Backend dependency diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index ffdb4fd56..62628a659 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -4,51 +4,58 @@ description: Connect the SurfSense MCP server to Claude Code, Codex, OpenCode, C --- import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; -import { Step, Steps } from 'fumadocs-ui/components/steps'; # SurfSense MCP Server The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. -It talks to SurfSense purely over the REST API — point it at SurfSense Cloud or your own self-hosted instance by changing one environment variable. +Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. -## Prerequisites +## Create an API key - - +You need a SurfSense API key either way. In SurfSense, open **API Playground → API Keys** in your workspace sidebar: -### Install uv +1. Toggle **API key access** on for the workspace. +2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. -The server runs with [uv](https://github.com/astral-sh/uv). Install it once, then from the SurfSense repository run: +## Connect (hosted) + +The hosted server runs at `https://mcp.surfsense.com/mcp`. Point your client at it and send the key as a Bearer token — there is nothing to install and no backend to run. For clients that read an `mcpServers` map (Cursor, and others): + +```json +{ + "mcpServers": { + "surfsense": { + "url": "https://mcp.surfsense.com/mcp", + "headers": { "Authorization": "Bearer ss_pat_your_key_here" } + } + } +} +``` + +Claude Code, from a terminal: + +```bash +claude mcp add --transport http surfsense https://mcp.surfsense.com/mcp \ + --header "Authorization: Bearer ss_pat_your_key_here" +``` + +Most MCP clients accept this `url` + `headers` form; check your client's docs for its exact remote-server field. + +## Self-host (stdio) + +Run the server yourself when you host your own backend or use a client without remote support. It runs with [uv](https://github.com/astral-sh/uv) — install it once, then from the SurfSense repository run: ```bash cd surfsense_mcp uv sync ``` - - - -### Create an API key - -In SurfSense, open **API Playground → API Keys** in your workspace sidebar: - -1. Toggle **API key access** on for the workspace. -2. Create a personal API key (`ss_pat_…`) and copy it — it is shown only once. - - - - -### Know your base URL +Point the server at your backend with `SURFSENSE_BASE_URL`: - **SurfSense Cloud**: `https://api.surfsense.com` - **Self-hosted**: wherever your backend runs, e.g. `http://localhost:8000` - - - -## Connect your agent - Every client below launches the same command — `uv run --directory /surfsense_mcp python -m surfsense_mcp` — and passes `SURFSENSE_BASE_URL` and `SURFSENSE_API_KEY` as environment variables. Replace the placeholder paths and key with yours. @@ -221,7 +228,7 @@ Run `/mcp` inside Gemini CLI to confirm the server and its tools. -The server uses stdio transport: your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. +In this mode your client launches the process on demand and shuts it down with the session. There is no daemon to manage — only your SurfSense backend needs to be up. (The hosted server above needs none of this.) ## Test it @@ -236,7 +243,7 @@ That calls `surfsense_list_workspaces` — the simplest end-to-end check of the ## Configuration reference -All settings are environment variables passed by the client: +For self-host (stdio), all settings are environment variables passed by the client. The hosted server needs only your API key in the `Authorization` header: | Variable | Required | Default | Purpose | |----------|----------|---------|---------| From 1c5b08561d4505391f5e7ce624908f14c3fa9da3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Jul 2026 18:06:04 +0200 Subject: [PATCH 016/160] feat(web): add hosted and self-host MCP client configs --- surfsense_web/app/(home)/mcp-server/page.tsx | 21 +- .../connectors-marketing/api-mcp-tabs.tsx | 2 +- .../components/mcp/agent-setup-tabs.tsx | 102 +++-- surfsense_web/lib/mcp/clients.ts | 379 +++++++++++++----- 4 files changed, 350 insertions(+), 154 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 6dfc69622..5e4961b1e 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -50,16 +50,13 @@ export const metadata: Metadata = { }, }; -/* Mirrors surfsense_mcp/README.md: the real Cursor config. */ +/* The hosted Cursor config; mirrors lib/mcp/clients.ts. */ const CURSOR_CONFIG = `{ "mcpServers": { "surfsense": { - "command": "uv", - "args": ["run", "--directory", ".../surfsense_mcp", - "python", "-m", "surfsense_mcp"], - "env": { - "SURFSENSE_BASE_URL": "https://api.surfsense.com", - "SURFSENSE_API_KEY": "ss_pat_..." + "url": "https://mcp.surfsense.com/mcp", + "headers": { + "Authorization": "Bearer ss_pat_..." } } } @@ -76,7 +73,7 @@ const STEPS = [ icon: TerminalSquare, title: "Add the server to your client", description: - "Drop the config into Cursor's mcp.json, run claude mcp add for Claude Code, or paste it into Claude Desktop. Point it at the cloud or your own self-hosted instance.", + "Point your client at https://mcp.surfsense.com/mcp with your key in an Authorization header — the hosted config for Cursor, Claude Code, and others is one paste. Prefer stdio? Switch to Self-host and run it against your own backend.", }, { icon: Server, @@ -135,7 +132,7 @@ const FAQ: FaqItem[] = [ { question: "Which MCP clients does it work with?", answer: - "Any MCP client that supports stdio servers. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI are documented with copy-paste configs on this page, and the same command works in custom agent harnesses built on the MCP SDK.", + "Any MCP client that speaks remote (streamable HTTP) or stdio. Claude Code, Codex, OpenCode, Cursor, Claude Desktop, VS Code, Windsurf, and Gemini CLI all have copy-paste configs on this page — Hosted for the one-paste https://mcp.surfsense.com/mcp endpoint, or Self-host for stdio against your own backend.", }, { question: "How is usage billed?", @@ -278,9 +275,9 @@ export default function McpServerPage() { Step-by-step setup for every agent

- Pick your client, follow its two steps, and paste the config. Replace the placeholder - path with your surfsense_mcp checkout and the key with one from API Playground → API - Keys — or grab a pre-filled config from the playground itself. + Pick your client, choose Hosted or Self-host, and + paste the config. Replace the key with one from API Playground → API Keys — or grab a + pre-filled config from the playground itself.

diff --git a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx index 6ca19c47b..f25faf95b 100644 --- a/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx +++ b/surfsense_web/components/connectors-marketing/api-mcp-tabs.tsx @@ -206,7 +206,7 @@ function buildMcp({ mcpTool }: ApiSample): string { const config = { mcpServers: { surfsense: { - url: "https://mcp.surfsense.com", + url: "https://mcp.surfsense.com/mcp", headers: { Authorization: "Bearer ${SURFSENSE_API_KEY}" }, }, }, diff --git a/surfsense_web/components/mcp/agent-setup-tabs.tsx b/surfsense_web/components/mcp/agent-setup-tabs.tsx index 1701b8924..3790441f2 100644 --- a/surfsense_web/components/mcp/agent-setup-tabs.tsx +++ b/surfsense_web/components/mcp/agent-setup-tabs.tsx @@ -9,6 +9,8 @@ import { DEFAULT_SERVER_DIR, MCP_CLIENTS, type McpSnippetOptions, + type McpTransport, + REMOTE_URL, } from "@/lib/mcp/clients"; function CopyButton({ text }: { text: string }) { @@ -38,48 +40,82 @@ function CopyButton({ text }: { text: string }) { ); } +const TRANSPORTS: { id: McpTransport; label: string; hint: string }[] = [ + { id: "remote", label: "Hosted", hint: "mcp.surfsense.com — nothing to install" }, + { id: "stdio", label: "Self-host", hint: "run the server against your own backend" }, +]; + /** - * Per-agent MCP setup instructions as tabs: pick a client, follow its steps, - * copy its exact config. Used on the /mcp-server marketing page and in the - * API playground; `options` fills in real values where the caller has them. + * Per-agent MCP setup instructions as tabs: pick a client, then Hosted or + * Self-host, and copy its exact config. Used on the /mcp-server marketing page + * and in the API playground; `options` fills in real values where the caller + * has them. */ export function AgentSetupTabs({ options }: { options?: Partial }) { + const [transport, setTransport] = useState("remote"); + const resolved: McpSnippetOptions = { - baseUrl: options?.baseUrl || "https://api.surfsense.com", + remoteUrl: options?.remoteUrl || REMOTE_URL, apiKey: options?.apiKey || API_KEY_PLACEHOLDER, + baseUrl: options?.baseUrl || "https://api.surfsense.com", serverDir: options?.serverDir || DEFAULT_SERVER_DIR, }; + const active = TRANSPORTS.find((t) => t.id === transport) ?? TRANSPORTS[0]; + return ( - - - {MCP_CLIENTS.map((client) => ( - - {client.label} - - ))} - - {MCP_CLIENTS.map((client) => { - const config = client.buildConfig(resolved); - return ( - -
    - {client.steps.map((step) => ( -
  1. {step}
  2. - ))} -
-
-

{client.configFile}

-
- -
-									{config}
-								
+
+
+
+ {TRANSPORTS.map((t) => ( + + ))} +
+ {active.hint} +
+ + + + {MCP_CLIENTS.map((client) => ( + + {client.label} + + ))} + + {MCP_CLIENTS.map((client) => { + const snippet = client[transport]; + const config = snippet.build(resolved); + return ( + +
    + {snippet.steps.map((step) => ( +
  1. {step}
  2. + ))} +
+
+

+ {snippet.configFile} +

+
+ +
+										{config}
+									
+
-
- - ); - })} - + + ); + })} + +
); } diff --git a/surfsense_web/lib/mcp/clients.ts b/surfsense_web/lib/mcp/clients.ts index 923e99049..795832ad1 100644 --- a/surfsense_web/lib/mcp/clients.ts +++ b/surfsense_web/lib/mcp/clients.ts @@ -1,42 +1,75 @@ /** - * MCP client setup catalog: one entry per popular agent, with the exact - * config file, steps, and snippet needed to connect the SurfSense MCP server. - * Shared by the marketing /mcp-server page and the API playground so the - * instructions can never drift apart. + * MCP client setup catalog: one entry per popular agent, each with a hosted + * (remote) snippet and a self-host (stdio) snippet, plus the exact config file + * and steps. Shared by the marketing /mcp-server page and the API playground so + * the instructions can never drift apart. + * + * Remote snippets point at the hosted server and pass the key as a Bearer token; + * every client's exact remote field is verified against its own docs (Windsurf + * uses `serverUrl`, Gemini CLI `httpUrl`, VS Code needs `type: "http"`, OpenCode + * `type: "remote"` + `oauth: false`, Codex needs the rmcp flag, and Claude + * Desktop has no config-file remote support so it uses the `mcp-remote` bridge). */ +export type McpTransport = "remote" | "stdio"; + export interface McpSnippetOptions { - /** SurfSense backend URL the server should call. */ - baseUrl: string; + /** Hosted MCP endpoint (Bearer-authenticated). */ + remoteUrl: string; /** API key value or placeholder to show in the snippet. */ apiKey: string; - /** Absolute path to the surfsense_mcp directory. */ + /** SurfSense backend URL a self-hosted server should call. */ + baseUrl: string; + /** Absolute path to the surfsense_mcp directory (self-host). */ serverDir: string; } +export interface McpSnippet { + /** Where the snippet goes: a file path or "Terminal". */ + configFile: string; + language: "json" | "toml" | "bash"; + steps: string[]; + build: (options: McpSnippetOptions) => string; +} + export interface McpClient { id: string; label: string; - /** Where the snippet goes: a file path or "Terminal". */ - configFile: string; - language: "json" | "toml" | "bash"; - steps: string[]; - buildConfig: (options: McpSnippetOptions) => string; + remote: McpSnippet; + stdio: McpSnippet; } +export const REMOTE_URL = "https://mcp.surfsense.com/mcp"; export const DEFAULT_SERVER_DIR = "/path/to/SurfSense/surfsense_mcp"; export const API_KEY_PLACEHOLDER = "ss_pat_your_key_here"; -function serverArgs(serverDir: string): string[] { - return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; -} - function json(value: unknown): string { return JSON.stringify(value, null, 2); } -/** The `mcpServers` JSON shape shared by Cursor, Claude Desktop, Windsurf, and Gemini CLI. */ -function standardJson({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { +function bearer(apiKey: string): string { + return `Bearer ${apiKey}`; +} + +function serverArgs(serverDir: string): string[] { + return ["run", "--directory", serverDir, "python", "-m", "surfsense_mcp"]; +} + +/** The `mcpServers` remote shape shared by Cursor, Windsurf, and Gemini CLI. */ +function remoteMcpServers(urlField: "url" | "serverUrl" | "httpUrl") { + return ({ remoteUrl, apiKey }: McpSnippetOptions): string => + json({ + mcpServers: { + surfsense: { + [urlField]: remoteUrl, + headers: { Authorization: bearer(apiKey) }, + }, + }, + }); +} + +/** The `mcpServers` stdio shape shared by Cursor, Claude Desktop, Windsurf, Gemini CLI. */ +function stdioMcpServers({ baseUrl, apiKey, serverDir }: McpSnippetOptions): string { return json({ mcpServers: { surfsense: { @@ -52,125 +85,255 @@ export const MCP_CLIENTS: McpClient[] = [ { id: "claude-code", label: "Claude Code", - configFile: "Terminal", - language: "bash", - steps: [ - "Run this command in a terminal (any directory).", - "Start Claude Code and run /mcp — surfsense should be listed as connected.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - [ - "claude mcp add surfsense \\", - ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, - ` -e SURFSENSE_API_KEY=${apiKey} \\`, - ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, - ].join("\n"), + remote: { + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + build: ({ remoteUrl, apiKey }) => + [ + `claude mcp add --transport http surfsense ${remoteUrl} \\`, + ` --header "Authorization: ${bearer(apiKey)}"`, + ].join("\n"), + }, + stdio: { + configFile: "Terminal", + language: "bash", + steps: [ + "Run this command in a terminal (any directory).", + "Start Claude Code and run /mcp — surfsense should be listed as connected.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + [ + "claude mcp add surfsense \\", + ` -e SURFSENSE_BASE_URL=${baseUrl} \\`, + ` -e SURFSENSE_API_KEY=${apiKey} \\`, + ` -- uv run --directory ${serverDir} python -m surfsense_mcp`, + ].join("\n"), + }, }, { id: "codex", label: "Codex", - configFile: "~/.codex/config.toml", - language: "toml", - steps: [ - "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", - "Restart Codex; `codex mcp list` should show surfsense.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - [ - "[mcp_servers.surfsense]", - 'command = "uv"', - `args = ${JSON.stringify(serverArgs(serverDir))}`, - "", - "[mcp_servers.surfsense.env]", - `SURFSENSE_BASE_URL = "${baseUrl}"`, - `SURFSENSE_API_KEY = "${apiKey}"`, - ].join("\n"), + remote: { + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml. The rmcp flag must sit above every [mcp_servers.*] table.", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + build: ({ remoteUrl, apiKey }) => + [ + "experimental_use_rmcp_client = true", + "", + "[mcp_servers.surfsense]", + `url = "${remoteUrl}"`, + "", + "[mcp_servers.surfsense.http_headers]", + `Authorization = "${bearer(apiKey)}"`, + ].join("\n"), + }, + stdio: { + configFile: "~/.codex/config.toml", + language: "toml", + steps: [ + "Add this to ~/.codex/config.toml (or run `codex mcp add surfsense -- uv run --directory python -m surfsense_mcp`).", + "Restart Codex; `codex mcp list` should show surfsense.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + [ + "[mcp_servers.surfsense]", + 'command = "uv"', + `args = ${JSON.stringify(serverArgs(serverDir))}`, + "", + "[mcp_servers.surfsense.env]", + `SURFSENSE_BASE_URL = "${baseUrl}"`, + `SURFSENSE_API_KEY = "${apiKey}"`, + ].join("\n"), + }, }, { id: "opencode", label: "OpenCode", - configFile: "opencode.json", - language: "json", - steps: [ - "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", - "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - json({ - $schema: "https://opencode.ai/config.json", - mcp: { - surfsense: { - type: "local", - command: ["uv", ...serverArgs(serverDir)], - enabled: true, - environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + remote: { + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "`oauth: false` tells OpenCode to use the Bearer key instead of starting an OAuth flow.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "remote", + url: remoteUrl, + enabled: true, + oauth: false, + headers: { Authorization: bearer(apiKey) }, + }, }, - }, - }), + }), + }, + stdio: { + configFile: "opencode.json", + language: "json", + steps: [ + "Add this to opencode.json in your project root (or ~/.config/opencode/opencode.json for all projects).", + "Note OpenCode's format: the key is `mcp`, the command is one array, and env vars go under `environment`.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + json({ + $schema: "https://opencode.ai/config.json", + mcp: { + surfsense: { + type: "local", + command: ["uv", ...serverArgs(serverDir)], + enabled: true, + environment: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, }, { id: "cursor", label: "Cursor", - configFile: "~/.cursor/mcp.json", - language: "json", - steps: [ - "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", - "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + build: remoteMcpServers("url"), + }, + stdio: { + configFile: "~/.cursor/mcp.json", + language: "json", + steps: [ + "Add this to ~/.cursor/mcp.json (global, keeps the key out of your repo) or a project's .cursor/mcp.json.", + "Refresh the server in Cursor Settings → MCP; its 18 tools should appear.", + ], + build: stdioMcpServers, + }, }, { id: "claude-desktop", label: "Claude Desktop", - configFile: "claude_desktop_config.json", - language: "json", - steps: [ - "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", - "Restart Claude Desktop; surfsense appears under the tools icon.", - ], - buildConfig: standardJson, + remote: { + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Claude Desktop can't take a remote URL directly, so this uses the mcp-remote bridge (needs Node 18+).", + "Open Settings → Developer → Edit Config, add this, and restart Claude Desktop.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + mcpServers: { + surfsense: { + command: "npx", + args: ["-y", "mcp-remote", remoteUrl, "--header", `Authorization: ${bearer(apiKey)}`], + }, + }, + }), + }, + stdio: { + configFile: "claude_desktop_config.json", + language: "json", + steps: [ + "Open Settings → Developer → Edit Config to reach claude_desktop_config.json and add this.", + "Restart Claude Desktop; surfsense appears under the tools icon.", + ], + build: stdioMcpServers, + }, }, { id: "vscode", label: "VS Code", - configFile: ".vscode/mcp.json", - language: "json", - steps: [ - "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", - "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", - ], - buildConfig: ({ baseUrl, apiKey, serverDir }) => - json({ - servers: { - surfsense: { - type: "stdio", - command: "uv", - args: serverArgs(serverDir), - env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + remote: { + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "VS Code requires an explicit `type` field — `http` for the hosted server.", + ], + build: ({ remoteUrl, apiKey }) => + json({ + servers: { + surfsense: { + type: "http", + url: remoteUrl, + headers: { Authorization: bearer(apiKey) }, + }, }, - }, - }), + }), + }, + stdio: { + configFile: ".vscode/mcp.json", + language: "json", + steps: [ + "Add this to .vscode/mcp.json in your workspace (or run the MCP: Add Server command).", + "Open Copilot Chat in agent mode and click the tools icon to confirm surfsense is loaded.", + ], + build: ({ baseUrl, apiKey, serverDir }) => + json({ + servers: { + surfsense: { + type: "stdio", + command: "uv", + args: serverArgs(serverDir), + env: { SURFSENSE_BASE_URL: baseUrl, SURFSENSE_API_KEY: apiKey }, + }, + }, + }), + }, }, { id: "windsurf", label: "Windsurf", - configFile: "~/.codeium/windsurf/mcp_config.json", - language: "json", - steps: [ - "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", - "Press the refresh button in the MCP panel to pick up the server.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Windsurf uses `serverUrl` (not `url`) for remote servers; press refresh in the MCP panel.", + ], + build: remoteMcpServers("serverUrl"), + }, + stdio: { + configFile: "~/.codeium/windsurf/mcp_config.json", + language: "json", + steps: [ + "Add this to ~/.codeium/windsurf/mcp_config.json (or Windsurf Settings → Cascade → MCP Servers).", + "Press the refresh button in the MCP panel to pick up the server.", + ], + build: stdioMcpServers, + }, }, { id: "gemini-cli", label: "Gemini CLI", - configFile: "~/.gemini/settings.json", - language: "json", - steps: [ - "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", - "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", - ], - buildConfig: standardJson, + remote: { + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Gemini CLI uses `httpUrl` for streamable-HTTP servers; run /mcp to confirm surfsense.", + ], + build: remoteMcpServers("httpUrl"), + }, + stdio: { + configFile: "~/.gemini/settings.json", + language: "json", + steps: [ + "Add this to ~/.gemini/settings.json (or .gemini/settings.json in a project).", + "Run /mcp inside Gemini CLI to confirm the surfsense server and its tools.", + ], + build: stdioMcpServers, + }, }, ]; From ef3f9f2e25ea9661351adb712e2d4f7588af462a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:39:58 +0200 Subject: [PATCH 017/160] feat(tiktok): item output schemas --- .../proprietary/platforms/tiktok/__init__.py | 0 .../platforms/tiktok/schemas/__init__.py | 21 +++ .../platforms/tiktok/schemas/items.py | 132 ++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py new file mode 100644 index 000000000..28326a544 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -0,0 +1,21 @@ +"""Apify-compatible input and output contracts for the TikTok scraper.""" + +from __future__ import annotations + +from .items import ( + AuthorMeta, + CommentItem, + ErrorItem, + MusicMeta, + TikTokVideoItem, + VideoMeta, +) + +__all__ = [ + "AuthorMeta", + "CommentItem", + "ErrorItem", + "MusicMeta", + "TikTokVideoItem", + "VideoMeta", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py new file mode 100644 index 000000000..c4ff87618 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py @@ -0,0 +1,132 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Output items keyed to the Clockworks TikTok actor's shape. + +Every model is open (``extra="allow"``) and defaults unsourced fields to +``None``/``[]`` so the MVP can populate a reliable subset and expand the +contract additively without breaking consumers. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class AuthorMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + id: str | None = None + name: str | None = None + nickName: str | None = None + profileUrl: str | None = None + verified: bool | None = None + signature: str | None = None + avatar: str | None = None + privateAccount: bool | None = None + fans: int | None = None + following: int | None = None + heart: int | None = None + video: int | None = None + + +class MusicMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + musicId: str | None = None + musicName: str | None = None + musicAuthor: str | None = None + musicOriginal: bool | None = None + playUrl: str | None = None + + +class VideoMeta(BaseModel): + model_config = ConfigDict(extra="allow") + + height: int | None = None + width: int | None = None + duration: int | None = None + coverUrl: str | None = None + format: str | None = None + definition: str | None = None + + +class TikTokVideoItem(BaseModel): + """A single scraped post (video or slideshow).""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + text: str | None = None + textLanguage: str | None = None + createTime: int | None = None + createTimeISO: str | None = None + isAd: bool | None = None + + authorMeta: AuthorMeta = Field(default_factory=AuthorMeta) + musicMeta: MusicMeta = Field(default_factory=MusicMeta) + videoMeta: VideoMeta = Field(default_factory=VideoMeta) + + webVideoUrl: str | None = None + mediaUrls: list[str] = Field(default_factory=list) + + diggCount: int | None = None + shareCount: int | None = None + playCount: int | None = None + collectCount: int | None = None + commentCount: int | None = None + + hashtags: list[dict[str, Any]] = Field(default_factory=list) + mentions: list[str] = Field(default_factory=list) + + isSlideshow: bool | None = None + isPinned: bool | None = None + isSponsored: bool | None = None + + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class CommentItem(BaseModel): + """A single comment or reply under a post.""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + text: str | None = None + videoWebUrl: str | None = None + diggCount: int | None = None + replyCommentTotal: int | None = None + createTime: int | None = None + createTimeISO: str | None = None + + uid: str | None = None + uniqueId: str | None = None + nickName: str | None = None + avatar: str | None = None + + repliesToId: str | None = None + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + +class ErrorItem(BaseModel): + """Per-input failure, distinguished from normal items by ``errorCode``. + + Mirrors the actor's convention so a private/deleted/empty target surfaces + as an item instead of silently vanishing from the results. + """ + + model_config = ConfigDict(extra="allow") + + url: str | None = None + input: str | None = None + error: str | None = None + errorCode: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) From b2d387f994d415a37383d6d2b1e4ff0755e13eae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:07 +0200 Subject: [PATCH 018/160] feat(tiktok): URL target resolver --- .../platforms/tiktok/targets/__init__.py | 8 +++ .../platforms/tiktok/targets/resolver.py | 50 +++++++++++++++++++ .../platforms/tiktok/targets/types.py | 25 ++++++++++ .../tests/unit/platforms/tiktok/__init__.py | 0 .../platforms/tiktok/test_target_resolver.py | 47 +++++++++++++++++ 5 files changed, 130 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py new file mode 100644 index 000000000..ac7410154 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/__init__.py @@ -0,0 +1,8 @@ +"""TikTok URL classification into scrape targets.""" + +from __future__ import annotations + +from .resolver import resolve_target +from .types import TargetKind, TikTokTarget + +__all__ = ["TargetKind", "TikTokTarget", "resolve_target"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py new file mode 100644 index 000000000..ffd15ba2a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/resolver.py @@ -0,0 +1,50 @@ +"""Classify a TikTok URL into a :class:`TikTokTarget`, or ``None``.""" + +from __future__ import annotations + +from urllib.parse import parse_qs, unquote, urlparse + +from .types import SearchSection, TikTokTarget + +_TIKTOK_HOSTS = frozenset({"tiktok.com", "www.tiktok.com", "m.tiktok.com"}) +_SEARCH_SECTIONS: frozenset[SearchSection] = frozenset({"video", "user"}) + + +def _is_tiktok_host(hostname: str | None) -> bool: + return bool(hostname) and hostname.lower() in _TIKTOK_HOSTS + + +def resolve_target(url: str) -> TikTokTarget | None: + parsed = urlparse(url) + if not _is_tiktok_host(parsed.hostname): + return None + + segments = [s for s in (parsed.path or "").split("/") if s] + if not segments: + return None + + # Profile / video live under /@username[...]. + if segments[0].startswith("@"): + username = segments[0][1:] + if not username: + return None + if len(segments) >= 3 and segments[1] == "video" and segments[2]: + return TikTokTarget("video", segments[2], url, username=username) + return TikTokTarget("profile", username, url) + + if segments[0] == "tag" and len(segments) >= 2 and segments[1]: + return TikTokTarget("hashtag", unquote(segments[1]), url) + + if segments[0] == "search": + query = parse_qs(parsed.query).get("q", [None])[0] + if not query: + return None + section = segments[1] if len(segments) >= 2 else None + return TikTokTarget( + "search", + unquote(query), + url, + section=section if section in _SEARCH_SECTIONS else None, + ) + + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py new file mode 100644 index 000000000..c53d5239d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py @@ -0,0 +1,25 @@ +"""Scrape-target value object produced by URL classification.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +TargetKind = Literal["video", "profile", "hashtag", "search"] +SearchSection = Literal["video", "user"] + + +@dataclass(frozen=True, slots=True) +class TikTokTarget: + """One classified scrape target. + + ``value`` holds the kind-specific identifier: video id, username, hashtag + name, or search query. ``username`` is set for videos (needed to build the + canonical post URL). ``section`` narrows a search to videos or users. + """ + + kind: TargetKind + value: str + url: str + username: str | None = None + section: SearchSection | None = None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/__init__.py b/surfsense_backend/tests/unit/platforms/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py new file mode 100644 index 000000000..c118f4f86 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_target_resolver.py @@ -0,0 +1,47 @@ +"""URL classification for the TikTok scraper (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.targets import resolve_target + + +def test_resolve_video_carries_username_and_id(): + target = resolve_target("https://www.tiktok.com/@scout2015/video/6718335390845095173") + assert target is not None + assert target.kind == "video" + assert target.value == "6718335390845095173" + assert target.username == "scout2015" + + +def test_resolve_profile(): + target = resolve_target("https://www.tiktok.com/@scout2015") + assert target is not None + assert target.kind == "profile" + assert target.value == "scout2015" + + +def test_resolve_hashtag(): + target = resolve_target("https://www.tiktok.com/tag/funny") + assert target is not None + assert target.kind == "hashtag" + assert target.value == "funny" + + +def test_resolve_search_top_video_and_user_sections(): + top = resolve_target("https://www.tiktok.com/search?q=cats") + assert top is not None + assert top.kind == "search" + assert top.value == "cats" + assert top.section is None + + videos = resolve_target("https://www.tiktok.com/search/video?q=cats") + assert videos is not None and videos.section == "video" + + users = resolve_target("https://www.tiktok.com/search/user?q=cats") + assert users is not None and users.section == "user" + + +def test_resolve_rejects_non_tiktok_and_unknown_paths(): + assert resolve_target("https://example.com/@scout2015") is None + assert resolve_target("https://www.tiktok.com/") is None + assert resolve_target("https://www.tiktok.com/foundation") is None From 8840fd2c53f1dfd10985069723f152898dbbbfb6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:21 +0200 Subject: [PATCH 019/160] feat(tiktok): rehydration blob extractor --- .../platforms/tiktok/extraction/__init__.py | 7 ++++ .../platforms/tiktok/extraction/hydration.py | 28 ++++++++++++++++ .../unit/platforms/tiktok/test_hydration.py | 32 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py new file mode 100644 index 000000000..3bd6b160d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -0,0 +1,7 @@ +"""Turn raw TikTok page/API payloads into normalized items.""" + +from __future__ import annotations + +from .hydration import extract_rehydration_data + +__all__ = ["extract_rehydration_data"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py new file mode 100644 index 000000000..4cc466849 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/hydration.py @@ -0,0 +1,28 @@ +"""Extract the ``__UNIVERSAL_DATA_FOR_REHYDRATION__`` JSON embedded in page HTML. + +TikTok server-renders the first page of data into a single script tag; parsing +it yields page-one items (video/profile/hashtag) without any signed API call. +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_BLOB_RE = re.compile( + r']*id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)', + re.DOTALL, +) + + +def extract_rehydration_data(html: str) -> dict[str, Any] | None: + """Return the parsed rehydration blob, or ``None`` if absent/unparseable.""" + match = _BLOB_RE.search(html) + if not match: + return None + try: + data = json.loads(match.group(1)) + except (json.JSONDecodeError, ValueError): + return None + return data if isinstance(data, dict) else None diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py new file mode 100644 index 000000000..cdca568f7 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_hydration.py @@ -0,0 +1,32 @@ +"""Rehydration-blob extraction from TikTok page HTML (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import extract_rehydration_data + +_BLOB = ( + '" +) + + +def test_extracts_default_scope_from_blob(): + data = extract_rehydration_data(_BLOB) + assert data is not None + item = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"] + assert item["id"] == "123" + + +def test_returns_none_when_blob_absent(): + assert extract_rehydration_data("no blob here") is None + + +def test_returns_none_when_blob_json_malformed(): + broken = ( + '' + ) + assert extract_rehydration_data(broken) is None From 0bbcf99852c20cb19c03a621149a304dfd2543c0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:38 +0200 Subject: [PATCH 020/160] feat(tiktok): author normalizer --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/author.py | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 3bd6b160d..fa4df8731 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from .author import parse_author from .hydration import extract_rehydration_data -__all__ = ["extract_rehydration_data"] +__all__ = ["extract_rehydration_data", "parse_author"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py new file mode 100644 index 000000000..414de39b6 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py @@ -0,0 +1,31 @@ +"""Normalize TikTok author/profile payloads into an ``authorMeta`` dict.""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def build_author_meta(author: dict[str, Any], stats: dict[str, Any]) -> dict[str, Any]: + """Map an author object + its stats to the ``authorMeta`` output shape.""" + username = author.get("uniqueId") + return { + "id": author.get("id"), + "name": username, + "nickName": author.get("nickname"), + "profileUrl": _PROFILE_URL.format(username=username) if username else None, + "verified": author.get("verified"), + "signature": author.get("signature"), + "avatar": author.get("avatarLarger") or author.get("avatarMedium"), + "privateAccount": author.get("privateAccount"), + "fans": stats.get("followerCount"), + "following": stats.get("followingCount"), + "heart": stats.get("heartCount"), + "video": stats.get("videoCount"), + } + + +def parse_author(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a ``webapp.user-detail`` ``userInfo`` (``{user, stats}``) to authorMeta.""" + return build_author_meta(user_info.get("user") or {}, user_info.get("stats") or {}) From ef66063bfc7e43bd329450fadca7e969df5aaf13 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:40:50 +0200 Subject: [PATCH 021/160] feat(tiktok): video normalizer and parser tests --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/timestamps.py | 18 ++++ .../platforms/tiktok/extraction/video.py | 73 +++++++++++++ .../unit/platforms/tiktok/test_parsers.py | 102 ++++++++++++++++++ 4 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index fa4df8731..9b4f71c68 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,5 +4,6 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .video import parse_video -__all__ = ["extract_rehydration_data", "parse_author"] +__all__ = ["extract_rehydration_data", "parse_author", "parse_video"] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py new file mode 100644 index 000000000..4b155c137 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/timestamps.py @@ -0,0 +1,18 @@ +"""Millisecond-ISO timestamps matching the actor's output shape.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def epoch_to_iso(seconds: int | None) -> str | None: + """Convert a Unix-seconds timestamp to ``YYYY-MM-DDTHH:MM:SS.000Z``.""" + if not seconds: + return None + stamp = datetime.fromtimestamp(seconds, tz=UTC) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def now_iso() -> str: + """Current UTC time in the millisecond-ISO output shape.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py new file mode 100644 index 000000000..1bfe66452 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/video.py @@ -0,0 +1,73 @@ +"""Normalize a raw TikTok ``itemStruct`` into a :class:`TikTokVideoItem` dict.""" + +from __future__ import annotations + +from typing import Any + +from ..schemas.items import TikTokVideoItem +from .author import build_author_meta +from .timestamps import epoch_to_iso + +_VIDEO_URL = "https://www.tiktok.com/@{username}/video/{video_id}" + + +def _music_meta(music: dict[str, Any]) -> dict[str, Any]: + return { + "musicId": music.get("id"), + "musicName": music.get("title"), + "musicAuthor": music.get("authorName"), + "musicOriginal": music.get("original"), + "playUrl": music.get("playUrl"), + } + + +def _video_meta(video: dict[str, Any]) -> dict[str, Any]: + return { + "height": video.get("height"), + "width": video.get("width"), + "duration": video.get("duration"), + "coverUrl": video.get("cover"), + "format": video.get("format"), + "definition": video.get("definition"), + } + + +def _hashtags(challenges: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {"id": c.get("id"), "name": c.get("title")} + for c in challenges + if isinstance(c, dict) + ] + + +def parse_video(item: dict[str, Any]) -> dict[str, Any]: + """Map an ``itemStruct`` to the flat output contract, filling known fields.""" + author = item.get("author") or {} + author_stats = item.get("authorStats") or {} + stats = item.get("stats") or {} + username = author.get("uniqueId") + video_id = item.get("id") + + web_url = ( + _VIDEO_URL.format(username=username, video_id=video_id) + if username and video_id + else None + ) + create_time = item.get("createTime") + + return TikTokVideoItem( + id=video_id, + text=item.get("desc"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + authorMeta=build_author_meta(author, author_stats), + musicMeta=_music_meta(item.get("music") or {}), + videoMeta=_video_meta(item.get("video") or {}), + webVideoUrl=web_url, + diggCount=stats.get("diggCount"), + shareCount=stats.get("shareCount"), + playCount=stats.get("playCount"), + collectCount=stats.get("collectCount"), + commentCount=stats.get("commentCount"), + hashtags=_hashtags(item.get("challenges") or []), + ).to_output() diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py new file mode 100644 index 000000000..03fa63133 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_parsers.py @@ -0,0 +1,102 @@ +"""Raw TikTok payload -> normalized item mapping (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import parse_author, parse_video + +_ITEM_STRUCT = { + "id": "7534061113365859586", + "desc": "haha #comeramabanana", + "createTime": 1_700_000_000, + "author": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": False, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + }, + "authorStats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, + "stats": { + "diggCount": 5344, + "shareCount": 701, + "playCount": 55700, + "commentCount": 24, + "collectCount": 291, + }, + "music": { + "id": "7529", + "title": "som original", + "authorName": "fox_rus0", + "original": True, + "playUrl": "https://cdn/music.mp3", + }, + "video": { + "height": 1024, + "width": 576, + "duration": 16, + "cover": "https://cdn/cover.jpg", + "format": "mp4", + "definition": "540p", + }, + "challenges": [{"id": "4982299", "title": "comeramabanana"}], +} + + +def test_parse_video_maps_core_and_derived_fields(): + item = parse_video(_ITEM_STRUCT) + + assert item["id"] == "7534061113365859586" + assert item["text"] == "haha #comeramabanana" + assert item["createTimeISO"] == "2023-11-14T22:13:20.000Z" + + assert item["authorMeta"]["name"] == "bruniela_" + assert item["authorMeta"]["nickName"] == "Bruni" + assert item["authorMeta"]["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert item["authorMeta"]["fans"] == 51200 + + assert item["musicMeta"]["musicName"] == "som original" + assert item["videoMeta"]["duration"] == 16 + + assert item["diggCount"] == 5344 + assert item["playCount"] == 55700 + + assert item["hashtags"] == [{"id": "4982299", "name": "comeramabanana"}] + assert ( + item["webVideoUrl"] + == "https://www.tiktok.com/@bruniela_/video/7534061113365859586" + ) + + +_USER_INFO = { + "user": { + "id": "6733", + "uniqueId": "bruniela_", + "nickname": "Bruni", + "verified": True, + "signature": "bio here", + "avatarLarger": "https://cdn/avatar.jpg", + "privateAccount": False, + }, + "stats": { + "followerCount": 51200, + "followingCount": 269, + "heartCount": 3_000_000, + "videoCount": 259, + }, +} + + +def test_parse_author_maps_user_and_stats(): + author = parse_author(_USER_INFO) + assert author["name"] == "bruniela_" + assert author["nickName"] == "Bruni" + assert author["verified"] is True + assert author["profileUrl"] == "https://www.tiktok.com/@bruniela_" + assert author["fans"] == 51200 + assert author["video"] == 259 From 5688ab0678b6571bd57ab98f20470270b2c8d8a2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:51:17 +0200 Subject: [PATCH 022/160] feat(tiktok): scrape input schema --- .../platforms/tiktok/schemas/__init__.py | 3 + .../platforms/tiktok/schemas/input.py | 60 +++++++++++++++++++ .../tests/unit/platforms/tiktok/test_input.py | 26 ++++++++ 3 files changed, 89 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_input.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py index 28326a544..3593f0e28 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from .input import StartUrl, TikTokScrapeInput from .items import ( AuthorMeta, CommentItem, @@ -16,6 +17,8 @@ __all__ = [ "CommentItem", "ErrorItem", "MusicMeta", + "StartUrl", + "TikTokScrapeInput", "TikTokVideoItem", "VideoMeta", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py new file mode 100644 index 000000000..fa87772fe --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -0,0 +1,60 @@ +# ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API +"""Input surface for the TikTok scraper, shaped to the Clockworks actor. + +Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first +path does not yet act on (media downloads, follower add-ons) are still accepted +via ``extra="allow"`` for contract parity and land inert. + +Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is +caller policy applied by the collector, never baked into the flows. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +ProfileSorting = Literal["latest", "popular", "oldest"] +ProfileSection = Literal["videos", "reposts"] +SearchSection = Literal["", "/video", "/user"] + + +class StartUrl(BaseModel): + """A single direct URL entry (``{"url": ...}``; extra keys ignored).""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class TikTokScrapeInput(BaseModel): + model_config = ConfigDict(extra="allow") + + # Discovery + startUrls: list[StartUrl] = Field(default_factory=list) + hashtags: list[str] = Field(default_factory=list) + profiles: list[str] = Field(default_factory=list) + searchQueries: list[str] = Field(default_factory=list) + postURLs: list[str] = Field(default_factory=list) + + # Per-target count + resultsPerPage: int = Field(default=1, ge=1) + + # Profile options + profileScrapeSections: list[ProfileSection] = Field( + default_factory=lambda: ["videos"] + ) + profileSorting: ProfileSorting = "latest" + excludePinnedPosts: bool = False + + # Search options + searchSection: SearchSection = "" + maxProfilesPerQuery: int = Field(default=10, ge=1) + + # Incremental filters (ISO date or relative " days" per the actor) + oldestPostDateUnified: str | None = None + newestPostDate: str | None = None + + # Proxy + proxyCountryCode: str = "None" diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_input.py b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py new file mode 100644 index 000000000..c080ffec3 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_input.py @@ -0,0 +1,26 @@ +"""Input surface for the TikTok scraper (anonymous, Apify-shaped).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.schemas import TikTokScrapeInput + + +def test_input_has_no_auth_fields(): + forbidden = {"username", "password", "token", "login", "auth", "credentials"} + assert forbidden.isdisjoint(TikTokScrapeInput.model_fields) + + +def test_input_defaults(): + model = TikTokScrapeInput() + assert model.resultsPerPage == 1 + assert model.profileSorting == "latest" + assert model.proxyCountryCode == "None" + assert model.hashtags == [] + assert model.profiles == [] + assert model.searchQueries == [] + assert model.postURLs == [] + + +def test_input_allows_extra_inert_fields(): + model = TikTokScrapeInput(shouldDownloadVideos=True, videoKvStoreIdOrName="x") + assert model.model_dump().get("shouldDownloadVideos") is True From 44b3e640d36c72423433881925f77c1abea92569 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 15:51:17 +0200 Subject: [PATCH 023/160] feat(tiktok): rotate-on-block fetch session --- .../platforms/tiktok/session/__init__.py | 15 ++ .../platforms/tiktok/session/client.py | 130 +++++++++++++++++ .../platforms/tiktok/session/errors.py | 11 ++ .../platforms/tiktok/session/proxy.py | 113 +++++++++++++++ .../platforms/tiktok/test_fetch_resilience.py | 135 ++++++++++++++++++ 5 files changed, 404 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/client.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py new file mode 100644 index 000000000..659305055 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -0,0 +1,15 @@ +"""Cookie-warmed, rotate-on-block proxy session and page-fetch seam.""" + +from __future__ import annotations + +from .client import fetch_html +from .errors import TikTokAccessBlockedError +from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session + +__all__ = [ + "TikTokAccessBlockedError", + "bind_proxy_holder", + "fetch_html", + "open_proxy_holder", + "proxy_session", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py new file mode 100644 index 000000000..e5be1e419 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/client.py @@ -0,0 +1,130 @@ +"""GET TikTok page HTML through a cookie-warmed, sticky-IP proxy session. + +The warm-up mints TikTok's anonymous device cookie (``ttwid``) on the first +homepage hit; the target page then server-renders its rehydration blob. Rotates +the residential IP and re-warms on 403, backs off on 429, and raises +:class:`TikTokAccessBlockedError` only when every rotated IP refuses access. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +from contextlib import suppress +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_proxy_url + +from .errors import TikTokAccessBlockedError +from .proxy import _REQUEST_TIMEOUT_S, _current_session, proxy_session + +logger = logging.getLogger(__name__) + +# 403 => IP blocked; rotate and re-warm. 429 => rate limited; back off same IP. +_ROTATE_STATUS = 403 +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +_HOME_URL = "https://www.tiktok.com/" +_TTWID_COOKIE = "ttwid" +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + + +def _response_cookie_names(page: Any) -> set[str]: + cookies = getattr(page, "cookies", None) + return set(cookies.keys()) if isinstance(cookies, dict) else set() + + +def _page_html(page: Any) -> str | None: + for attr in ("text", "body"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + return val + return None + + +async def warm_session(session: Any) -> bool: + """Mint an anonymous ``ttwid`` cookie; ``True`` if the session can now fetch.""" + with suppress(Exception): + page = await session.get(_HOME_URL, headers=_HEADERS) + if _TTWID_COOKIE in _response_cookie_names(page): + return True + return False + + +async def _get_page(session: Any, url: str) -> Any: + if session is not None: + return await session.get(url, headers=_HEADERS) + return await AsyncFetcher.get( + url, + headers=_HEADERS, + proxy=get_proxy_url(), + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + + +async def fetch_html(url: str) -> str | None: + """Return page HTML, or ``None`` on 404 / non-block failure.""" + holder = _current_session.get() + if holder is None: + async with proxy_session(): + return await fetch_html(url) + + attempt = 0 + backoffs = 0 + while True: + session = holder.session + try: + if session is not None and not holder.warmed: + warmed_ok = await warm_session(session) + holder.warmed = True + if not warmed_ok: + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + raise TikTokAccessBlockedError( + f"could not warm session after {attempt} IP rotations: {url}" + ) + + await holder.pace() + page = await _get_page(session, url) + status = page.status + + if status == 200: + return _page_html(page) + if status == 404: + return None + if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: + backoffs += 1 + delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1)) + logger.warning("[tiktok] 429 on %s; backing off %.1fs", url, delay) + await asyncio.sleep(delay + random.uniform(0, 1)) + continue + if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + if status == _ROTATE_STATUS: + raise TikTokAccessBlockedError( + f"TikTok refused {url} on {attempt} rotated IPs (403)" + ) + logger.warning("[tiktok] GET %s returned %s", url, status) + return None + except TikTokAccessBlockedError: + raise + except Exception as e: + logger.warning("[tiktok] GET %s failed: %s", url, e) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + return None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py new file mode 100644 index 000000000..4d9fc712d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -0,0 +1,11 @@ +"""Fetch-seam errors surfaced to the capability layer.""" + +from __future__ import annotations + + +class TikTokAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather + than returning empty data. The route maps it to a 403. + """ diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py new file mode 100644 index 000000000..dca6c776b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/proxy.py @@ -0,0 +1,113 @@ +"""Rotate-on-block sticky proxy session, bound per-flow via a ContextVar. + +Reusing one keep-alive connection pins a single residential exit IP so the +warmed cookie jar (``ttwid``/``msToken``, bound to that IP) stays valid across +the warm-up and every subsequent fetch. Ported from the Reddit sibling; the +TikTok-specific warm-up lives in :mod:`client`. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from typing import Any + +from scrapling.fetchers import FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + +# Pace each sticky IP so a fast exit can't burst past TikTok's per-IP threshold. +_MIN_INTERVAL_S = 0.5 +_PACE_JITTER_S = 0.25 +# A healthy fetch lands in ~1-2s; cap a dead IP at one bounded wait before it +# falls through to a rotation. +_REQUEST_TIMEOUT_S = 15.0 + +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "tiktok_proxy_session", default=None +) + + +class _RotatingSession: + """Owns one live ``FetcherSession`` (sticky IP); ``rotate()`` swaps the IP. + + Used sequentially within a single flow (never shared across concurrent + tasks), so no locking is needed. ``session`` is ``None`` only when no proxy + is configured. + """ + + def __init__(self) -> None: + self._cm: Any | None = None + self.session: Any | None = None + self.rotations = 0 + self.warmed = False + self._last_at = 0.0 + + async def _open(self) -> None: + proxy = get_proxy_url() + self.warmed = False + if proxy is None: + self._cm = self.session = None + return + self._cm = FetcherSession( + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, + ) + self.session = await self._cm.__aenter__() + + async def close(self) -> None: + if self._cm is not None: + with suppress(Exception): + await self._cm.__aexit__(None, None, None) + self._cm = self.session = None + + async def rotate(self) -> Any | None: + """Drop the current IP and connect through a fresh one.""" + await self.close() + self.rotations += 1 + await self._open() + logger.info("[tiktok] rotated proxy session (rotation #%d)", self.rotations) + return self.session + + async def pace(self) -> None: + """Sleep to hold this sticky IP under TikTok's per-IP rate threshold.""" + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at) + if wait > 0: + await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S)) + self._last_at = time.monotonic() + + +async def open_proxy_holder() -> _RotatingSession: + """Open a warm rotate-on-block session holder (caller owns ``close()``).""" + holder = _RotatingSession() + await holder._open() + return holder + + +@asynccontextmanager +async def bind_proxy_holder(holder: _RotatingSession): + """Route this task's fetches through ``holder`` for the enclosed block.""" + token = _current_session.set(holder) + try: + yield holder + finally: + _current_session.reset(token) + + +@asynccontextmanager +async def proxy_session(): + """Open one reused, rotate-on-block proxy session for a continuation chain.""" + holder = await open_proxy_holder() + try: + async with bind_proxy_holder(holder): + yield holder + finally: + await holder.close() diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py new file mode 100644 index 000000000..484157964 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_fetch_resilience.py @@ -0,0 +1,135 @@ +"""Fetch-seam resilience for the TikTok scraper (no network, fake sessions). + +Fake sessions drive the cookie warm-up + rotate-on-block + backoff branches +deterministically; a live first IP normally warms and returns 200s. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import ( + TikTokAccessBlockedError, + client, +) +from app.proprietary.platforms.tiktok.session.proxy import _current_session + +_HTML = "ok" + + +class _FakePage: + def __init__(self, status: int, *, cookies: dict | None = None, body: str = _HTML): + self.status = status + self.cookies = cookies or {} + self.body = body + + @property + def text(self) -> str: + return self.body + + +class _FakeSession: + """One 'IP': homepage warm mints ``ttwid`` per flag; page GETs return ``status``.""" + + def __init__(self, status: int = 200, *, warms: bool = True, body: str = _HTML): + self.status = status + self.warms = warms + self.body = body + self.page_calls = 0 + self.warm_calls = 0 + + async def get(self, url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.tiktok.com": + self.warm_calls += 1 + return _FakePage(200, cookies={"ttwid": "x"} if self.warms else {}) + self.page_calls += 1 + return _FakePage(self.status, body=self.body) + + +class _FakeHolder: + def __init__(self, sessions: list[_FakeSession]) -> None: + self._sessions = sessions + self.session = sessions[0] + self.rotations = 0 + self.warmed = False + + async def rotate(self): + self.rotations += 1 + self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] + self.warmed = False + return self.session + + async def pace(self) -> None: + return None + + async def close(self) -> None: + return None + + +def _no_sleep(monkeypatch) -> None: + async def _noop(_seconds): + return None + + monkeypatch.setattr(client.asyncio, "sleep", _noop) + + +async def test_warms_then_returns_html(): + holder = _FakeHolder([_FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 0 + assert holder.session.warm_calls == 1 + + +async def test_rotates_when_warm_fails_then_succeeds(): + holder = _FakeHolder([_FakeSession(200, warms=False), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + + +async def test_404_returns_none_without_rotating(): + holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@missing") + finally: + _current_session.reset(token) + assert result is None + assert holder.rotations == 0 + + +async def test_rotates_and_rewarms_on_403(): + holder = _FakeHolder([_FakeSession(403), _FakeSession(200, warms=True)]) + token = _current_session.set(holder) + try: + result = await client.fetch_html("https://www.tiktok.com/@scout2015") + finally: + _current_session.reset(token) + assert result == _HTML + assert holder.rotations == 1 + assert holder.session.warm_calls == 1 + + +async def test_persistent_403_raises_blocked(monkeypatch): + _no_sleep(monkeypatch) + holder = _FakeHolder( + [_FakeSession(403) for _ in range(client._MAX_ROTATIONS + 1)] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await client.fetch_html("https://www.tiktok.com/@scout2015") + except TikTokAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == client._MAX_ROTATIONS From daa0856c443e547739b25cf82e50b80a42964237 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 16:07:07 +0200 Subject: [PATCH 024/160] feat(tiktok): blob-first orchestrator + video flow --- .../proprietary/platforms/tiktok/__init__.py | 19 ++++ .../platforms/tiktok/extraction/__init__.py | 9 +- .../platforms/tiktok/extraction/scopes.py | 30 +++++++ .../platforms/tiktok/flows/__init__.py | 8 ++ .../platforms/tiktok/flows/video.py | 26 ++++++ .../platforms/tiktok/orchestrator.py | 90 +++++++++++++++++++ .../platforms/tiktok/schemas/input.py | 9 +- .../platforms/tiktok/session/errors.py | 3 +- .../platforms/tiktok/test_orchestrator.py | 73 +++++++++++++++ .../unit/platforms/tiktok/test_scopes.py | 30 +++++++ 10 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index e69de29bb..27867df73 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -0,0 +1,19 @@ +"""Anonymous, blob-first TikTok scraper (public interface). + +The capability layer depends only on the names re-exported here: the input +schema, the collector/generator, the video item shape, and the hard-block error. +""" + +from __future__ import annotations + +from .orchestrator import iter_tiktok, scrape_tiktok +from .schemas import TikTokScrapeInput, TikTokVideoItem +from .session import TikTokAccessBlockedError + +__all__ = [ + "TikTokAccessBlockedError", + "TikTokScrapeInput", + "TikTokVideoItem", + "iter_tiktok", + "scrape_tiktok", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 9b4f71c68..0e6b1c3b7 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,6 +4,13 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .scopes import user_info, video_item_struct from .video import parse_video -__all__ = ["extract_rehydration_data", "parse_author", "parse_video"] +__all__ = [ + "extract_rehydration_data", + "parse_author", + "parse_video", + "user_info", + "video_item_struct", +] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py new file mode 100644 index 000000000..a20a72b2b --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/scopes.py @@ -0,0 +1,30 @@ +"""Navigate the rehydration blob to the scopes the flows consume.""" + +from __future__ import annotations + +from typing import Any + +_DEFAULT = "__DEFAULT_SCOPE__" + + +def _scope(data: dict[str, Any], name: str) -> dict[str, Any] | None: + scope = (data.get(_DEFAULT) or {}).get(name) + return scope if isinstance(scope, dict) else None + + +def video_item_struct(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``itemStruct`` of a video-detail page, or ``None``.""" + scope = _scope(data, "webapp.video-detail") + if not scope: + return None + item = (scope.get("itemInfo") or {}).get("itemStruct") + return item if isinstance(item, dict) else None + + +def user_info(data: dict[str, Any]) -> dict[str, Any] | None: + """The ``userInfo`` (``{user, stats}``) of a profile page, or ``None``.""" + scope = _scope(data, "webapp.user-detail") + if not scope: + return None + info = scope.get("userInfo") + return info if isinstance(info, dict) else None diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py new file mode 100644 index 000000000..f2f4b67cc --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -0,0 +1,8 @@ +"""Per-target scrape flows: resolved target -> normalized items.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Awaitable, Callable + +FetchFn = Callable[[str], Awaitable[str | None]] +FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py new file mode 100644 index 000000000..48e558677 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/video.py @@ -0,0 +1,26 @@ +"""Video-URL flow: fetch a post page, read its rehydration blob, emit one item.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_video, video_item_struct +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn + + +async def iter_video(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + if not html: + return + data = extract_rehydration_data(html) + if not data: + return + item = video_item_struct(data) + if item is None: + return + out = parse_video(item) + out["scrapedAt"] = now_iso() + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py new file mode 100644 index 000000000..d924e3a9e --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -0,0 +1,90 @@ +"""Resolve a :class:`TikTokScrapeInput` into targets and stream their items. + +Targets run sequentially on one warm sticky IP; ``limit`` is collector policy +applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to +its flow via :func:`_dispatch`. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from .flows import FetchFn +from .flows.video import iter_video +from .schemas import TikTokScrapeInput +from .session import fetch_html, proxy_session +from .targets import resolve_target +from .targets.types import TikTokTarget + +logger = logging.getLogger(__name__) + +_PROFILE_URL = "https://www.tiktok.com/@{name}" +_HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" +_SEARCH_URL = "https://www.tiktok.com/search?q={query}" + + +async def _empty() -> AsyncIterator[dict[str, Any]]: + for _ in (): + yield {} + + +def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: + """Build the target list from every input source, dropping unresolved URLs.""" + targets: list[TikTokTarget] = [] + for entry in input_model.startUrls: + resolved = resolve_target(entry.url) + if resolved is not None: + targets.append(resolved) + for url in input_model.postURLs: + resolved = resolve_target(url) + if resolved is not None: + targets.append(resolved) + for profile in input_model.profiles: + name = profile.lstrip("@") + targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name))) + for tag in input_model.hashtags: + targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag))) + for query in input_model.searchQueries: + targets.append( + TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query))) + ) + return targets + + +def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: + if target.kind == "video": + return iter_video(target, fetch=fetch) + # Listings come from the signed item_list API, not the blob. + logger.debug("[tiktok] no blob flow for %s target", target.kind) + return _empty() + + +async def iter_tiktok( + input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html +) -> AsyncIterator[dict[str, Any]]: + """Yield normalized items for every resolved target, in order.""" + async with proxy_session(): + for target in _resolve_targets(input_model): + async for item in _dispatch(target, fetch=fetch): + yield item + + +async def scrape_tiktok( + input_model: TikTokScrapeInput, + *, + limit: int | None = None, + fetch: FetchFn = fetch_html, +) -> list[dict[str, Any]]: + """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_tiktok(input_model, fetch=fetch): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py index fa87772fe..f710eed72 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/input.py @@ -1,12 +1,11 @@ # ruff: noqa: N815 - field names mirror the public camelCase TikTok/Apify API """Input surface for the TikTok scraper, shaped to the Clockworks actor. -Anonymous only: no auth-shaped field exists here. Fields the Phase-1 blob-first -path does not yet act on (media downloads, follower add-ons) are still accepted -via ``extra="allow"`` for contract parity and land inert. +Anonymous only: no auth-shaped field exists here. ``extra="allow"`` keeps the +contract in parity with the actor for fields the scraper does not read. -Caps (``resultsPerPage``) are per-target counts; the cross-target ceiling is -caller policy applied by the collector, never baked into the flows. +``resultsPerPage`` is a per-target count; the cross-target ceiling is caller +policy applied by the collector, never baked into a flow. """ from __future__ import annotations diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py index 4d9fc712d..708cb0212 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/errors.py @@ -6,6 +6,5 @@ from __future__ import annotations class TikTokAccessBlockedError(RuntimeError): """Raised when every rotated IP is refused anonymous access. - Anonymous-only: we cannot log in, so a hard block is surfaced loudly rather - than returning empty data. The route maps it to a 403. + Distinguishes a hard block from an empty result; the route maps it to 403. """ diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py new file mode 100644 index 000000000..85c601047 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -0,0 +1,73 @@ +"""End-to-end orchestration over a fake fetch (no network). + +Drives the public collector: input -> target -> blob-first flow -> items. +""" + +from __future__ import annotations + +import json + +from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + +def _video_page(video_id: str, username: str) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": { + "itemInfo": { + "itemStruct": { + "id": video_id, + "desc": "hello", + "author": {"uniqueId": username}, + "stats": {"diggCount": 5}, + } + } + } + } + } + return ( + '' + ) + + +async def test_scrape_video_url_returns_parsed_item(): + url = "https://www.tiktok.com/@scout2015/video/123" + + async def fake_fetch(_url: str) -> str: + return _video_page("123", "scout2015") + + items = await scrape_tiktok(TikTokScrapeInput(postURLs=[url]), fetch=fake_fetch) + + assert len(items) == 1 + assert items[0]["id"] == "123" + assert items[0]["diggCount"] == 5 + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@scout2015/video/123" + assert items[0]["scrapedAt"] is not None + + +async def test_scrape_honors_limit_across_targets(): + urls = [ + "https://www.tiktok.com/@a/video/1", + "https://www.tiktok.com/@b/video/2", + ] + + async def fake_fetch(url: str) -> str: + vid = url.rsplit("/", 1)[1] + user = url.split("@")[1].split("/")[0] + return _video_page(vid, user) + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=urls), limit=1, fetch=fake_fetch + ) + assert len(items) == 1 + + +async def test_scrape_skips_unrecognized_urls(): + async def fake_fetch(_url: str) -> str: + return "" + + items = await scrape_tiktok( + TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch + ) + assert items == [] diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py new file mode 100644 index 000000000..7a52dd60d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_scopes.py @@ -0,0 +1,30 @@ +"""Navigating the rehydration blob to its useful scopes (pure, no network).""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import user_info, video_item_struct + + +def test_video_item_struct_navigates_video_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": {"itemInfo": {"itemStruct": {"id": "123"}}} + } + } + item = video_item_struct(data) + assert item == {"id": "123"} + + +def test_user_info_navigates_user_detail_scope(): + data = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": {"userInfo": {"user": {"uniqueId": "scout2015"}}} + } + } + info = user_info(data) + assert info == {"user": {"uniqueId": "scout2015"}} + + +def test_scopes_return_none_when_absent(): + assert video_item_struct({}) is None + assert user_info({"__DEFAULT_SCOPE__": {}}) is None From ad8b73477a281a175cd579c12105f5ed02245207 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 16:37:36 +0200 Subject: [PATCH 025/160] feat(tiktok): browser-driven signed listings --- .../platforms/tiktok/extraction/__init__.py | 2 + .../platforms/tiktok/extraction/item_list.py | 30 ++++++ .../platforms/tiktok/flows/__init__.py | 5 + .../platforms/tiktok/flows/listing.py | 37 ++++++++ .../platforms/tiktok/orchestrator.py | 41 +++++---- .../platforms/tiktok/session/__init__.py | 2 + .../platforms/tiktok/session/listing.py | 92 +++++++++++++++++++ .../unit/platforms/tiktok/test_item_list.py | 25 +++++ .../platforms/tiktok/test_orchestrator.py | 24 +++++ 9 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 0e6b1c3b7..d70a494a5 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -4,11 +4,13 @@ from __future__ import annotations from .author import parse_author from .hydration import extract_rehydration_data +from .item_list import items_from_response from .scopes import user_info, video_item_struct from .video import parse_video __all__ = [ "extract_rehydration_data", + "items_from_response", "parse_author", "parse_video", "user_info", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py new file mode 100644 index 000000000..7dddf891f --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/item_list.py @@ -0,0 +1,30 @@ +"""Item structs from a captured ``item_list`` / search API response body. + +Profile and hashtag listings return ``{"itemList": [...]}``; search returns +``{"data": [{"item": {...}}]}``. Both element shapes are the same itemStruct +:func:`parse_video` already consumes. +""" + +from __future__ import annotations + +from typing import Any + + +def items_from_response(body: Any) -> list[dict[str, Any]]: + """Return the itemStructs carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + + item_list = body.get("itemList") + if isinstance(item_list, list): + return [i for i in item_list if isinstance(i, dict)] + + data = body.get("data") + if isinstance(data, list): + return [ + entry["item"] + for entry in data + if isinstance(entry, dict) and isinstance(entry.get("item"), dict) + ] + + return [] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index f2f4b67cc..a7809567d 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -5,4 +5,9 @@ from __future__ import annotations from collections.abc import AsyncIterator, Awaitable, Callable FetchFn = Callable[[str], Awaitable[str | None]] +"""Fetch a page's HTML by URL (blob-first video flow).""" + +FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a listing page and return up to ``count`` captured itemStructs.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py new file mode 100644 index 000000000..44fcc33e2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py @@ -0,0 +1,37 @@ +"""Listing flow shared by profile, hashtag, and search targets. + +The browser seam returns raw itemStructs captured from the signed ``item_list`` +XHRs; this maps each to the output contract, drops duplicate video ids, and +stops at the per-target ``cap``. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_video +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchListingFn + + +async def iter_listing( + target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for item in await fetch_listing(target.url, cap): + out = parse_video(item) + video_id = out.get("id") + if video_id is not None: + if video_id in seen: + continue + seen.add(video_id) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index d924e3a9e..c7f2e9008 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -2,35 +2,29 @@ Targets run sequentially on one warm sticky IP; ``limit`` is collector policy applied by :func:`scrape_tiktok`, never baked into a flow. Each kind routes to -its flow via :func:`_dispatch`. +its flow via :func:`_dispatch`: video URLs read the rehydration blob over HTTP, +listings capture signed item_list XHRs through the stealth browser. """ from __future__ import annotations -import logging from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn +from .flows import FetchFn, FetchListingFn +from .flows.listing import iter_listing from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, proxy_session +from .session import fetch_html, fetch_item_list, proxy_session from .targets import resolve_target from .targets.types import TikTokTarget -logger = logging.getLogger(__name__) - _PROFILE_URL = "https://www.tiktok.com/@{name}" _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" _SEARCH_URL = "https://www.tiktok.com/search?q={query}" -async def _empty() -> AsyncIterator[dict[str, Any]]: - for _ in (): - yield {} - - def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: """Build the target list from every input source, dropping unresolved URLs.""" targets: list[TikTokTarget] = [] @@ -54,21 +48,31 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: return targets -def _dispatch(target: TikTokTarget, *, fetch: FetchFn) -> AsyncIterator[dict[str, Any]]: +def _dispatch( + target: TikTokTarget, + *, + cap: int, + fetch: FetchFn, + fetch_listing: FetchListingFn, +) -> AsyncIterator[dict[str, Any]]: if target.kind == "video": return iter_video(target, fetch=fetch) - # Listings come from the signed item_list API, not the blob. - logger.debug("[tiktok] no blob flow for %s target", target.kind) - return _empty() + return iter_listing(target, cap=cap, fetch_listing=fetch_listing) async def iter_tiktok( - input_model: TikTokScrapeInput, *, fetch: FetchFn = fetch_html + input_model: TikTokScrapeInput, + *, + fetch: FetchFn = fetch_html, + fetch_listing: FetchListingFn = fetch_item_list, ) -> AsyncIterator[dict[str, Any]]: """Yield normalized items for every resolved target, in order.""" + cap = input_model.resultsPerPage async with proxy_session(): for target in _resolve_targets(input_model): - async for item in _dispatch(target, fetch=fetch): + async for item in _dispatch( + target, cap=cap, fetch=fetch, fetch_listing=fetch_listing + ): yield item @@ -77,12 +81,13 @@ async def scrape_tiktok( *, limit: int | None = None, fetch: FetchFn = fetch_html, + fetch_listing: FetchListingFn = fetch_item_list, ) -> list[dict[str, Any]]: """Collect :func:`iter_tiktok` into a list, honoring an optional ``limit``.""" from app.capabilities.core.progress import emit_progress results: list[dict[str, Any]] = [] - async for item in iter_tiktok(input_model, fetch=fetch): + async for item in iter_tiktok(input_model, fetch=fetch, fetch_listing=fetch_listing): results.append(item) emit_progress("scraping", current=len(results), total=limit, unit="item") if limit is not None and len(results) >= limit: diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 659305055..409aa5987 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,12 +4,14 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError +from .listing import fetch_item_list from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ "TikTokAccessBlockedError", "bind_proxy_holder", "fetch_html", + "fetch_item_list", "open_proxy_holder", "proxy_session", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py new file mode 100644 index 000000000..0d19281a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -0,0 +1,92 @@ +"""Browser-driven listing fetch: let TikTok sign its own ``item_list`` XHRs. + +Profile/hashtag/search listings need signed requests (``X-Gnarly``) whose +algorithm rev's monthly and reads a browser canvas fingerprint. Rather than port +and chase that signer, we load the page in the stealth browser we already run +(patchright-Chromium, via the web-crawler tier) and capture the itemStruct JSON +the page's own scripts fetch while scrolling. The browser is the client, so it +signs correctly for whatever version TikTok ships. + +The pure response-shape parsing lives in :func:`items_from_response`; this module +is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from scrapling.fetchers import StealthyFetcher + +from app.proprietary.web_crawler.stealth import ( + build_stealthy_kwargs, + get_stealth_config, +) +from app.utils.proxy import get_proxy_url + +from ..extraction import items_from_response + +logger = logging.getLogger(__name__) + +# XHR paths that carry itemStructs for the three listing kinds. +_ITEM_LIST_MARKERS = ( + "/api/post/item_list", + "/api/challenge/item_list", + "/api/search/", +) +# Bounded scroll: a dead page can't loop forever, and a live one stops early +# once enough items are captured. +_SCROLL_MAX_ROUNDS = 20 +_SCROLL_SETTLE_MS = 1200 + + +def _build_page_action(collected: list[dict[str, Any]], target_count: int): + """A sync ``page_action`` that captures item_list XHRs while scrolling.""" + + def _on_response(response: Any) -> None: + try: + if not any(marker in response.url for marker in _ITEM_LIST_MARKERS): + return + body = response.json() + except Exception: + return + collected.extend(items_from_response(body)) + + def page_action(page: Any) -> Any: + page.on("response", _on_response) + try: + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + except Exception as exc: + logger.debug("[tiktok] listing scroll aborted: %s", exc) + return page + + return page_action + + +def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: + collected: list[dict[str, Any]] = [] + kwargs = build_stealthy_kwargs(get_stealth_config()) + StealthyFetcher.fetch( + url, + headless=True, + network_idle=False, + proxy=get_proxy_url(), + page_action=_build_page_action(collected, target_count), + **kwargs, + ) + return collected[:target_count] + + +async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" + return await asyncio.to_thread(_fetch_sync, page_url, target_count) diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py new file mode 100644 index 000000000..00ab89b02 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_item_list.py @@ -0,0 +1,25 @@ +"""Pulling item structs out of captured item_list / search API responses.""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.extraction import items_from_response + + +def test_reads_item_list_shape(): + body = {"itemList": [{"id": "1"}, {"id": "2"}], "hasMore": True} + assert items_from_response(body) == [{"id": "1"}, {"id": "2"}] + + +def test_reads_search_data_shape(): + body = {"data": [{"type": 1, "item": {"id": "9"}}, {"type": 4, "item": {}}]} + assert items_from_response(body) == [{"id": "9"}, {}] + + +def test_skips_malformed_entries(): + body = {"data": [{"type": 1}, "junk", {"item": {"id": "7"}}]} + assert items_from_response(body) == [{"id": "7"}] + + +def test_returns_empty_for_unrelated_json(): + assert items_from_response({"statusCode": 0}) == [] + assert items_from_response("nope") == [] diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 85c601047..8520a1d85 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -71,3 +71,27 @@ async def test_scrape_skips_unrecognized_urls(): TikTokScrapeInput(postURLs=["https://example.com/x"]), fetch=fake_fetch ) assert items == [] + + +async def test_scrape_profile_returns_listing_items(): + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [ + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "2", "author": {"uniqueId": "a"}}, + ] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), fetch_listing=fake_listing + ) + assert [i["id"] for i in items] == ["1", "2"] + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" + + +async def test_listing_dedupes_then_caps_per_target(): + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}] + + items = await scrape_tiktok( + TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing + ) + assert [i["id"] for i in items] == ["1", "2"] From 5a326f3818158e40dea0fc8847b4f563ba1d4486 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 17:14:13 +0200 Subject: [PATCH 026/160] fix(tiktok): warm browser listing + drop ctx-var-in-generator --- .../platforms/tiktok/orchestrator.py | 20 ++++--- .../platforms/tiktok/session/listing.py | 60 ++++++++++++++++--- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index c7f2e9008..95bc10fb9 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -16,7 +16,7 @@ from .flows import FetchFn, FetchListingFn from .flows.listing import iter_listing from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list, proxy_session +from .session import fetch_html, fetch_item_list from .targets import resolve_target from .targets.types import TikTokTarget @@ -66,14 +66,18 @@ async def iter_tiktok( fetch: FetchFn = fetch_html, fetch_listing: FetchListingFn = fetch_item_list, ) -> AsyncIterator[dict[str, Any]]: - """Yield normalized items for every resolved target, in order.""" + """Yield normalized items for every resolved target, in order. + + The video flow's ``fetch_html`` opens its own warmed proxy session per call + when none is bound; the listing flow drives its own browser. Neither binds a + ContextVar across these ``yield``s, so the generator stays context-safe. + """ cap = input_model.resultsPerPage - async with proxy_session(): - for target in _resolve_targets(input_model): - async for item in _dispatch( - target, cap=cap, fetch=fetch, fetch_listing=fetch_listing - ): - yield item + for target in _resolve_targets(input_model): + async for item in _dispatch( + target, cap=cap, fetch=fetch, fetch_listing=fetch_listing + ): + yield item async def scrape_tiktok( diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 0d19281a2..98b2e6e8b 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -9,12 +9,17 @@ signs correctly for whatever version TikTok ships. The pure response-shape parsing lives in :func:`items_from_response`; this module is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). + +Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning +empty ``item_list`` bodies (and 429s) after a few hits. Set +``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing. """ from __future__ import annotations import asyncio import logging +import os from typing import Any from scrapling.fetchers import StealthyFetcher @@ -35,27 +40,68 @@ _ITEM_LIST_MARKERS = ( "/api/challenge/item_list", "/api/search/", ) +_HOME_URL = "https://www.tiktok.com/" +_MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early # once enough items are captured. _SCROLL_MAX_ROUNDS = 20 -_SCROLL_SETTLE_MS = 1200 +_SCROLL_SETTLE_MS = 1500 +# Warm-up poll for the anonymous msToken cookie the item_list API requires. +_WARM_POLLS = 8 +_WARM_POLL_MS = 500 -def _build_page_action(collected: list[dict[str, Any]], target_count: int): - """A sync ``page_action`` that captures item_list XHRs while scrolling.""" +def _has_mstoken(page: Any) -> bool: + try: + return any(c.get("name") == _MSTOKEN_COOKIE for c in page.context.cookies()) + except Exception: + return False + + +def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int): + """A sync ``page_action`` that warms the session then captures item_list XHRs. + + A cold context returns an empty ``item_list`` body, so we first mint the + anonymous ``msToken`` (homepage hit), then navigate to the target with the + listener already attached so page-one fires into it; scrolling pages the rest. + """ + + debug = bool(os.getenv("TIKTOK_LISTING_DEBUG")) def _on_response(response: Any) -> None: + response_url = getattr(response, "url", "") + if debug and "/api/" in response_url: + print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}") try: - if not any(marker in response.url for marker in _ITEM_LIST_MARKERS): + if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): return body = response.json() - except Exception: + except Exception as exc: + if debug: + print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}") return - collected.extend(items_from_response(body)) + items = items_from_response(body) + if debug: + print(f" [match] +{len(items)} items from {response_url[:100]}") + collected.extend(items) + + def _warm(page: Any) -> None: + if _has_mstoken(page): + return + page.goto(_HOME_URL, wait_until="domcontentloaded") + for _ in range(_WARM_POLLS): + page.wait_for_timeout(_WARM_POLL_MS) + if _has_mstoken(page): + break def page_action(page: Any) -> Any: page.on("response", _on_response) try: + _warm(page) + # Navigate (back) to the target with the listener attached and a + # token in hand, so the page-one item_list fires into the capture. + page.goto(url, wait_until="domcontentloaded") + page.wait_for_timeout(_SCROLL_SETTLE_MS) last_height = 0 for _ in range(_SCROLL_MAX_ROUNDS): if len(collected) >= target_count: @@ -81,7 +127,7 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: headless=True, network_idle=False, proxy=get_proxy_url(), - page_action=_build_page_action(collected, target_count), + page_action=_build_page_action(collected, url, target_count), **kwargs, ) return collected[:target_count] From ea95a0529fbfba5864606bdef353df32224d8e7e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 17:14:13 +0200 Subject: [PATCH 027/160] test(tiktok): live e2e smoke script --- .../scripts/e2e_tiktok_scrape.py | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_tiktok_scrape.py diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py new file mode 100644 index 000000000..29cf1edc7 --- /dev/null +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -0,0 +1,201 @@ +"""Manual functional e2e for the TikTok scraper (blob + browser-listing seams). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_tiktok_scrape.py + +What it exercises (everything REAL — live network, live proxy, live browser): + + Stage 1 — proxy egress proof (informational). + Stage 2 — profile listing via the stealth browser (captures item_list XHRs). + Stage 3 — blob video path over HTTP (a video URL derived from Stage 2). + Stage 4 — hashtag listing via the stealth browser. + Stage 5 — full scrape_tiktok() pipeline on a profile. + +On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser +suites can pin against real-shaped data without network. + +This is NOT a pytest test (it needs a live stack + proxy + a real browser). It is +the manual counterpart to the unit suites and the truth check for +``session/listing.py``, which is otherwise unverified. +""" + +import asyncio +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +_FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" + +# Evergreen public targets: the official account and a broad hashtag. +_PROFILE = "tiktok" +_HASHTAG = "food" +_COUNT = 5 + + +def _mask(url: str | None) -> str: + if not url: + return "" + p = urlsplit(url) + creds = "***@" if p.username else "" + port = f":{p.port}" if p.port else "" + return f"{p.scheme}://{creds}{p.hostname or '?'}{port}" + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +def _dump_fixture(name: str, data: Any) -> None: + _FIXTURES.mkdir(parents=True, exist_ok=True) + path = _FIXTURES / name + path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}") + + +async def _discover_video_url() -> str | None: + """Find a live video URL over plain HTTP (independent of the browser path).""" + import re + + from app.proprietary.platforms.tiktok.session import fetch_html + + html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}") + if not html: + return None + match = re.search(r"/@[\w.]+/video/\d+", html) + return f"https://www.tiktok.com{match.group(0)}" if match else None + + +async def stage_proxy() -> bool: + _hr("STAGE 1 — proxy egress (informational)") + from app.utils.proxy import get_active_provider, get_proxy_url, is_pool_backed + + provider = get_active_provider() + proxy_url = get_proxy_url() + print(f" active proxy provider : {provider.name}") + print(f" proxy url : {_mask(proxy_url)}") + print(f" pool-backed (rotates) : {is_pool_backed()}") + if not proxy_url: + print(" [INFO] no proxy configured — TikTok may block anonymous access") + return True + + +async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}") + from app.proprietary.platforms.tiktok.session import fetch_item_list + + url = f"https://www.tiktok.com/@{_PROFILE}" + raw = await fetch_item_list(url, _COUNT) + ok = _check( + "captured itemStructs from item_list XHRs", + len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), + f"{len(raw)} struct(s)", + ) + if ok: + _dump_fixture("listing_item.json", raw[0]) + return ok, raw + + +async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]: + _hr("STAGE 3 — blob video path (HTTP)") + print(f" target: {video_url}") + from app.proprietary.platforms.tiktok.extraction import ( + extract_rehydration_data, + video_item_struct, + ) + from app.proprietary.platforms.tiktok.session import fetch_html + + html = await fetch_html(video_url) + if not _check("fetched page HTML", bool(html), f"{len(html or '')} chars"): + return False, None + data = extract_rehydration_data(html or "") + if not _check("extracted rehydration blob", data is not None): + return False, None + raw = video_item_struct(data or {}) + ok = _check( + "blob carries the video itemStruct", + raw is not None and bool(raw.get("id")), + f"id={None if raw is None else raw.get('id')}", + ) + if ok and raw is not None: + _dump_fixture("video_item_struct.json", raw) + return ok, raw + + +async def stage_hashtag_listing() -> bool: + _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}") + from app.proprietary.platforms.tiktok.session import fetch_item_list + + url = f"https://www.tiktok.com/tag/{_HASHTAG}" + raw = await fetch_item_list(url, _COUNT) + return _check( + "captured itemStructs for hashtag", + len(raw) > 0 and bool(raw[0].get("id")), + f"{len(raw)} struct(s)", + ) + + +async def stage_pipeline() -> bool: + _hr("STAGE 5 — full scrape_tiktok() pipeline") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3 + ) + ok = _check( + "pipeline returns normalized video items", + len(items) > 0 + and bool(items[0].get("id")) + and bool(items[0].get("webVideoUrl")), + f"{len(items)} item(s)", + ) + if items: + print(f" sample: {items[0].get('webVideoUrl')} — {items[0].get('text', '')[:60]!r}") + return ok + + +async def main() -> int: + print("TikTok scraper functional e2e — live network + proxy + browser") + results: dict[str, bool] = {} + + results["Stage 1 proxy"] = await stage_proxy() + + # Blob path first: HTTP-only, so it's validated independently of the + # proxy-sensitive browser listing path. + video_url = await _discover_video_url() + if video_url: + ok_video, _ = await stage_blob_video(video_url) + results["Stage 3 blob video"] = ok_video + else: + print("\n [SKIP] Stage 3 — could not discover a video URL over HTTP") + + ok_profile, _ = await stage_profile_listing() + results["Stage 2 profile listing"] = ok_profile + results["Stage 4 hashtag listing"] = await stage_hashtag_listing() + results["Stage 5 pipeline"] = await stage_pipeline() + + _hr("SUMMARY") + for name, ok in results.items(): + print(f" {'PASS' if ok else 'FAIL/SKIP'} — {name}") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) From 9dd39faa50331c6df0e07790520a395d51177d93 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 17:53:29 +0200 Subject: [PATCH 028/160] test(tiktok): pin live fixtures, trim listing debug scaffolding --- .../platforms/tiktok/session/listing.py | 29 +- .../scripts/e2e_tiktok_scrape.py | 52 +- .../tests/fixtures/tiktok/listing_item.json | 559 +++++++++++++++ .../fixtures/tiktok/video_item_struct.json | 660 ++++++++++++++++++ 4 files changed, 1254 insertions(+), 46 deletions(-) create mode 100644 surfsense_backend/tests/fixtures/tiktok/listing_item.json create mode 100644 surfsense_backend/tests/fixtures/tiktok/video_item_struct.json diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 98b2e6e8b..cc84a48b5 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -10,16 +10,15 @@ signs correctly for whatever version TikTok ships. The pure response-shape parsing lives in :func:`items_from_response`; this module is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). -Requires a residential proxy: TikTok throttles bare/datacenter IPs, returning -empty ``item_list`` bodies (and 429s) after a few hits. Set -``TIKTOK_LISTING_DEBUG=1`` to print captured XHR URLs while diagnosing. +Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile +feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an +empty 200 even to its own signed request, so profile targets yield nothing. """ from __future__ import annotations import asyncio import logging -import os from typing import Any from scrapling.fetchers import StealthyFetcher @@ -66,24 +65,16 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: listener already attached so page-one fires into it; scrolling pages the rest. """ - debug = bool(os.getenv("TIKTOK_LISTING_DEBUG")) - def _on_response(response: Any) -> None: response_url = getattr(response, "url", "") - if debug and "/api/" in response_url: - print(f" [xhr] {getattr(response, 'status', '?')} {response_url[:120]}") - try: - if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): - return - body = response.json() - except Exception as exc: - if debug: - print(f" [xhr-parse-fail] {exc} :: {response_url[:120]}") + if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): return - items = items_from_response(body) - if debug: - print(f" [match] +{len(items)} items from {response_url[:100]}") - collected.extend(items) + try: + body = response.json() + except Exception: + # An empty 200 (TikTok soft-block) or a body evicted before read. + return + collected.extend(items_from_response(body)) def _warm(page: Any) -> None: if _has_mstoken(page): diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 29cf1edc7..22bebff96 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,10 +7,11 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile listing via the stealth browser (captures item_list XHRs). - Stage 3 — blob video path over HTTP (a video URL derived from Stage 2). - Stage 4 — hashtag listing via the stealth browser. - Stage 5 — full scrape_tiktok() pipeline on a profile. + Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok; + expected empty until a stronger anti-detection path exists). + Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). + Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). + Stage 5 — full scrape_tiktok() pipeline on a hashtag. On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -39,8 +40,8 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" -# Evergreen public targets: the official account and a broad hashtag. -_PROFILE = "tiktok" +# Evergreen public targets: a regular high-volume creator and a broad hashtag. +_PROFILE = "nasa" _HASHTAG = "food" _COUNT = 5 @@ -70,17 +71,11 @@ def _dump_fixture(name: str, data: Any) -> None: print(f" [fixture] wrote {path.relative_to(_BACKEND_ROOT)}") -async def _discover_video_url() -> str | None: - """Find a live video URL over plain HTTP (independent of the browser path).""" - import re - - from app.proprietary.platforms.tiktok.session import fetch_html - - html = await fetch_html(f"https://www.tiktok.com/@{_PROFILE}") - if not html: - return None - match = re.search(r"/@[\w.]+/video/\d+", html) - return f"https://www.tiktok.com{match.group(0)}" if match else None +def _url_from_struct(struct: dict[str, Any]) -> str | None: + """Build a canonical video URL from a captured itemStruct.""" + author = (struct.get("author") or {}).get("uniqueId") + vid = struct.get("id") + return f"https://www.tiktok.com/@{author}/video/{vid}" if author and vid else None async def stage_proxy() -> bool: @@ -108,8 +103,6 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), f"{len(raw)} struct(s)", ) - if ok: - _dump_fixture("listing_item.json", raw[0]) return ok, raw @@ -139,17 +132,20 @@ async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None] return ok, raw -async def stage_hashtag_listing() -> bool: +async def stage_hashtag_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 4 — hashtag listing (browser): #{_HASHTAG}") from app.proprietary.platforms.tiktok.session import fetch_item_list url = f"https://www.tiktok.com/tag/{_HASHTAG}" raw = await fetch_item_list(url, _COUNT) - return _check( + ok = _check( "captured itemStructs for hashtag", len(raw) > 0 and bool(raw[0].get("id")), f"{len(raw)} struct(s)", ) + if ok: + _dump_fixture("listing_item.json", raw[0]) + return ok, raw async def stage_pipeline() -> bool: @@ -157,7 +153,7 @@ async def stage_pipeline() -> bool: from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok items = await scrape_tiktok( - TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=3), limit=3 + TikTokScrapeInput(hashtags=[_HASHTAG], resultsPerPage=3), limit=3 ) ok = _check( "pipeline returns normalized video items", @@ -177,18 +173,20 @@ async def main() -> int: results["Stage 1 proxy"] = await stage_proxy() - # Blob path first: HTTP-only, so it's validated independently of the - # proxy-sensitive browser listing path. - video_url = await _discover_video_url() + # Hashtag listing is the reliable browser path; use one of its captured + # structs to build a real video URL for the HTTP blob path. + ok_tag, tag_structs = await stage_hashtag_listing() + results["Stage 4 hashtag listing"] = ok_tag + + video_url = _url_from_struct(tag_structs[0]) if tag_structs else None if video_url: ok_video, _ = await stage_blob_video(video_url) results["Stage 3 blob video"] = ok_video else: - print("\n [SKIP] Stage 3 — could not discover a video URL over HTTP") + print("\n [SKIP] Stage 3 — no captured struct to build a video URL") ok_profile, _ = await stage_profile_listing() results["Stage 2 profile listing"] = ok_profile - results["Stage 4 hashtag listing"] = await stage_hashtag_listing() results["Stage 5 pipeline"] = await stage_pipeline() _hr("SUMMARY") diff --git a/surfsense_backend/tests/fixtures/tiktok/listing_item.json b/surfsense_backend/tests/fixtures/tiktok/listing_item.json new file mode 100644 index 000000000..374a9bf67 --- /dev/null +++ b/surfsense_backend/tests/fixtures/tiktok/listing_item.json @@ -0,0 +1,559 @@ +{ + "AIGCDescription": "", + "CategoryType": 111, + "IsHDBitrate": false, + "anchors": [ + { + "description": "CapCut · Video Editor", + "extraInfo": { + "subtype": "" + }, + "icon": { + "urlList": [ + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=1EMaRlc0Snbs9Ltwr0s95%2BDvpFk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ] + }, + "id": "0", + "keyword": "CapCut · Editing made easy", + "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}", + "schema": "https://www.capcut.com/editor?scenario=tiktok&utm_source=tiktok_anchor_tool&utm_medium=tiktok_anchor&utm_campaign=70361230", + "thumbnail": { + "height": 64, + "urlList": [ + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=c7mUcXdyEOFc0SMcLAkkib3v9RU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ], + "width": 64 + }, + "type": 54 + } + ], + "author": { + "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "commentSetting": 0, + "downloadSetting": 3, + "duetSetting": 0, + "ftc": false, + "id": "7501138183701103662", + "isADVirtual": false, + "isEmbedBanned": false, + "nickname": "Cookwithhali", + "openFavorite": false, + "privateAccount": false, + "relation": 0, + "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q", + "secret": false, + "shortDramaCreator": {}, + "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com", + "stitchSetting": 0, + "uniqueId": "cookwithhali", + "verified": false + }, + "authorStats": { + "diggCount": 223, + "followerCount": 372600, + "followingCount": 0, + "friendCount": 0, + "heart": 8000000, + "heartCount": 8000000, + "videoCount": 279 + }, + "authorStatsV2": { + "diggCount": "223", + "followerCount": "372600", + "followingCount": "0", + "friendCount": "0", + "heart": "8000000", + "heartCount": "8000000", + "videoCount": "279" + }, + "backendSourceEventTracking": "", + "challenges": [ + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "6983", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "bread" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "285169", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "garlicbread" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "1643317566954502", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "breadtok" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.", + "id": "7250", + "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "title": "baking" + }, + { + "coverLarger": "", + "coverMedium": "", + "coverThumb": "", + "desc": "", + "id": "10488587", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "title": "homemadebread" + } + ], + "collected": false, + "contents": [ + { + "desc": "Pull Apart Garlic Bread " + }, + { + "desc": "" + }, + { + "desc": "Ingredients" + }, + { + "desc": "" + }, + { + "desc": "* 1 tsp yeast" + }, + { + "desc": "* 1 tbsp sugar" + }, + { + "desc": "* 270g milk (about 1 cup + 2 tbsp)" + }, + { + "desc": "* 380g flour (about 3 cups)" + }, + { + "desc": "* 1 tsp salt" + }, + { + "desc": "* 43g softened butter (about 3 tbsp)" + }, + { + "desc": "" + }, + { + "desc": "Butter Mixture" + }, + { + "desc": "" + }, + { + "desc": "* 100g butter, softened" + }, + { + "desc": "* 1 tbsp chopped cilantro" + }, + { + "desc": "* 1 garlic clove, minced" + }, + { + "desc": "* Shredded cheese" + }, + { + "desc": "" + }, + { + "desc": "Instructions" + }, + { + "desc": "" + }, + { + "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined." + }, + { + "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated." + }, + { + "desc": "3. Cover and let the dough rest for 2 hours or until doubled." + }, + { + "desc": "4. Mix the softened butter with the cilantro and minced garlic." + }, + { + "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese." + }, + { + "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan." + }, + { + "desc": "7. Cover and let rest for 20 minutes." + }, + { + "desc": "8. Bake at 370°F for 30 minutes, or until golden brown." + }, + { + "desc": "9. Brush with the remaining garlic butter mixture while still warm." + }, + { + "desc": "" + }, + { + "desc": "#bread #garlicbread #breadtok #baking #homemadebread ", + "textExtra": [ + { + "awemeId": "", + "end": 6, + "hashtagName": "bread", + "isCommerce": false, + "start": 0, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 19, + "hashtagName": "garlicbread", + "isCommerce": false, + "start": 7, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 29, + "hashtagName": "breadtok", + "isCommerce": false, + "start": 20, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 37, + "hashtagName": "baking", + "isCommerce": false, + "start": 30, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 52, + "hashtagName": "homemadebread", + "isCommerce": false, + "start": 38, + "subType": 0, + "type": 1 + } + ] + } + ], + "createTime": 1782509863, + "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ", + "digged": false, + "diversificationId": 10040, + "duetDisplay": 0, + "duetEnabled": true, + "forFriend": false, + "id": "7655821461772406047", + "isAd": false, + "isReviewing": false, + "itemCommentStatus": 0, + "item_control": { + "can_repost": true + }, + "music": { + "authorName": "Cookwithhali", + "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverThumb": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=8c557171&x-expires=1783695600&x-signature=VaAysM3zFkbl%2FOT74d0h2kdkLqo%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "duration": 62, + "id": "7655821492566919966", + "isCopyrighted": false, + "is_commerce_music": true, + "is_unlimited_music": false, + "original": true, + "playUrl": "https://v16m.tiktokcdn-us.com/ecd83422a3e7bf5450bd92ebf07e6fce/6a4ec449/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7Th2sbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=2026070815413036F7D0D0506FB569C78C&btag=e00050000", + "private": false, + "shoot_duration": 62, + "title": "original sound", + "tt2dsp": {} + }, + "officalItem": false, + "originalItem": false, + "privateItem": false, + "secret": false, + "shareEnabled": true, + "stats": { + "collectCount": 396400, + "commentCount": 3720, + "diggCount": 754100, + "playCount": 8300000, + "shareCount": 144500 + }, + "statsV2": { + "collectCount": "396397", + "commentCount": "3720", + "diggCount": "754100", + "playCount": "8300000", + "repostCount": "0", + "shareCount": "144500" + }, + "stickersOnItem": [ + { + "stickerText": [ + "Cheesy Pull Apart Garlic Bread! 🧄🧀\n", + "(Bakery-Style & No-Knead)" + ], + "stickerType": 4 + } + ], + "stitchDisplay": 0, + "stitchEnabled": true, + "textExtra": [ + { + "awemeId": "", + "end": 1030, + "hashtagName": "bread", + "isCommerce": false, + "start": 1024, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1043, + "hashtagName": "garlicbread", + "isCommerce": false, + "start": 1031, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1053, + "hashtagName": "breadtok", + "isCommerce": false, + "start": 1044, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1061, + "hashtagName": "baking", + "isCommerce": false, + "start": 1054, + "subType": 0, + "type": 1 + }, + { + "awemeId": "", + "end": 1076, + "hashtagName": "homemadebread", + "isCommerce": false, + "start": 1062, + "subType": 0, + "type": 1 + } + ], + "textLanguage": "en", + "textTranslatable": true, + "video": { + "PlayAddrStruct": { + "DataSize": 13761060, + "FileCs": "c:0-51456-f8d1", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "VQScore": "69.69", + "bitrate": 1773703, + "bitrateInfo": [ + { + "Bitrate": 1836180, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_lowest_1080_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}", + "PlayAddr": { + "DataSize": 14245777, + "FileCs": "c:0-52754-ed06", + "FileHash": "38996a98cfeb2714fca1b94768090c37", + "Height": 1920, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=db26af0a98ff05e504d7121b25048527&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 1080 + }, + "QualityType": 2, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 1773703, + "BitrateFPS": 29, + "CodecType": "h264", + "Format": "mp4", + "GearName": "normal_720_0", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}", + "PlayAddr": { + "DataSize": 13761060, + "FileCs": "c:0-51456-f8d1", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "QualityType": 10, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}" + }, + { + "Bitrate": 966380, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_lower_720_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}", + "PlayAddr": { + "DataSize": 7497540, + "FileCs": "c:0-52786-7f65", + "FileHash": "1376df24c917669727b27273697fb826", + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=e3690d67b2ecae289320f5db6c338d05&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 720 + }, + "QualityType": 14, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 799620, + "BitrateFPS": 29, + "CodecType": "h265_hvc1", + "Format": "mp4", + "GearName": "adapt_540_1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}", + "PlayAddr": { + "DataSize": 6203756, + "FileCs": "c:0-52786-7f6d", + "FileHash": "41f62f740ad13045000450f28b9ff2cb", + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=05aaeb04a6cf8d012bba46bb131f00af&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 576 + }, + "QualityType": 28, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 622546, + "BitrateFPS": 29, + "CodecType": "h264", + "Format": "mp4", + "GearName": "lowest_540_0", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}", + "PlayAddr": { + "DataSize": 4829948, + "FileCs": "c:0-51513-3d83", + "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d", + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=fa8ddc2bcbad8ebb4394c4a8b55b28df&tk=tt_chain_token&btag=e00088000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "Width": 576 + }, + "QualityType": 25, + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}" + } + ], + "claInfo": { + "enableAutoCaption": true, + "hasOriginalAudio": true, + "noCaptionReason": 3 + }, + "codecType": "h264", + "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "definition": "720p", + "duration": 62, + "dynamicCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=JWF%2F5g2txpFXhNhJzOwleSJgKik%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "encodeUserTag": "", + "encodedType": "normal", + "format": "mp4", + "height": 1280, + "id": "7655821461772406047", + "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12CGneI3wUcfzAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698153&l=2026070815413036F7D0D0506FB569C78C&ply_type=2&policy=2&signature=aa5be80671c4a73d54517ee8e4a471ad&tk=tt_chain_token&btag=e00088000", + "ratio": "720p", + "size": 13761060, + "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "videoQuality": "normal", + "volumeInfo": { + "Loudness": -48.5, + "Peak": 0.18408 + }, + "width": 720, + "zoomCover": { + "240": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=ZONw1V50ZmupszDX0NiC%2Bf8BimU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "480": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=LztWYGkvUC4%2BvhtQb0TO0l1ky0s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1" + } + } +} \ No newline at end of file diff --git a/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json new file mode 100644 index 000000000..af3bd651b --- /dev/null +++ b/surfsense_backend/tests/fixtures/tiktok/video_item_struct.json @@ -0,0 +1,660 @@ +{ + "id": "7655821461772406047", + "desc": "Pull Apart Garlic Bread Ingredients * 1 tsp yeast * 1 tbsp sugar * 270g milk (about 1 cup + 2 tbsp) * 380g flour (about 3 cups) * 1 tsp salt * 43g softened butter (about 3 tbsp) Butter Mixture * 100g butter, softened * 1 tbsp chopped cilantro * 1 garlic clove, minced * Shredded cheese Instructions 1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined. 2. Add the salt and softened butter, then coil fold until the butter is fully incorporated. 3. Cover and let the dough rest for 2 hours or until doubled. 4. Mix the softened butter with the cilantro and minced garlic. 5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese. 6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan. 7. Cover and let rest for 20 minutes. 8. Bake at 370°F for 30 minutes, or until golden brown. 9. Brush with the remaining garlic butter mixture while still warm. #bread #garlicbread #breadtok #baking #homemadebread ", + "createTime": "1782509863", + "scheduleTime": 0, + "video": { + "id": "7655821461772406047", + "height": 1280, + "width": 720, + "duration": 62, + "ratio": "720p", + "cover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=ZIp3Rdhs6GNzMatSMsNWHFZX%2F7s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "originCover": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoQGFsU9VvLxieEDDAQAPpRTqA8B5EzvEGfF~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=yHDGczHGo3joyDeerZP6Dp4S1Jc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "dynamicCover": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oYVoPG0RxQTGE89EEqvAuADpFvfGABVMxQBeza~tplv-tiktokx-origin.image?dr=9636&x-expires=1783695600&x-signature=Jz%2BPtpSXdbjTV4qFbG4H7JFD0f8%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5", + "playAddr": "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "downloadAddr": "", + "shareCover": [ + "" + ], + "reflowCover": "", + "bitrate": 1773703, + "encodedType": "normal", + "format": "mp4", + "videoQuality": "normal", + "encodeUserTag": "", + "codecType": "h264", + "definition": "720p", + "subtitleInfos": [], + "zoomCover": { + "240": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:240:240.avif?dr=9616&x-expires=1783695600&x-signature=cBXoX16WOKJyUth9fnaohYv%2B%2BBs%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "480": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:480:480.avif?dr=9616&x-expires=1783695600&x-signature=Ff5B1aV4teSPQc6JXMoidMXqJWk%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "720": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:720:720.avif?dr=9616&x-expires=1783695600&x-signature=X%2F3HQ%2BkiKqEAxnuk%2BdFfBfeOsko%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1", + "960": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-p-0068-tx2/oAiefnCIc6NNSgtgrKLRA8JGAMoVfgJSe8eiRS~tplv-photomode-zoomcover:960:960.avif?dr=9616&x-expires=1783695600&x-signature=aJaYG%2BDAo%2BhO7nZ8t8XP7g0DArU%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=43f4a2f9&idc=useast5&ftpl=1" + }, + "volumeInfo": { + "Loudness": -48.5, + "Peak": 0.18408 + }, + "bitrateInfo": [ + { + "Bitrate": 1836180, + "QualityType": 2, + "BitrateFPS": 29, + "GearName": "adapt_lowest_1080_1", + "PlayAddr": { + "DataSize": "14245777", + "Width": 1080, + "Height": 1920, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oIygxEfEVAP1qvGVTIBARoAvb9QYFGEKDp5v9e/?a=1988&bti=ODszNWYuMDE6&&bt=1793&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aDlmNGg7ZTdlaGY5ZzUzZ0BpanJneW45cmVpOzMzaTczNEBjXl5hMzAvXy8xNDAuYmIyYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=0648319cea95ee08fd7b4f93a95cb14c&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=5de0e21d367c4e2bb42b386ed1aca6fe&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjAzMmU1YzJlMTAwMDJiYjIwMmFiN2RhNTIzOWVhYWRk&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_1080p_1836180", + "FileHash": "38996a98cfeb2714fca1b94768090c37", + "FileCs": "c:0-52754-ed06" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 96.83, \"v960\": 97.64, \"v864\": 98.256, \"v720\": 98.838}, \"ori\": {\"v1080\": 91.834, \"v960\": 93.423, \"v864\": 94.54, \"v720\": 95.872}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 385342}, {\\\"time\\\": 2, \\\"offset\\\": 561014}, {\\\"time\\\": 3, \\\"offset\\\": 703814}, {\\\"time\\\": 4, \\\"offset\\\": 855867}, {\\\"time\\\": 5, \\\"offset\\\": 1019333}, {\\\"time\\\": 10, \\\"offset\\\": 2028412}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 96.83, \\\"v960\\\": 97.64, \\\"v864\\\": 98.256, \\\"v720\\\": 98.838}, \\\"ori\\\": {\\\"v1080\\\": 91.834, \\\"v960\\\": 93.423, \\\"v864\\\": 94.54, \\\"v720\\\": 95.872}}}\",\"ufq\":\"{\\\"enh\\\":89.317,\\\"playback\\\":{\\\"ori\\\":85.151,\\\"srv1\\\":90.147},\\\"src\\\":85.957,\\\"trans\\\":85.151,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"11b0345f3bf185ddbe7c48bd7301980f\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwCWAACdzkgS\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwCWAACgAhyAHgWWdzkySUQs0RJJJJify4d/1+bPl/1+M4gQ5qAgICCAAAADAIAAAA8E\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 1773703, + "QualityType": 10, + "BitrateFPS": 29, + "GearName": "normal_720_0", + "PlayAddr": { + "DataSize": "13761060", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "FileCs": "c:0-51456-f8d1" + }, + "CodecType": "h264", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}, \"ori\": {\"v1080\": 84.77, \"v960\": 88.554, \"v864\": 88.982, \"v720\": 92.772}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 400712}, {\\\"time\\\": 2, \\\"offset\\\": 659243}, {\\\"time\\\": 3, \\\"offset\\\": 886968}, {\\\"time\\\": 4, \\\"offset\\\": 1101663}, {\\\"time\\\": 5, \\\"offset\\\": 1368565}, {\\\"time\\\": 10, \\\"offset\\\": 2375704}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}, \\\"ori\\\": {\\\"v1080\\\": 84.77, \\\"v960\\\": 88.554, \\\"v864\\\": 88.982, \\\"v720\\\": 92.772}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"65b88c41c0963253ad31b6f68981a242\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":64193}" + }, + { + "Bitrate": 966380, + "QualityType": 14, + "BitrateFPS": 29, + "GearName": "adapt_lower_720_1", + "PlayAddr": { + "DataSize": "7497540", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/og99rgOGqJED8xGeBpFEAADvEPARnbTIQTfoVv/?a=1988&bti=ODszNWYuMDE6&&bt=943&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=OTNkNDk2NzlmM2VkPGg7OkBpanJneW45cmVpOzMzaTczNEBeMmBiYWBfNWIxLzEyNF81YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=3a0c7313f1a5d9908cc2f5b0dd24d048&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=87b87e594a2b40ec86b4c3d6e1ca50a6&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRjNWNmZGVmMmFkNTIyNWQ1ZTA1NzExM2NjOTVlNzMy&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_720p_966380", + "FileHash": "1376df24c917669727b27273697fb826", + "FileCs": "c:0-52786-7f65" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 90.05, \"v960\": 91.958, \"v864\": 93.608, \"v720\": 95.869}, \"ori\": {\"v1080\": 82.259, \"v960\": 85.339, \"v864\": 87.288, \"v720\": 90.851}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 284307}, {\\\"time\\\": 2, \\\"offset\\\": 357012}, {\\\"time\\\": 3, \\\"offset\\\": 438159}, {\\\"time\\\": 4, \\\"offset\\\": 532720}, {\\\"time\\\": 5, \\\"offset\\\": 636824}, {\\\"time\\\": 10, \\\"offset\\\": 1142427}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 90.05, \\\"v960\\\": 91.958, \\\"v864\\\": 93.608, \\\"v720\\\": 95.869}, \\\"ori\\\": {\\\"v1080\\\": 82.259, \\\"v960\\\": 85.339, \\\"v864\\\": 87.288, \\\"v720\\\": 90.851}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":76.711,\\\"srv1\\\":84.502},\\\"src\\\":85.957,\\\"trans\\\":76.711,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"97eb57dffd9e7a0333979f6ca6c9418d\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 91.625, \\\"sr20\\\": 97.355}}\",\"audio_bit_rate\":96215,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBaIAUBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 799620, + "QualityType": 28, + "BitrateFPS": 29, + "GearName": "adapt_540_1", + "PlayAddr": { + "DataSize": "6203756", + "Width": 576, + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/oogvkDPHIAVvTxgEpE9EMoALxQRRA1FGqeB9Gf/?a=1988&bti=ODszNWYuMDE6&&bt=780&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NThkNGRmNTo4MztpOTM3ZkBpanJneW45cmVpOzMzaTczNEAyNDMuNC0xNWExYjBiX2FgYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=a8516d4d8e79814753dc2cd0fde9f7bd&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=ccd4b0e63fe840d587420c7a116a7552&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLmRlMWFkM2IxY2JkODUxODVlZDJjMDFjYTJmZWE0NTEx&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_bytevc1_540p_799620", + "FileHash": "41f62f740ad13045000450f28b9ff2cb", + "FileCs": "c:0-52786-7f6d" + }, + "CodecType": "h265_hvc1", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 87.061, \"v960\": 89.844, \"v864\": 91.813, \"v720\": 94.499}, \"ori\": {\"v1080\": 77.717, \"v960\": 80.972, \"v864\": 83.632, \"v720\": 87.945}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 254747}, {\\\"time\\\": 2, \\\"offset\\\": 310409}, {\\\"time\\\": 3, \\\"offset\\\": 378126}, {\\\"time\\\": 4, \\\"offset\\\": 458158}, {\\\"time\\\": 5, \\\"offset\\\": 546764}, {\\\"time\\\": 10, \\\"offset\\\": 946117}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 87.061, \\\"v960\\\": 89.844, \\\"v864\\\": 91.813, \\\"v720\\\": 94.499}, \\\"ori\\\": {\\\"v1080\\\": 77.717, \\\"v960\\\": 80.972, \\\"v864\\\": 83.632, \\\"v720\\\": 87.945}}}\",\"ufq\":\"{\\\"enh\\\":89.767,\\\"playback\\\":{\\\"ori\\\":73.304,\\\"srv1\\\":82.648},\\\"src\\\":85.957,\\\"trans\\\":73.304,\\\"version\\\":\\\"v1.2\\\"}\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"86fda13c754bde67159dea14b7d55d6a\",\"dec_info\":\"\",\"gearvqm\":\"{\\\"v3.0\\\": {\\\"ori\\\": 87.963, \\\"sr20\\\": 95.722}}\",\"audio_bit_rate\":64143,\"ps_info\":\"{\\\"vps\\\": \\\"AQwC//8BYAAAAwAAAwAAAwAAAwC6AACIJElgSA==\\\", \\\"sps\\\": \\\"AQMBYAAAAwAAAwAAAwAAAwC6AACgBIIAQBZYgkSXJIkQTPCEREREREYR8xPOX+/+v82fy/+v8yh+S/3/1/mz+X/1/jOEAg5qAgICCAAAAwAIAAADAPBA\\\", \\\"pps\\\": \\\"AcElPAiQ\\\"}\"}" + }, + { + "Bitrate": 622546, + "QualityType": 25, + "BitrateFPS": 29, + "GearName": "lowest_540_0", + "PlayAddr": { + "DataSize": "4829948", + "Width": 576, + "Height": 1024, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-ve-0068c001-tx2/o8vENAqIe9xD3BG8AvVUvfEqFEARhgPpHGoTJQ/?a=1988&bti=ODszNWYuMDE6&&bt=607&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=aGhoaDQ8ZTdlNGhmM2Q8ZkBpanJneW45cmVpOzMzaTczNEA1NjRgNTJjNi8xMl42NmIuYSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=f44ac110e4573a7bc11c73155b9141d0&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=1ce99a90ac5e412b94fb5981ce7e9e8e&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjU0ZWZiNDU4NWVmYzFlY2IyNmEyODAyMGY0ZWJjMGZj&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_540p_622546", + "FileHash": "a74fd8fabed8b36131a8e31bf78e9d1d", + "FileCs": "c:0-51513-3d83" + }, + "CodecType": "h264", + "MVMAF": "{\"v2.0\": {\"srv1\": {\"v1080\": 72.483, \"v960\": 75.424, \"v864\": 79.843, \"v720\": 84.897}, \"ori\": {\"v1080\": 64.527, \"v960\": 67.93, \"v864\": 72.279, \"v720\": 74.495}}}", + "Format": "mp4", + "VideoExtra": "{\"PktOffsetMap\":\"[{\\\"time\\\": 1, \\\"offset\\\": 176720}, {\\\"time\\\": 2, \\\"offset\\\": 268125}, {\\\"time\\\": 3, \\\"offset\\\": 342814}, {\\\"time\\\": 4, \\\"offset\\\": 414132}, {\\\"time\\\": 5, \\\"offset\\\": 511717}, {\\\"time\\\": 10, \\\"offset\\\": 841405}]\",\"mvmaf\":\"{\\\"v2.0\\\": {\\\"srv1\\\": {\\\"v1080\\\": 72.483, \\\"v960\\\": 75.424, \\\"v864\\\": 79.843, \\\"v720\\\": 84.897}, \\\"ori\\\": {\\\"v1080\\\": 64.527, \\\"v960\\\": 67.93, \\\"v864\\\": 72.279, \\\"v720\\\": 74.495}}}\",\"ufq\":\"\",\"volume_info_json\":\"\",\"transcode_feature_id\":\"25b3faebdda74a9e438e851988a6efcc\",\"dec_info\":\"\",\"gearvqm\":\"\",\"audio_bit_rate\":32099}" + } + ], + "size": "13761060", + "VQScore": "69.69", + "claInfo": { + "hasOriginalAudio": true, + "enableAutoCaption": true, + "captionInfos": [], + "noCaptionReason": 3 + }, + "videoID": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "PlayAddrStruct": { + "DataSize": "13761060", + "Width": 720, + "Height": 1280, + "Uri": "v15044gf0000d8vf0bvog65kkvkd8ohg", + "UrlList": [ + "https://v16-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://v19-webapp-prime.us.tiktok.com/video/tos/useast8/tos-useast8-pve-0068-tx2/oYDTerfAgGxvqFaEctpBAvVGEPIE8Q9ARoxPNB/?a=1988&bti=ODszNWYuMDE6&&bt=1732&ft=aEeq8qT0mIoPD12qGneI3wU6LRAbMeF~O5&mime_type=video_mp4&rc=NWZnZDVoNDQzN2c1ZGU0Z0BpanJneW45cmVpOzMzaTczNEBgMTBgYjQuNWExLmNgXmI2YSNlNWhoMmRzLXNhLS1kMTJzcw%3D%3D&expire=1783698158&l=20260708154135271AF032BAC01F42EF1E&ply_type=2&policy=2&signature=1bb5fb748a97085b872e357d1cd5406e&tk=tt_chain_token&btag=e00090000", + "https://www.tiktok.com/aweme/v1/play/?faid=1988&file_id=9de1abcaf21d44d081117ee2ca3d4cc2&is_play_url=1&item_id=7655821461772406047&line=0&ply_type=2&signaturev3=dmlkZW9faWQ7ZmlsZV9pZDtpdGVtX2lkLjcwMGExZmI3NDg4YzFiZTIwZjczM2IyMmM0OTk2YTI2&tk=tt_chain_token&video_id=v15044gf0000d8vf0bvog65kkvkd8ohg" + ], + "UrlKey": "v15044gf0000d8vf0bvog65kkvkd8ohg_h264_720p_1773703", + "FileHash": "485174c8492db027adbbf6dca7c7c201", + "FileCs": "c:0-51456-f8d1" + } + }, + "author": { + "id": "7501138183701103662", + "shortId": "", + "uniqueId": "cookwithhali", + "nickname": "Cookwithhali", + "avatarLarger": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "avatarThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "signature": "From my kitchen to your screen! 🍞\nFollow for more ✨\nIG: halikit25\n\n📩 Halikitchen25@gmail.com", + "createTime": 1746494976, + "verified": false, + "secUid": "MS4wLjABAAAAywZ6lCXK3u4zKz8eQxxcjVblLiugz6zMysU98G7dyKVxu6IvMOJ97mpIfpDxUL4q", + "ftc": false, + "relation": 0, + "openFavorite": false, + "commentSetting": 0, + "duetSetting": 0, + "stitchSetting": 0, + "privateAccount": false, + "secret": false, + "isADVirtual": false, + "roomId": "", + "uniqueIdModifyTime": 0, + "ttSeller": false, + "downloadSetting": 3, + "recommendReason": "", + "nowInvitationCardUrl": "", + "nickNameModifyTime": 0, + "isEmbedBanned": false, + "canExpPlaylist": false, + "suggestAccountBind": false, + "UserStoryStatus": 0, + "shortDramaCreator": {} + }, + "music": { + "id": "7655821492566919966", + "title": "original sound", + "playUrl": "https://v16m.tiktokcdn-us.com/84ccc1629d7199ad68f976d419148401/6a4ec44e/video/tos/useast8/tos-useast8-v-27dcd7-tx2/ocBIhANWILSemeAAINEGoTTPe58AJaRegg6eKS/?a=1233&bti=ODszNWYuMDE6&&bt=125&ft=GSDrKInz7ThasbSGXq8Zmo&mime_type=audio_mpeg&rc=anY5cXA5cmRpOzMzaTU8NEBpanY5cXA5cmRpOzMzaTU8NEBpYzZrMmRjL3NhLS1kMTJzYSNpYzZrMmRjL3NhLS1kMTJzcw%3D%3D&vvpl=1&l=20260708154135271AF032BAC01F42EF1E&btag=e00050000", + "coverLarge": "https://p19-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:1080:1080.jpeg?dr=9640&refresh_token=c6d77c8a&x-expires=1783695600&x-signature=mUTuvDWLEGrAxXtWZlBE7gUxCl8%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverMedium": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:720:720.jpeg?dr=9640&refresh_token=dfe18cb8&x-expires=1783695600&x-signature=cP14vkr0ndxwhW%2B1pWfefXfTsls%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "coverThumb": "https://p16-common-sign.tiktokcdn-us.com/tos-useast8-avt-0068-tx2/b29bd2d826cffba1af22f99793a7d5dc~tplv-tiktokx-cropcenter:100:100.jpeg?dr=9640&refresh_token=84bba9f5&x-expires=1783695600&x-signature=Et06On6iQeXFbtycXoVodzl0fdM%3D&t=4d5b0474&ps=13740610&shp=a5d48078&shcp=81f88b70&idc=useast5", + "authorName": "Cookwithhali", + "original": true, + "private": false, + "duration": 62, + "scheduleSearchTime": 0, + "collected": false, + "preciseDuration": { + "preciseDuration": 62.093063, + "preciseShootDuration": 62.093063, + "preciseAuditionDuration": 62.093063, + "preciseVideoDuration": 62.093063 + }, + "isCopyrighted": false, + "tt2dsp": { + "tt_to_dsp_song_infos": [] + }, + "is_unlimited_music": false, + "is_commerce_music": true, + "shoot_duration": 62 + }, + "challenges": [ + { + "id": "6983", + "title": "bread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "285169", + "title": "garlicbread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "1643317566954502", + "title": "breadtok", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "7250", + "title": "baking", + "desc": "Whether you're baking the perfect pie or just searching for a recipe, you've come to the right place for all things #Baking.", + "profileLarger": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileMedium": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "profileThumb": "https://p19-common-sign.tiktokcdn-us.com/musically-maliva-obj/1aa8a54136dc06390b83508fdd683160~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=nigLPQXvvDM%2BxFQezn3ELBH%2BLwQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + }, + { + "id": "10488587", + "title": "homemadebread", + "desc": "", + "profileLarger": "", + "profileMedium": "", + "profileThumb": "", + "coverLarger": "", + "coverMedium": "", + "coverThumb": "" + } + ], + "stats": { + "diggCount": 754100, + "shareCount": 144500, + "commentCount": 3720, + "playCount": 8300000, + "collectCount": "396399" + }, + "statsV2": { + "diggCount": "754100", + "shareCount": "144500", + "commentCount": "3720", + "playCount": "8300000", + "collectCount": "396399", + "repostCount": "0" + }, + "warnInfo": [], + "originalItem": false, + "officalItem": false, + "textExtra": [ + { + "awemeId": "", + "start": 1024, + "end": 1030, + "hashtagId": "6983", + "hashtagName": "bread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1031, + "end": 1043, + "hashtagId": "285169", + "hashtagName": "garlicbread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1044, + "end": 1053, + "hashtagId": "1643317566954502", + "hashtagName": "breadtok", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1054, + "end": 1061, + "hashtagId": "7250", + "hashtagName": "baking", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 1062, + "end": 1076, + "hashtagId": "10488587", + "hashtagName": "homemadebread", + "type": 1, + "subType": 0, + "isCommerce": false + } + ], + "secret": false, + "forFriend": false, + "digged": false, + "itemCommentStatus": 0, + "takeDown": 0, + "effectStickers": [], + "authorStats": { + "followerCount": 372600, + "followingCount": 0, + "heart": 8000000, + "heartCount": 8000000, + "videoCount": 279, + "diggCount": 223, + "friendCount": 0 + }, + "privateItem": false, + "duetEnabled": true, + "stitchEnabled": true, + "stickersOnItem": [ + { + "stickerText": [ + "Cheesy Pull Apart Garlic Bread! 🧄🧀\n", + "(Bakery-Style & No-Knead)" + ], + "stickerType": 4 + } + ], + "isAd": false, + "shareEnabled": true, + "comments": [], + "duetDisplay": 0, + "stitchDisplay": 0, + "indexEnabled": true, + "diversificationLabels": [ + "Cooking", + "Food & Drink", + "Lifestyle" + ], + "locationCreated": "US", + "suggestedWords": [ + "Garlic Bread", + "garlic bread recipe", + "pull apart bread", + "Sourdough Bread", + "How To Make Garlic Bread", + "Baking", + "food", + "bread", + "Baking Recipes", + "Garlic Pull Apart Bread Recipe" + ], + "contents": [ + { + "desc": "Pull Apart Garlic Bread ", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Ingredients", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "* 1 tsp yeast", + "textExtra": [] + }, + { + "desc": "* 1 tbsp sugar", + "textExtra": [] + }, + { + "desc": "* 270g milk (about 1 cup + 2 tbsp)", + "textExtra": [] + }, + { + "desc": "* 380g flour (about 3 cups)", + "textExtra": [] + }, + { + "desc": "* 1 tsp salt", + "textExtra": [] + }, + { + "desc": "* 43g softened butter (about 3 tbsp)", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Butter Mixture", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "* 100g butter, softened", + "textExtra": [] + }, + { + "desc": "* 1 tbsp chopped cilantro", + "textExtra": [] + }, + { + "desc": "* 1 garlic clove, minced", + "textExtra": [] + }, + { + "desc": "* Shredded cheese", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "Instructions", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "1. In a bowl, mix the sugar and yeast. Add the milk, then add the flour and mix until combined.", + "textExtra": [] + }, + { + "desc": "2. Add the salt and softened butter, then coil fold until the butter is fully incorporated.", + "textExtra": [] + }, + { + "desc": "3. Cover and let the dough rest for 2 hours or until doubled.", + "textExtra": [] + }, + { + "desc": "4. Mix the softened butter with the cilantro and minced garlic.", + "textExtra": [] + }, + { + "desc": "5. Roll the dough into a rectangle and spread the garlic butter mixture evenly over it. Sprinkle with shredded cheddar cheese.", + "textExtra": [] + }, + { + "desc": "6. Cut the dough in half, stack one half on top of the other, then cut into strips. Transfer the pieces into a loaf pan.", + "textExtra": [] + }, + { + "desc": "7. Cover and let rest for 20 minutes.", + "textExtra": [] + }, + { + "desc": "8. Bake at 370°F for 30 minutes, or until golden brown.", + "textExtra": [] + }, + { + "desc": "9. Brush with the remaining garlic butter mixture while still warm.", + "textExtra": [] + }, + { + "desc": "", + "textExtra": [] + }, + { + "desc": "#bread #garlicbread #breadtok #baking #homemadebread ", + "textExtra": [ + { + "awemeId": "", + "start": 0, + "end": 6, + "hashtagId": "6983", + "hashtagName": "bread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 7, + "end": 19, + "hashtagId": "285169", + "hashtagName": "garlicbread", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 20, + "end": 29, + "hashtagId": "1643317566954502", + "hashtagName": "breadtok", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 30, + "end": 37, + "hashtagId": "7250", + "hashtagName": "baking", + "type": 1, + "subType": 0, + "isCommerce": false + }, + { + "awemeId": "", + "start": 38, + "end": 52, + "hashtagId": "10488587", + "hashtagName": "homemadebread", + "type": 1, + "subType": 0, + "isCommerce": false + } + ] + } + ], + "diversificationId": 10040, + "anchors": [ + { + "id": "0", + "type": 54, + "keyword": "CapCut · Editing made easy", + "icon": { + "uri": "tiktok-obj/capcut_logo_64px_bk.png", + "urlList": [ + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=lMwgk8e0wLaM3WkpnsMqYcxZN6w%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=44xwWHkyYJdE%2FIwqS3zAAjU4XQ4%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/capcut_logo_64px_bk.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=lzVxquDTAbut%2BvshGEhsFsYQc2s%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ] + }, + "schema": "https://www.capcut.com/tools/desktop-video-editor?channel=capcutpc_ttweb_anchor_edit1&download_channel=capcutpc_ttweb_anchor_edit1&enter_from=capcutpc_ttweb_anchor_edit1&force_dp=1&from_page=capcutpc_ttweb_anchor_edit1&type=tools", + "logExtra": "{\"anchor_id\":0,\"anchor_name\":\"CapCut · Editing made easy\",\"anchor_title_detail\":\"None\",\"anchor_title_id\":21502485763,\"anchor_type\":\"TT_CAPCUT\",\"capability_key\":\"default\",\"ccep_vip\":0,\"if_race_trigger\":0,\"is_two_line\":0,\"landing_page_style_id\":21502486275,\"maker_source\":\"\",\"publish_key\":\"default\",\"template_id\":\"none\",\"tutorial_id\":\"none\",\"viamaker_anchor_capability_key_weight\":1,\"viamaker_anchor_style_idx\":-1,\"viamaker_anchor_style_source\":3,\"video_source\":0,\"video_type_id\":1}", + "description": "CapCut · Video Editor", + "thumbnail": { + "uri": "tiktok-obj/64x_Capcut3x.png", + "urlList": [ + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=SJ6q86bjtCoRArlVNS1DjDDFWoc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p19-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.image?dr=9627&x-expires=1783544400&x-signature=GOZUldk%2BrxINygKT0jwxeJz9VLQ%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5", + "https://p16-common-sign.tiktokcdn-us.com/tiktok-obj/64x_Capcut3x.png~tplv-tiktokx-origin.jpeg?dr=9627&x-expires=1783544400&x-signature=NBemEand1jjAoV81cQ1ANQzVqqc%3D&t=4d5b0474&ps=13740610&shp=81f88b70&shcp=9b759fb9&idc=useast5" + ], + "width": 64, + "height": 64 + }, + "extraInfo": { + "subtype": "" + } + } + ], + "collected": false, + "channelTags": [], + "item_control": { + "can_repost": true + }, + "IsAigc": false, + "AIGCDescription": "", + "backendSourceEventTracking": "", + "CategoryType": 111, + "textLanguage": "en", + "textTranslatable": true, + "authorStatsV2": { + "followerCount": "372600", + "followingCount": "0", + "heart": "8000000", + "heartCount": "8000000", + "videoCount": "279", + "diggCount": "223", + "friendCount": "0" + }, + "isReviewing": false, + "creatorAIComment": { + "hasAITopic": false, + "categoryList": [], + "eligibleVideo": false, + "notEligibleReason": 101 + } +} \ No newline at end of file From 2943d8b23cb1b03f5b97a07eecf9298b2fafe2d1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 18:21:44 +0200 Subject: [PATCH 029/160] feat(tiktok): tiktok.scrape capability + billing wire-up --- .../app/capabilities/core/billing.py | 2 + .../app/capabilities/core/types.py | 1 + .../app/capabilities/tiktok/__init__.py | 5 ++ .../capabilities/tiktok/scrape/__init__.py | 1 + .../capabilities/tiktok/scrape/definition.py | 23 +++++ .../capabilities/tiktok/scrape/executor.py | 48 +++++++++++ .../app/capabilities/tiktok/scrape/schemas.py | 86 +++++++++++++++++++ surfsense_backend/app/config/__init__.py | 3 + surfsense_backend/app/routes/__init__.py | 1 + .../unit/capabilities/tiktok/__init__.py | 0 .../capabilities/tiktok/scrape/__init__.py | 0 .../tiktok/scrape/test_executor.py | 79 +++++++++++++++++ .../tiktok/scrape/test_schemas.py | 45 ++++++++++ .../unit/capabilities/tiktok/test_registry.py | 23 +++++ 14 files changed, 317 insertions(+) create mode 100644 surfsense_backend/app/capabilities/tiktok/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/scrape/schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 71aa45c58..048797473 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -35,6 +35,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW", BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", + BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", } @@ -51,6 +52,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "review", BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", + BillingUnit.TIKTOK_VIDEO: "video", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index c87601832..b45641a2d 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -25,6 +25,7 @@ class BillingUnit(StrEnum): GOOGLE_MAPS_REVIEW = "google_maps_review" YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" + TIKTOK_VIDEO = "tiktok_video" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..ed3f32682 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -0,0 +1,5 @@ +"""``tiktok.*`` namespace: platform-native TikTok data verbs.""" + +from __future__ import annotations + +from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..9e5acc165 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/__init__.py @@ -0,0 +1 @@ +"""``tiktok.scrape``: public TikTok videos over the browser-driven scraper.""" diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py new file mode 100644 index 000000000..9f8632f3b --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.scrape`` capability registration (billed per video; see config +``TIKTOK_MICROS_PER_VIDEO``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +TIKTOK_SCRAPE = Capability( + name="tiktok.scrape", + description=( + "Scrape public TikTok videos. Use urls, profiles, hashtags, or " + "search_queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_SCRAPE) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/executor.py b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py new file mode 100644 index 000000000..7ef71486d --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/executor.py @@ -0,0 +1,48 @@ +"""``tiktok.scrape`` executor: verb input → scraper → TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import ( + TikTokAccessBlockedError, + TikTokScrapeInput, + scrape_tiktok, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_tiktok + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = TikTokScrapeInput( + startUrls=[{"url": url} for url in payload.urls], + profiles=payload.profiles, + hashtags=payload.hashtags, + searchQueries=payload.search_queries, + resultsPerPage=payload.results_per_page, + ) + emit_progress( + "starting", "Resolving TikTok targets", total=payload.max_items, unit="item" + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except TikTokAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"TikTok refused anonymous access: {exc}", + code="TIKTOK_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py new file mode 100644 index 000000000..697a94043 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -0,0 +1,86 @@ +"""``tiktok.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``TikTokScrapeInput`` +(``app/proprietary/platforms/tiktok``). The executor maps this to the full +scraper input; the scraper's ``TikTokVideoItem`` is reused verbatim as the +output element. Any TikTok URL kind (video, profile, hashtag, search) goes in +``urls``; ``profiles``/``hashtags``/``search_queries`` are typed shortcuts. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.tiktok import TikTokVideoItem + +MAX_TIKTOK_SOURCES = 20 +"""Per-call cap on each source list: bounds a synchronous request's fan-out.""" + +MAX_TIKTOK_ITEMS = 100 +"""Hard ceiling on items returned per call, regardless of the per-target count.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description=( + "TikTok URLs to scrape: a video, a profile (/@), a hashtag " + "(/tag/), or a search URL. Provide these OR profiles/hashtags/" + "search_queries (at least one source is required)." + ), + ) + profiles: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Profile usernames (with or without a leading '@').", + ) + hashtags: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Hashtag names to scrape, without the leading '#'.", + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_TIKTOK_SOURCES, + description="Search terms to run on TikTok.", + ) + results_per_page: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max videos to pull per profile/hashtag/search target.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total items to return across all sources.", + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ScrapeInput: + if not any((self.urls, self.profiles, self.hashtags, self.search_queries)): + raise ValueError( + "Provide at least one of 'urls', 'profiles', 'hashtags', or " + "'search_queries'." + ) + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable items for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per video returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned item = one billable unit.""" + return len(self.items) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 5d8e96881..c03ac3598 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -714,6 +714,9 @@ class Config: # Kept separate from the video rate so comments can be re-tuned toward the # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")) + # Browser-driven listings make TikTok heavier per item than the API-backed + # video meter, so it sits a touch above YouTube's video rate. + TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1a5b182b8..3c28fd65c 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends import app.capabilities.google_maps import app.capabilities.google_search import app.capabilities.reddit +import app.capabilities.tiktok import app.capabilities.web import app.capabilities.youtube # noqa: F401 from app.automations.api import router as automations_router diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py new file mode 100644 index 000000000..1bf07b0c3 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_executor.py @@ -0,0 +1,79 @@ +"""``tiktok.scrape`` executor: verb input → actor input mapping → typed items. + +Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's +own payload→TikTokScrapeInput mapping and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.scrape.executor import build_scrape_executor +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.tiktok import TikTokAccessBlockedError, TikTokScrapeInput + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + """Records the actor input + limit it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[TikTokScrapeInput, int | None]] = [] + + async def __call__( + self, actor_input: TikTokScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((actor_input, limit)) + return self._items + + +async def test_maps_urls_to_start_urls_and_wraps_items(): + scraper = _FakeScraper([{"id": "123", "text": "hello"}]) + execute = build_scrape_executor(scrape_fn=scraper) + + out = await execute(ScrapeInput(urls=["https://www.tiktok.com/@nasa/video/123"])) + + assert isinstance(out, ScrapeOutput) + assert len(out.items) == 1 + assert out.items[0].id == "123" + + (actor_input, _limit) = scraper.calls[0] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.tiktok.com/@nasa/video/123" + ] + + +async def test_forwards_typed_sources_and_limit(): + scraper = _FakeScraper([]) + execute = build_scrape_executor(scrape_fn=scraper) + + await execute( + ScrapeInput( + profiles=["nasa"], + hashtags=["food"], + search_queries=["cats"], + results_per_page=7, + max_items=25, + ) + ) + + (actor_input, limit) = scraper.calls[0] + assert actor_input.profiles == ["nasa"] + assert actor_input.hashtags == ["food"] + assert actor_input.searchQueries == ["cats"] + assert actor_input.resultsPerPage == 7 + # The outer collection limit is the caller's total-item cap. + assert limit == 25 + + +async def test_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input: TikTokScrapeInput, *, limit: int | None = None): + raise TikTokAccessBlockedError("all IPs refused") + + execute = build_scrape_executor(scrape_fn=_blocked) + + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(hashtags=["food"])) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py new file mode 100644 index 000000000..e7c978bcc --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -0,0 +1,45 @@ +"""``tiktok.scrape`` input guards: a source is required and the batch is bounded.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, + ScrapeInput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_source(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_accepts_urls_only(): + payload = ScrapeInput(urls=["https://www.tiktok.com/@nasa"]) + assert payload.profiles == [] + + +def test_accepts_hashtags_only(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.hashtags == ["food"] + + +def test_defaults_and_bounds(): + payload = ScrapeInput(hashtags=["food"]) + assert payload.max_items == 10 + assert payload.results_per_page == 10 + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=0) + with pytest.raises(ValidationError): + ScrapeInput(hashtags=["food"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_sources_than_the_cap(): + too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + ScrapeInput(hashtags=too_many) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py new file mode 100644 index 000000000..3a85fe87f --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -0,0 +1,23 @@ +"""The tiktok namespace registers its verb as one Capability the doors/agent read.""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + tiktok, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core import BillingUnit +from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput + +pytestmark = pytest.mark.unit + + +def test_tiktok_scrape_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.scrape") + + assert cap.name == "tiktok.scrape" + assert cap.input_schema is ScrapeInput + assert cap.output_schema is ScrapeOutput + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO From ed1c3a1f3d43cb77e214c88d87e1a793080ac9d8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:20:20 +0200 Subject: [PATCH 030/160] feat(tiktok): expose tiktok.scrape on MCP, chat subagent, playground --- .../agents/chat/multi_agent_chat/constants.py | 1 + .../subagents/builtins/tiktok/__init__.py | 1 + .../subagents/builtins/tiktok/agent.py | 43 +++++++++ .../subagents/builtins/tiktok/description.md | 2 + .../builtins/tiktok/system_prompt.md | 64 ++++++++++++++ .../builtins/tiktok/tools/__init__.py | 0 .../subagents/builtins/tiktok/tools/index.py | 27 ++++++ .../multi_agent_chat/subagents/registry.py | 4 + .../test_subagent_composition.py | 1 + .../mcp_server/features/scrapers/__init__.py | 4 +- .../features/scrapers/platforms/tiktok.py | 88 +++++++++++++++++++ surfsense_mcp/mcp_server/selfcheck.py | 1 + surfsense_web/lib/playground/catalog.ts | 7 ++ .../lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/tiktok.svg | 6 ++ 15 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py create mode 100644 surfsense_web/public/connectors/tiktok.svg diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index b132618d7..c396fceaf 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -36,6 +36,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_maps": frozenset(), "google_search": frozenset(), "reddit": frozenset(), + "tiktok": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py new file mode 100644 index 000000000..ec7d5955f --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/__init__.py @@ -0,0 +1 @@ +"""``tiktok`` builtin subagent: structured public TikTok videos and listings.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py new file mode 100644 index 000000000..2c7dd014b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/agent.py @@ -0,0 +1,43 @@ +"""``tiktok`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured data from public TikTok videos, hashtags, and searches." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md new file mode 100644 index 000000000..fc13d864d --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -0,0 +1,2 @@ +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL. Also compares fresh TikTok results against earlier findings in this chat. +Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, or scrape a specific video URL. Triggers include "search TikTok for X", "trending TikTok videos about X", "videos with #X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md new file mode 100644 index 000000000..49c20164e --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -0,0 +1,64 @@ +You are the SurfSense TikTok sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live TikTok data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it. + + + +- `tiktok_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. +- Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. +- Profiles: a creator's `profiles` feed can come back empty — TikTok restricts the profile video endpoint. Prefer `hashtags`, `search_queries`, or a direct video URL, and treat an empty profile result as a known limit, not a failure to retry endlessly. +- Controlling volume: use `max_items` for the total cap and `results_per_page` per target. +- Requested counts: `max_items` defaults to only 10 — when the task asks for N videos, set `max_items` and `results_per_page` above N. A call that caps below the target can never satisfy it. +- Batch multiple hashtags or search terms into one call rather than many single-term calls. + +- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent captions, URLs, authors, or counts. + + + +- Do not read arbitrary web pages — that belongs to the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Reddit belongs to the Reddit specialist; YouTube belongs to the YouTube specialist; Google results belong to the Google Search specialist. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable hashtag, query, or URL — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: one entry per distinct video or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py new file mode 100644 index 000000000..e790d44f0 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -0,0 +1,27 @@ +"""``tiktok`` sub-agent tools: the TikTok scrape capability verb.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE + +NAME = "tiktok" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [TIKTOK_SCRAPE] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 34895a514..71c2e6f3d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -33,6 +33,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( build_subagent as build_reddit_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( + build_subagent as build_tiktok_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -84,6 +87,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "memory": build_memory_subagent, "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, + "tiktok": build_tiktok_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index c3ad04250..66e8f28db 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -37,6 +37,7 @@ _EXPECTED_SUBAGENTS = frozenset( "memory", "onedrive", "reddit", + "tiktok", "web_crawler", "youtube", } diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index dfa2f3ab2..c0e7dea07 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -13,9 +13,9 @@ from mcp.server.fastmcp import FastMCP from ...core.client import SurfSenseClient from ...core.workspace_context import WorkspaceContext from . import run_history -from .platforms import google_maps, google_search, reddit, web, youtube +from .platforms import google_maps, google_search, reddit, tiktok, web, youtube -_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) +_REGISTRARS = (web, google_search, reddit, youtube, tiktok, google_maps, run_history) def register( diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py new file mode 100644 index 000000000..1e5f472ee --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -0,0 +1,88 @@ +"""TikTok scraper tool.""" + +from __future__ import annotations + +from typing import Annotated + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the TikTok tool.""" + + @mcp.tool( + name="surfsense_tiktok_scrape", + title="Search or scrape TikTok", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="TikTok URLs: a video, a profile " + "('https://www.tiktok.com/@nasa'), a hashtag " + "('https://www.tiktok.com/tag/food'), or a search URL. Provide " + "urls OR profiles/hashtags/search_queries." + ), + ] = None, + profiles: Annotated[ + list[str] | None, + Field( + description="Profile usernames to scrape, with or without a " + "leading '@', e.g. ['nasa']." + ), + ] = None, + hashtags: Annotated[ + list[str] | None, + Field( + description="Hashtag names to scrape, without the '#', e.g. " + "['food']." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field(description="Terms to search TikTok for, e.g. ['cooking']."), + ] = None, + results_per_page: Annotated[ + int, + Field(ge=1, description="Max videos per profile/hashtag/search target."), + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum videos to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape public TikTok videos. + + Use this for ANY TikTok research — a creator's videos, a hashtag feed, + a search, or a specific video URL — instead of a generic web search. + Returns videos with text, author, stats, music, and the web URL. + Example: hashtags=['food'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="scrape", + payload={ + "urls": urls, + "profiles": profiles, + "hashtags": hashtags, + "search_queries": search_queries, + "results_per_page": results_per_page, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index 20224c173..e5ac6bd2b 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -23,6 +23,7 @@ EXPECTED_TOOLS = { "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_tiktok_scrape", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_list_scraper_runs", diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 9b508228a..4503662f8 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -3,6 +3,7 @@ import { GoogleMapsIcon, GoogleSearchIcon, RedditIcon, + TikTokIcon, WebIcon, YouTubeIcon, } from "./platform-icons"; @@ -48,6 +49,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { name: "youtube.comments", verb: "comments", label: "Comments" }, ], }, + { + id: "tiktok", + label: "TikTok", + icon: TikTokIcon, + verbs: [{ name: "tiktok.scrape", verb: "scrape", label: "Scrape" }], + }, { id: "google_maps", label: "Google Maps", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index e7a443c2f..474bff317 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -24,6 +24,7 @@ function brandIcon(src: string, alt: string) { export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); +export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/tiktok.svg b/surfsense_web/public/connectors/tiktok.svg new file mode 100644 index 000000000..c3c3fe06e --- /dev/null +++ b/surfsense_web/public/connectors/tiktok.svg @@ -0,0 +1,6 @@ + + + + + + From ea6befdf595e91613b40d3c5f26f2f4dad116df5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:21:29 +0200 Subject: [PATCH 031/160] docs(tiktok): native connector page + nav card --- .../content/docs/connectors/native/index.mdx | 7 ++- .../content/docs/connectors/native/meta.json | 2 +- .../content/docs/connectors/native/tiktok.mdx | 45 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/tiktok.mdx diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx index 5c5d24444..f915b5533 100644 --- a/surfsense_web/content/docs/connectors/native/index.mdx +++ b/surfsense_web/content/docs/connectors/native/index.mdx @@ -1,6 +1,6 @@ --- title: Native Connectors -description: SurfSense's built-in scraper APIs for Reddit, YouTube, Google Maps, Google Search, and the web +description: SurfSense's built-in scraper APIs for Reddit, YouTube, TikTok, Google Maps, Google Search, and the web --- import { Card, Cards } from 'fumadocs-ui/components/card'; @@ -18,6 +18,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform, description="Videos, channels, playlists, subtitles, and comments" href="/docs/connectors/native/youtube" /> + `), a hashtag (`/tag/`), or a search URL (max 20 sources per call) | +| `profiles` | — | Profile usernames, with or without a leading `@` | +| `hashtags` | — | Hashtag names, without the `#` | +| `search_queries` | — | Search terms to run on TikTok | +| `results_per_page` | `10` | Max videos per profile/hashtag/search target | +| `max_items` | `10` | Max total videos returned across all sources (hard cap 100) | + +## Example + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "hashtags": ["food"], + "max_items": 25 + }' +``` + +The response is `{ "items": [...] }` — one item per video. Billing is per returned item. + + +Video, hashtag, and search targets are the reliable paths. TikTok restricts the profile video endpoint for automated clients, so a `profiles` target can return no items even when the account is public — prefer hashtags, search, or a direct video URL. + + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok → Scrape** in your workspace. From 5f9472b122157c3b9ab8e590ae26bcdd2f0cfea9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:30:38 +0200 Subject: [PATCH 032/160] feat(tiktok): advertise tiktok across agent prompts, MCP instructions, docs --- .../main_agent/system_prompt/prompts/identity/private.md | 6 +++--- .../main_agent/system_prompt/prompts/identity/team.md | 6 +++--- .../main_agent/system_prompt/prompts/kb_first.md | 5 +++-- .../main_agent/system_prompt/prompts/routing.md | 1 + surfsense_mcp/README.md | 1 + surfsense_mcp/mcp_server/server.py | 5 +++-- surfsense_web/content/docs/how-to/mcp-server.mdx | 4 ++-- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md index 378c052d4..0c7e4071c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md @@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put that intelligence to work alongside their own knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the - web crawler return structured, current platform data (posts, comments, - transcripts, reviews, SERPs, full page content). +- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search, + and the web crawler return structured, current platform data (posts, + comments, transcripts, videos, reviews, SERPs, full page content). - **The user's own context** — their knowledge base, connected apps, and persistent memory. - **Deliverables** — reports, podcasts, and presentations built from what the diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md index 2765c06b7..6c1076a89 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md @@ -6,9 +6,9 @@ reviews are moving, and what is being said across the open web — and to put that intelligence to work alongside the team's shared knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live market data** — Reddit, YouTube, Google Maps, Google Search, and the - web crawler return structured, current platform data (posts, comments, - transcripts, reviews, SERPs, full page content). +- **Live market data** — Reddit, YouTube, TikTok, Google Maps, Google Search, + and the web crawler return structured, current platform data (posts, + comments, transcripts, videos, reviews, SERPs, full page content). - **The team's own context** — its shared knowledge base, connected apps, and persistent team memory. - **Deliverables** — reports, podcasts, and presentations built from what the diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index c69ed400c..ef753b93c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -1,8 +1,9 @@ CRITICAL — ground factual answers in what you actually receive this turn: - **live platform data** via the market specialists — - `task(reddit, ...)`, `task(youtube, ...)`, `task(google_maps, ...)`, - `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about + `task(reddit, ...)`, `task(youtube, ...)`, `task(tiktok, ...)`, + `task(google_maps, ...)`, `task(google_search, ...)`, + `task(web_crawler, ...)`. Anything about competitors, markets, rankings, reviews, or audience sentiment is answered from what these return **this turn**, never from your training data: your general knowledge of companies, prices, and rankings is stale by definition, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index eec860f3c..fb818cabc 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -31,6 +31,7 @@ bounded fan-out (≤20 sites) the user already requested. about a brand, product, or topic is answered from the platform where they say it — `task(reddit, …)` for community discussion and threads, `task(youtube, …)` for video content, transcripts, and comment sections, +`task(tiktok, …)` for short-form video trends by hashtag or search, `task(google_maps, …)` for customer reviews of physical businesses. Web search only finds articles *about* the conversation; the platform specialists return the conversation itself, structured and current. For diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index c18a857e1..5bac5f8b2 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -21,6 +21,7 @@ Connect it two ways: **Scrapers (all platforms)** - `surfsense_web_crawl`, `surfsense_google_search`, `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, + `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews` - `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` — retrieve past results in full (useful when a large result was truncated inline) diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 8ea9bc919..c0398ed50 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -36,8 +36,9 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " - "communities), YouTube (videos, transcripts, comments), Google " - "Maps (places, reviews), Google Search results, or reading " + "communities), YouTube (videos, transcripts, comments), TikTok " + "(videos by hashtag, search, or URL), Google Maps (places, " + "reviews), Google Search results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " "surfsense_get_scraper_run." diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index f5f419219..67b218d58 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 19 native, typed tools: every scraper (Reddit, YouTube, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From f731d371abd803514735b73d36b19b2205496875 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 8 Jul 2026 20:36:46 +0200 Subject: [PATCH 033/160] feat(tiktok): add TikTok SEO landing page and site-wide marketing parity --- surfsense_web/app/(home)/mcp-server/page.tsx | 13 +- .../components/homepage/hero-section.tsx | 2 +- .../components/homepage/persona-paths.tsx | 2 +- .../components/pricing/pricing-section.tsx | 4 +- surfsense_web/components/seo/json-ld.tsx | 4 +- .../content/docs/connectors/index.mdx | 2 +- .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/reddit.tsx | 2 +- .../lib/connectors-marketing/tiktok.tsx | 277 ++++++++++++++++++ .../lib/connectors-marketing/youtube.tsx | 2 +- 10 files changed, 297 insertions(+), 13 deletions(-) create mode 100644 surfsense_web/lib/connectors-marketing/tiktok.tsx diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 5e4961b1e..22c867c73 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -93,6 +93,7 @@ const TOOL_GROUPS = [ "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_tiktok_scrape", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", @@ -127,7 +128,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, TikTok, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -215,8 +216,9 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Google Maps, Google Search, and the open web, and search, - read, and write your knowledge base. One API key, typed tools, pay as you go. + scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, and + search, read, and write your knowledge base. One API key, typed tools, pay as you + go.

+ diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 89d1e46f5..febe8a62b 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -719,7 +719,7 @@ export function HeroSection() { > SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market with live data from platforms - like Reddit, YouTube, Google Maps, Google Search, and the open web. + like Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/homepage/persona-paths.tsx b/surfsense_web/components/homepage/persona-paths.tsx index 9c03c5667..50eadba0f 100644 --- a/surfsense_web/components/homepage/persona-paths.tsx +++ b/surfsense_web/components/homepage/persona-paths.tsx @@ -35,7 +35,7 @@ const PATHS: { eyebrow: "For developers & agents", title: "The whole platform is programmable", description: - "Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.", + "Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.", links: [ { label: "Read the docs", href: "/docs" }, { label: "SurfSense MCP server", href: "/mcp-server" }, diff --git a/surfsense_web/components/pricing/pricing-section.tsx b/surfsense_web/components/pricing/pricing-section.tsx index 74c76ad36..1af296e57 100644 --- a/surfsense_web/components/pricing/pricing-section.tsx +++ b/surfsense_web/components/pricing/pricing-section.tsx @@ -35,7 +35,7 @@ const demoPlans = [ billingText: "Your first $5 of credit is free. No subscription, ever", features: [ "$5 of free credit to start, one balance for everything", - "Platform connectors: Reddit, YouTube, Google Maps, Google Search, and the open web", + "Platform connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web", "Call every connector as a REST API with your key or through the MCP server", "Pay per item returned and per page crawled. Failed calls are never billed", "Premium models like GPT-5.5, Claude Sonnet 5, Gemini 3.1 Pro billed at provider cost", @@ -95,7 +95,7 @@ const faqData: FAQSection[] = [ { question: "How does Pay As You Go work?", answer: - "There is no monthly subscription. Start with $5 of free credit, and when you need more, add any amount. $1 buys exactly $1 of credit, added to your balance immediately. You can enable automatic refills when your balance runs low, and turn them off any time." + "There is no monthly subscription. Start with $5 of free credit, and when you need more, add any amount. $1 buys exactly $1 of credit, added to your balance immediately. You can enable automatic refills when your balance runs low, and turn them off any time.", }, { question: "What happens if I run out of credit?", diff --git a/surfsense_web/components/seo/json-ld.tsx b/surfsense_web/components/seo/json-ld.tsx index c4b3ec09c..063f6a0c0 100644 --- a/surfsense_web/components/seo/json-ld.tsx +++ b/surfsense_web/components/seo/json-ld.tsx @@ -76,11 +76,11 @@ export function SoftwareApplicationJsonLd() { "Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go", }, description: - "SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, Google Maps, Google Search, and the open web, through one API or MCP server.", + "SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, through one API or MCP server.", url: "https://www.surfsense.com", downloadUrl: "https://github.com/MODSetter/SurfSense/releases", featureList: [ - "Platform-native connectors: Reddit, YouTube, Google Maps, Google Search, Web Crawl", + "Platform-native connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, Web Crawl", "MCP server that exposes every connector as a native agent tool", "Agent harness with retries, structured output, and credit metering", "Competitor, brand, and rank monitoring with briefs and alerts", diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 1ee9209e6..ab136c372 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, TikTok, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> Date: Wed, 8 Jul 2026 23:14:50 +0200 Subject: [PATCH 034/160] feat(tiktok): emit graceful ErrorItem for blocked/empty listings Profile and search feeds are trust-gated: an anonymous headless session gets an empty item_list (profile) or no results XHR (search), while hashtag feeds load. A zero-item listing now yields one honest ErrorItem (errorCode="no_items") instead of vanishing silently, and ErrorItems are excluded from billing so a blocked target is surfaced but never charged. --- .../app/capabilities/tiktok/scrape/schemas.py | 5 +- .../platforms/tiktok/flows/listing.py | 18 ++++++ .../scripts/e2e_tiktok_scrape.py | 59 +++++++++++++++---- .../tiktok/scrape/test_schemas.py | 13 ++++ .../platforms/tiktok/test_orchestrator.py | 15 +++++ 5 files changed, 96 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index 697a94043..2f5005353 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -82,5 +82,6 @@ class ScrapeOutput(BaseModel): @property def billable_units(self) -> int: - """One returned item = one billable unit.""" - return len(self.items) + """One returned video = one billable unit; ErrorItems (``errorCode`` set, + for blocked/empty targets) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py index 44fcc33e2..3e930271f 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/listing.py @@ -12,9 +12,19 @@ from typing import Any from ..extraction import parse_video from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem from ..targets.types import TikTokTarget from . import FetchListingFn +# Profile and search feeds are trust-gated: an anonymous headless session gets an +# empty body (profile) or no results XHR (search), while hashtag feeds load. We +# can't tell "genuinely empty" from "blocked" here, so a zero-item listing emits +# one honest ErrorItem instead of vanishing silently. +_EMPTY_LISTING_MESSAGE = ( + "No videos returned. The target may be empty/private/removed, or TikTok " + "withheld this feed from anonymous access (common for profiles and search)." +) + async def iter_listing( target: TikTokTarget, *, cap: int, fetch_listing: FetchListingFn @@ -35,3 +45,11 @@ async def iter_listing( emitted += 1 if emitted >= cap: return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_LISTING_MESSAGE, + errorCode="no_items", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 22bebff96..eedeaacac 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,11 +7,14 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile listing via the stealth browser (soft-blocked by TikTok; - expected empty until a stronger anti-detection path exists). + Stage 2 — profile via the full pipeline: TikTok soft-blocks the anonymous + profile feed, so this asserts graceful degradation — real videos OR + a single honest ErrorItem, never a silent empty. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. + Stage 6 — search via the full pipeline: same graceful-degrade contract as + profile (results feed doesn't load for anonymous sessions). On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -40,9 +43,11 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): _FIXTURES = _BACKEND_ROOT / "tests" / "fixtures" / "tiktok" -# Evergreen public targets: a regular high-volume creator and a broad hashtag. +# Evergreen public targets: a regular high-volume creator, a broad hashtag, and +# a common search term. _PROFILE = "nasa" _HASHTAG = "food" +_SEARCH = "meal prep" _COUNT = 5 @@ -93,17 +98,24 @@ async def stage_proxy() -> bool: async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: - _hr(f"STAGE 2 — profile listing (browser): @{_PROFILE}") - from app.proprietary.platforms.tiktok.session import fetch_item_list + _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - url = f"https://www.tiktok.com/@{_PROFILE}" - raw = await fetch_item_list(url, _COUNT) - ok = _check( - "captured itemStructs from item_list XHRs", - len(raw) > 0 and isinstance(raw[0], dict) and bool(raw[0].get("id")), - f"{len(raw)} struct(s)", + # TikTok soft-blocks the profile feed for anonymous headless sessions + # (empty 200). The shipped contract is a single honest ErrorItem, not a + # silent empty. This stage verifies that graceful degradation, and passes if + # EITHER real videos come back (session got trusted) OR an ErrorItem does. + items = await scrape_tiktok( + TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) - return ok, raw + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "profile yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items async def stage_blob_video(video_url: str) -> tuple[bool, dict[str, Any] | None]: @@ -167,6 +179,26 @@ async def stage_pipeline() -> bool: return ok +async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 6 — search listing graceful-degrade: {_SEARCH!r}") + from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok + + # The search results feed doesn't load for anonymous headless sessions + # (results XHR never fires on a cold deep-link). Same contract as profile: + # verify a graceful ErrorItem instead of a silent empty. + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=[_SEARCH], resultsPerPage=_COUNT), limit=_COUNT + ) + has_video = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_items" for it in items) + ok = _check( + "search yields videos or a graceful ErrorItem (never silent empty)", + has_video or has_error, + f"{len(items)} item(s); video={has_video} error={has_error}", + ) + return ok, items + + async def main() -> int: print("TikTok scraper functional e2e — live network + proxy + browser") results: dict[str, bool] = {} @@ -185,6 +217,9 @@ async def main() -> int: else: print("\n [SKIP] Stage 3 — no captured struct to build a video URL") + ok_search, _ = await stage_search_listing() + results["Stage 6 search listing"] = ok_search + ok_profile, _ = await stage_profile_listing() results["Stage 2 profile listing"] = ok_profile results["Stage 5 pipeline"] = await stage_pipeline() diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py index e7c978bcc..f9cec8c3d 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/scrape/test_schemas.py @@ -9,6 +9,7 @@ from app.capabilities.tiktok.scrape.schemas import ( MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES, ScrapeInput, + ScrapeOutput, ) pytestmark = pytest.mark.unit @@ -43,3 +44,15 @@ def test_rejects_more_sources_than_the_cap(): too_many = [f"tag{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] with pytest.raises(ValidationError): ScrapeInput(hashtags=too_many) + + +def test_error_items_are_not_billed(): + # Real videos count; ErrorItems (blocked/empty targets) are surfaced free. + out = ScrapeOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "nasa", "error": "blocked"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 8520a1d85..777ed49de 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -95,3 +95,18 @@ async def test_listing_dedupes_then_caps_per_target(): TikTokScrapeInput(hashtags=["x"], resultsPerPage=2), fetch_listing=fake_listing ) assert [i["id"] for i in items] == ["1", "2"] + + +async def test_empty_listing_emits_error_item(): + # A trust-gated/empty feed (0 videos) must surface one honest ErrorItem, + # tagged with errorCode, rather than vanishing silently. + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5), + fetch_listing=fake_listing, + ) + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" + assert items[0]["input"] == "nasa" From 3bd864ed958df144983b27fb6e946fab715bb447 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:08:10 +0530 Subject: [PATCH 035/160] feat(env): add Instagram scraping configuration parameters to .env.example --- surfsense_backend/.env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6bbf32d3f..a030b75aa 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -290,6 +290,8 @@ MICROS_PER_PAGE=1000 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# INSTAGRAM_SCRAPE_MICROS_PER_ITEM=3500 +# INSTAGRAM_SCRAPE_MICROS_PER_COMMENT=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 From 8486a4f91471539f966c330edcf2b9496e4315bb Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:08:23 +0530 Subject: [PATCH 036/160] feat(constants): add Instagram to SUBAGENT_TO_REQUIRED_CONNECTOR_MAP for enhanced multi-agent chat capabilities --- surfsense_backend/app/agents/chat/multi_agent_chat/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index b132618d7..40e8dd6d5 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -36,6 +36,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_maps": frozenset(), "google_search": frozenset(), "reddit": frozenset(), + "instagram": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", From f204db7bce422ebac048b035936cd453f0c93e50 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:44 +0530 Subject: [PATCH 037/160] feat(billing): add INSTAGRAM_ITEM and INSTAGRAM_COMMENT units --- surfsense_backend/app/capabilities/core/types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index c87601832..e0fef5379 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -25,6 +25,8 @@ class BillingUnit(StrEnum): GOOGLE_MAPS_REVIEW = "google_maps_review" YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" + INSTAGRAM_ITEM = "instagram_item" + INSTAGRAM_COMMENT = "instagram_comment" class BillableInput(Protocol): From 43660ee51bf346a547fc1a9e970ac46a94b1ac16 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:53 +0530 Subject: [PATCH 038/160] feat(billing): map Instagram units to rate keys and display nouns --- surfsense_backend/app/capabilities/core/billing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 71aa45c58..7c2fbba9f 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -35,6 +35,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW", BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", + BillingUnit.INSTAGRAM_ITEM: "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", + BillingUnit.INSTAGRAM_COMMENT: "INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", } @@ -51,6 +53,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "review", BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", + BillingUnit.INSTAGRAM_ITEM: "item", + BillingUnit.INSTAGRAM_COMMENT: "comment", } From 51c2dfdfe69e3362e7a3d9833487e55eaf6700fa Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:59 +0530 Subject: [PATCH 039/160] feat(config): add Instagram per-item and per-comment scrape rates --- surfsense_backend/app/config/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 74f7ec6c4..23411d979 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -719,6 +719,14 @@ class Config: # Kept separate from the video rate so comments can be re-tuned toward the # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")) + INSTAGRAM_SCRAPE_MICROS_PER_ITEM = int( + os.getenv("INSTAGRAM_SCRAPE_MICROS_PER_ITEM", "3500") + ) + # Kept separate from the item rate so comments can be re-tuned toward the + # cheaper per-comment market without touching post/reel pricing. + INSTAGRAM_SCRAPE_MICROS_PER_COMMENT = int( + os.getenv("INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", "1500") + ) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. From 842a0e666d7e3befea5a8bea506b29cf821d85ec Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:11 +0530 Subject: [PATCH 040/160] feat(instagram): add platform input/output schemas --- .../platforms/instagram/schemas.py | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/schemas.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py new file mode 100644 index 000000000..2175744a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -0,0 +1,187 @@ +# ruff: noqa: N815 +"""Input/output models for the Instagram scraper. + +The models mirror the public Instagram scraper actor spec so the endpoint can be +a drop-in: the input accepts the full documented surface, and every output field +is emitted (``None``/``[]`` when the anonymous web endpoints cannot source it +yet) so the contract expands additively — the same rule the Google Search and +YouTube models follow. + +**Anonymous only.** There is deliberately **no** authentication field on the +input (no username/password/token/cookie/``login*``) — the scraper holds only +Instagram's anonymous web-session cookies (``csrftoken``/``mid``) and can never +log in. Anything auth-shaped a caller sends lands in ``extra`` and is ignored. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +InstagramResultsType = Literal[ + "posts", "details", "comments", "reels", "mentions", "stories" +] +InstagramSearchType = Literal["hashtag", "profile", "place", "user"] +InstagramDetailKind = Literal["profile", "hashtag", "place"] + + +class InstagramScrapeInput(BaseModel): + """Instagram scraper input surface (anonymous, no auth fields). + + Field names mirror the public actor spec verbatim. ``resultsLimit`` / + ``searchLimit`` are collector policy applied by :func:`scrape_instagram`, + NOT ceilings baked into the streaming flows. Fields the acquisition layer + doesn't source yet are still accepted via ``extra="allow"`` for parity. + """ + + model_config = ConfigDict(extra="allow") + resultsType: InstagramResultsType = "posts" + directUrls: list[str] = Field(default_factory=list) + resultsLimit: int | None = Field(default=None, ge=1) + onlyPostsNewerThan: str | None = None + search: str | None = None + searchType: InstagramSearchType = "hashtag" + searchLimit: int | None = Field(default=None, ge=1, le=250) + addParentData: bool = False + skipPinnedPosts: bool = False + isNewestComments: bool = False + includeNestedComments: bool = False + addProfileStatistics: bool = False + + +class _ItemBase(BaseModel): + """Common error / provenance fields carried on every output item. + + Errors surface as item-level fields (never exceptions) so a partial run + still returns the items it could source, mirroring the actor's shape. + """ + + model_config = ConfigDict(extra="allow") + inputUrl: str | None = None + error: str | None = None + errorDescription: str | None = None + requestErrorMessages: list[str] = Field(default_factory=list) + + def to_output(self) -> dict[str, Any]: + """Serialize to the flat output dict shape (keeps extras).""" + return self.model_dump(exclude_none=False) + + +class InstagramMediaItem(_ItemBase): + """A post / reel / mention. One flat schema per the actor FAQ.""" + + id: str | None = None + type: Literal["Image", "Video", "Sidecar"] | None = None + shortCode: str | None = None + caption: str | None = None + hashtags: list[str] = Field(default_factory=list) + mentions: list[str] = Field(default_factory=list) + url: str | None = None + commentsCount: int | None = None + firstComment: str | None = None + latestComments: list[dict[str, Any]] = Field(default_factory=list) + dimensionsHeight: int | None = None + dimensionsWidth: int | None = None + displayUrl: str | None = None + images: list[str] = Field(default_factory=list) + videoUrl: str | None = None + alt: str | None = None + likesCount: int | None = None + videoViewCount: int | None = None + videoPlayCount: int | None = None + reshareCount: int | None = None + timestamp: str | None = None + childPosts: list[dict[str, Any]] = Field(default_factory=list) + ownerUsername: str | None = None + ownerId: str | None = None + ownerFullName: str | None = None + isPinned: bool | None = None + productType: str | None = None + videoDuration: float | None = None + paidPartnership: bool | None = None + taggedUsers: list[dict[str, Any]] = Field(default_factory=list) + musicInfo: dict[str, Any] | None = None + coauthorProducers: list[dict[str, Any]] = Field(default_factory=list) + locationName: str | None = None + locationId: str | None = None + isCommentsDisabled: bool | None = None + dataSource: dict[str, Any] | None = None + + +class InstagramComment(_ItemBase): + """A comment on a post / reel.""" + + id: str | None = None + postUrl: str | None = None + commentUrl: str | None = None + text: str | None = None + ownerUsername: str | None = None + ownerProfilePicUrl: str | None = None + timestamp: str | None = None + repliesCount: int | None = None + replies: list[dict[str, Any]] = Field(default_factory=list) + likesCount: int | None = None + owner: dict[str, Any] | None = None + + +class InstagramProfile(_ItemBase): + """A profile detail item (``detailKind = "profile"``).""" + + detailKind: Literal["profile"] = "profile" + id: str | None = None + username: str | None = None + url: str | None = None + fullName: str | None = None + biography: str | None = None + externalUrl: str | None = None + followersCount: int | None = None + followsCount: int | None = None + postsCount: int | None = None + highlightReelCount: int | None = None + igtvVideoCount: int | None = None + isBusinessAccount: bool | None = None + businessCategoryName: str | None = None + private: bool | None = None + verified: bool | None = None + profilePicUrl: str | None = None + profilePicUrlHD: str | None = None + relatedProfiles: list[dict[str, Any]] = Field(default_factory=list) + latestPosts: list[dict[str, Any]] = Field(default_factory=list) + statistics: dict[str, Any] | None = None + + +class InstagramHashtag(_ItemBase): + """A hashtag detail item (``detailKind = "hashtag"``).""" + + detailKind: Literal["hashtag"] = "hashtag" + id: str | None = None + name: str | None = None + url: str | None = None + postsCount: int | None = None + topPosts: list[dict[str, Any]] = Field(default_factory=list) + posts: list[dict[str, Any]] = Field(default_factory=list) + related: list[dict[str, Any]] = Field(default_factory=list) + searchTerm: str | None = None + searchSource: str | None = None + + +class InstagramPlace(_ItemBase): + """A place detail item (``detailKind = "place"``).""" + + detailKind: Literal["place"] = "place" + name: str | None = None + location_id: str | None = None + slug: str | None = None + lat: float | None = None + lng: float | None = None + location_address: str | None = None + location_city: str | None = None + location_zip: str | None = None + phone: str | None = None + website: str | None = None + category: str | None = None + media_count: int | None = None + posts: list[dict[str, Any]] = Field(default_factory=list) + searchTerm: str | None = None + searchSource: str | None = None From 8dc8ad6a36aa9a398ebd682a404b947e13512797 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:15 +0530 Subject: [PATCH 041/160] feat(instagram): add URL classifier and normalizer --- .../platforms/instagram/url_resolver.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py new file mode 100644 index 000000000..596b85c2c --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py @@ -0,0 +1,96 @@ +"""Classify and normalize an Instagram URL into a scrape job. + +Covers the supported ``directUrls`` shapes: a profile, a post (``/p/``), a reel +(``/reel/``), a hashtag (``/explore/tags/``), and a place +(``/explore/locations/``), plus bare profile IDs. Non-Instagram hosts resolve to +``None`` so the orchestrator can skip them. Mirrors the frozen ``ResolvedUrl`` +dataclass shape of ``../reddit/url_resolver.py``. + +Normalization rules (from the reference spec): +- ``_u/`` and ``/profilecard/`` segments are stripped. +- Story URLs (``/stories//...``) reduce to the profile. +- Location URLs are valid with the numeric ID alone (no trailing slug). +- Numeric post-ID URLs are only valid for the ``comments`` flow; elsewhere the + shortCode form is required, so a numeric-ID URL resolves with + ``numeric_post_id`` set and callers reject it outside comments. +- ``share/`` redirect resolution is handled at fetch time (network), not here. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal +from urllib.parse import urlparse + +ResolvedKind = Literal["profile", "post", "reel", "hashtag", "place"] + +_INSTAGRAM_HOSTS = frozenset( + {"m.instagram.com", "www.instagram.com", "instagram.com"} +) +_STRIP_SEGMENTS = frozenset({"_u", "profilecard"}) +_RESERVED = frozenset( + {"p", "s", "tv", "reel", "reels", "share", "explore", "stories", "accounts"} +) + + +@dataclass(frozen=True) +class ResolvedUrl: + """A classified Instagram target: the kind, its key, and the source URL.""" + + kind: ResolvedKind + value: str + url: str + slug: str | None = None + numeric_post_id: bool = False + + +def _is_instagram_host(hostname: str | None) -> bool: + if not hostname: + return False + return hostname.lower() in _INSTAGRAM_HOSTS + + +def _segments(url: str) -> list[str]: + parsed = urlparse(url) + if not _is_instagram_host(parsed.hostname): + return [] + if not parsed.path: + return [] + return [s for s in parsed.path.split("/") if s and s not in _STRIP_SEGMENTS] + + +def resolve_url(url: str) -> ResolvedUrl | None: + """Classify an Instagram URL into a scrape job, or ``None`` if unrecognized. + + A bare token with no ``http`` prefix and no ``/`` is treated as a profile ID + (the reference accepts bare profile handles alongside full URLs). + """ + if "instagram.com" not in url.lower(): + token = url.strip().lstrip("@") + if token and "/" not in token and "." not in token: + return ResolvedUrl( + "profile", token, f"https://www.instagram.com/{token}/" + ) + segments = _segments(url) + if not segments: + return None + head = segments[0] + if head == "explore" and len(segments) >= 3 and segments[1] == "tags": + return ResolvedUrl("hashtag", segments[2], url) + if head == "explore" and len(segments) >= 3 and segments[1] == "locations": + slug = segments[3] if len(segments) >= 4 else None + return ResolvedUrl("place", segments[2], url, slug=slug) + if head == "p" and len(segments) >= 2: + code = segments[1] + return ResolvedUrl("post", code, url, numeric_post_id=code.isdigit()) + if head in ("reel", "reels") and len(segments) >= 2: + code = segments[1] + return ResolvedUrl("reel", code, url, numeric_post_id=code.isdigit()) + if head == "stories" and len(segments) >= 2: + user = segments[1] + return ResolvedUrl( + "profile", user, f"https://www.instagram.com/{user}/" + ) + if head not in _RESERVED: + return ResolvedUrl("profile", head, url) + return None From 2d2900100483798940d84c8a806071fb27b810a5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:27 +0530 Subject: [PATCH 042/160] feat(instagram): add web-JSON parsers --- .../platforms/instagram/parsers.py | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py new file mode 100644 index 000000000..96d66a2c5 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -0,0 +1,223 @@ +"""Pure JSON -> item mapping for the Instagram scraper. + +Framework-agnostic and I/O-free so it can be unit-tested against captured +fixtures. Every function takes a raw Instagram web-JSON node and returns a plain +dict shaped like the public actor spec — no network, no proxy, no ``scrapedAt`` +stamp (the orchestrator adds provenance so these stay deterministic under +fixture tests). + +Instagram's web JSON nests media under GraphQL-style ``edge_*`` containers +(``edge_media_to_caption``, ``edge_liked_by``, ...) with ``taken_at_timestamp`` +epoch seconds. These parsers flatten that into the actor's camelCase item shape. +Fields the anonymous endpoints don't expose are left unset (``None``/``[]``) so +parity is additive. +""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from typing import Any + +_BASE = "https://www.instagram.com" +_HASHTAG_RE = re.compile(r"#(\w+)") +_MENTION_RE = re.compile(r"@([\w.]+)") +_TYPE_MAP = { + "GraphImage": "Image", + "GraphVideo": "Video", + "GraphSidecar": "Sidecar", + "XDTGraphImage": "Image", + "XDTGraphVideo": "Video", + "XDTGraphSidecar": "Sidecar", +} + + +def _int(value: Any) -> int | None: + """Coerce to int, or ``None`` (never coerces bools).""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + return None + + +def _utc_from_sec(value: Any) -> str | None: + """Epoch seconds -> millisecond ISO string, or ``None``.""" + if not isinstance(value, int | float) or isinstance(value, bool): + return None + dt = datetime.fromtimestamp(float(value), tz=UTC) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _edge_count(node: dict[str, Any], key: str) -> int | None: + """``node[key].count`` for a GraphQL ``edge_*`` container.""" + container = node.get(key) + if isinstance(container, dict): + return _int(container.get("count")) + return None + + +def _edges(container: Any) -> list[dict[str, Any]]: + """``container.edges[].node`` list for a GraphQL ``edge_*`` container.""" + if not isinstance(container, dict): + return [] + out = [] + for edge in container.get("edges") or []: + node = edge.get("node") if isinstance(edge, dict) else None + if not isinstance(node, dict): + continue + out.append(node) + return out + + +def _caption_text(node: dict[str, Any]) -> str | None: + """First caption edge's text (web feed) or a flat ``caption`` fallback.""" + edges = _edges(node.get("edge_media_to_caption")) + if edges: + text = edges[0].get("text") + if isinstance(text, str): + return text + cap = node.get("caption") + if isinstance(cap, dict): + cap = cap.get("text") + if isinstance(cap, str): + return cap + return None + + +def _likes_count(node: dict[str, Any]) -> int | None: + """Like count. ``-1`` (creator hid it) is passed through, never coerced.""" + for key in ("edge_liked_by", "edge_media_preview_like"): + count = _edge_count(node, key) + if count is not None: + return count + return _int(node.get("like_count")) + + +def _shortcode(node: dict[str, Any]) -> str | None: + code = node.get("shortcode") or node.get("code") + if isinstance(code, str): + return code + return None + + +def parse_media(node: dict[str, Any]) -> dict[str, Any]: + """Map a raw timeline/feed media node to a flat media item dict.""" + code = _shortcode(node) + caption = _caption_text(node) + typename = node.get("__typename") + owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} + dims = node.get("dimensions") if isinstance(node.get("dimensions"), dict) else {} + is_video = bool(node.get("is_video")) + return { + "id": node.get("id"), + "type": _TYPE_MAP.get(typename) or ("Video" if is_video else "Image"), + "shortCode": code, + "caption": caption, + "hashtags": _HASHTAG_RE.findall(caption) if caption else [], + "mentions": _MENTION_RE.findall(caption) if caption else [], + "url": f"{_BASE}/p/{code}/" if code else None, + "commentsCount": _edge_count(node, "edge_media_to_comment") + or _int(node.get("comment_count")), + "dimensionsHeight": _int(dims.get("height")), + "dimensionsWidth": _int(dims.get("width")), + "displayUrl": node.get("display_url"), + "videoUrl": node.get("video_url") if is_video else None, + "alt": node.get("accessibility_caption"), + "likesCount": _likes_count(node), + "videoViewCount": _int(node.get("video_view_count")) if is_video else None, + "timestamp": _utc_from_sec(node.get("taken_at_timestamp")), + "ownerUsername": owner.get("username"), + "ownerId": owner.get("id") or node.get("owner_id"), + "ownerFullName": owner.get("full_name"), + "isCommentsDisabled": node.get("comments_disabled"), + } + + +def parse_comment(node: dict[str, Any], *, post_url: str | None) -> dict[str, Any]: + """Map a raw comment node to a flat comment item dict.""" + owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} + code = _shortcode(node) + return { + "id": node.get("id"), + "postUrl": post_url, + "commentUrl": f"{_BASE}/p/{code}/c/{node.get('id')}/" if code else None, + "text": node.get("text"), + "ownerUsername": owner.get("username"), + "ownerProfilePicUrl": owner.get("profile_pic_url"), + "timestamp": _utc_from_sec(node.get("created_at")), + "repliesCount": _edge_count(node, "edge_threaded_comments") + or _int(node.get("child_comment_count")), + "likesCount": _edge_count(node, "edge_liked_by") + or _int(node.get("comment_like_count")), + "owner": {"id": owner.get("id"), "username": owner.get("username")} + if owner + else None, + } + + +def parse_profile(user: dict[str, Any]) -> dict[str, Any]: + """Map a raw ``web_profile_info`` ``data.user`` to a flat profile item dict.""" + username = user.get("username") + latest = [parse_media(n) for n in _edges(user.get("edge_owner_to_timeline_media"))] + return { + "detailKind": "profile", + "id": user.get("id"), + "username": username, + "url": f"{_BASE}/{username}/" if username else None, + "fullName": user.get("full_name"), + "biography": user.get("biography"), + "externalUrl": user.get("external_url"), + "followersCount": _edge_count(user, "edge_followed_by"), + "followsCount": _edge_count(user, "edge_follow"), + "postsCount": _edge_count(user, "edge_owner_to_timeline_media"), + "highlightReelCount": _int(user.get("highlight_reel_count")), + "igtvVideoCount": _edge_count(user, "edge_felix_video_timeline"), + "isBusinessAccount": user.get("is_business_account"), + "businessCategoryName": user.get("business_category_name"), + "private": user.get("is_private"), + "verified": user.get("is_verified"), + "profilePicUrl": user.get("profile_pic_url"), + "profilePicUrlHD": user.get("profile_pic_url_hd"), + "latestPosts": latest, + } + + +def parse_hashtag(data: dict[str, Any]) -> dict[str, Any]: + """Map a raw ``tags/web_info`` payload to a flat hashtag item dict.""" + node = data.get("data") if isinstance(data.get("data"), dict) else data + name = node.get("name") + top = _edges(node.get("edge_hashtag_to_top_posts")) + recent = _edges(node.get("edge_hashtag_to_media")) + return { + "detailKind": "hashtag", + "id": node.get("id"), + "name": name, + "url": f"{_BASE}/explore/tags/{name}/" if name else None, + "postsCount": _edge_count(node, "edge_hashtag_to_media"), + "topPosts": [parse_media(n) for n in top], + "posts": [parse_media(n) for n in recent], + } + + +def parse_place(data: dict[str, Any]) -> dict[str, Any]: + """Map a raw ``locations/web_info`` payload to a flat place item dict.""" + loc = data.get("location") if isinstance(data.get("location"), dict) else data + recent = _edges(loc.get("edge_location_to_media")) + return { + "detailKind": "place", + "name": loc.get("name"), + "location_id": str(loc.get("id")) if loc.get("id") is not None else None, + "slug": loc.get("slug"), + "lat": loc.get("lat"), + "lng": loc.get("lng"), + "location_address": loc.get("address_json") or loc.get("address"), + "location_city": loc.get("city"), + "phone": loc.get("phone"), + "website": loc.get("website"), + "category": loc.get("category"), + "media_count": _edge_count(loc, "edge_location_to_media"), + "posts": [parse_media(n) for n in recent], + } From b1eff478fd0dcb1e55ecc94e344c5795fc414b23 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:32 +0530 Subject: [PATCH 043/160] feat(instagram): add proxy-aware fetch with warm-up and rotate-on-block --- .../proprietary/platforms/instagram/fetch.py | 354 ++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/fetch.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py new file mode 100644 index 000000000..82b0bcc14 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -0,0 +1,354 @@ +"""Proxy-aware fetch seam for the Instagram scraper (no browser). + +All network I/O flows through :func:`fetch_json` and always egresses through the +residential proxy (a direct hit would expose and risk-block the server IP). + +Instagram's public web app exposes anonymous JSON endpoints that a logged-out +browser calls, guarded by the ``X-IG-App-ID`` web app id and a warmed +``csrftoken``/``mid`` cookie pair: + + warm one anonymous session (plain GET to ``www.instagram.com/`` mints + ``csrftoken``/``mid``), then GET the ``api/v1/*/web_info`` / + ``web_profile_info`` endpoints through that same Chrome-impersonated, + sticky-IP session with the ``X-IG-App-ID`` header. + +This is a direct port of ``../reddit/fetch.py``'s rotate-on-block sticky-session +pattern (``_RotatingSession`` + ``_current_session`` ContextVar + +``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with an +Instagram-specific :func:`warm_session` and header set. + +Honest ceiling: anonymous Instagram access is the most hostile of our platforms. +Login walls appear as 401/403 and rotate the exit IP; 429 backs off on the same +IP. Observed per-IP/session limits are documented in ``README.md``; the safe +``_FANOUT_CONCURRENCY`` is deliberately low. ponytail: the pacing/rotation +constants are calibrated to residential exits and may need retuning per pool — +watch for 401/403/429 log spam and adjust. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlencode + +from scrapling.fetchers import AsyncFetcher, FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + + +class InstagramAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + This is the Instagram login-wall branch: after warming and rotating, the + exit IPs still 401/403. We are anonymous-only and cannot log in, so instead + of silently returning nothing we surface it as a clear error (mirrors + reddit's ``RedditAccessBlockedError`` / google_maps' ``SignInRequiredError``). + The executor turns it into a 403 for REST callers. + """ + + +# Per-flow proxy session, set by ``bind_proxy_holder`` around one continuation +# chain. Reusing one keep-alive connection pins a single sticky exit IP so the +# warmed ``csrftoken``/``mid`` cookies (bound to that IP) stay valid across the +# warm-up + every subsequent web-endpoint fetch. A ContextVar keeps each +# concurrent fan-out flow on its own session/IP without threading a param +# through every call. +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "instagram_proxy_session", default=None +) + +# 401/403 => this IP hit the login wall; rotate to a fresh one and re-warm. +# 429 => rate limited; back off on the SAME IP (rotating wouldn't help and burns +# the pool). +_ROTATE_STATUSES = frozenset({401, 403}) +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +# Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't +# burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits; +# a pool with a stricter per-IP cap may need it raised — watch for 429 log spam. +_MIN_INTERVAL_S = 1.5 +_PACE_JITTER_S = 0.5 + +# A healthy fetch lands in ~1-2s; cap at 15s so a dead sticky IP costs one +# bounded wait, then the timeout falls into the generic exception branch of +# fetch_json and rotates to a fresh IP — same treatment as a 403. +_REQUEST_TIMEOUT_S = 15.0 + +# The anonymous web app id every logged-out instagram.com XHR carries. Without +# it the api/v1/*/web_info and web_profile_info endpoints 403 outright. +_IG_APP_ID = "936619743392459" +_HEADERS = { + "Accept-Language": "en-US,en;q=0.9", + "X-IG-App-ID": _IG_APP_ID, + "X-Requested-With": "XMLHttpRequest", + "Referer": "https://www.instagram.com/", +} + +# A plain GET to the home page mints the anonymous csrftoken/mid cookie pair. +_WARM_URL = "https://www.instagram.com/" +_BASE = "https://www.instagram.com" +_CSRF_COOKIE = "csrftoken" + + +def now_iso() -> str: + """UTC timestamp in the millisecond ISO shape used by scraper output.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _response_cookie_names(page: Any) -> set[str]: + """Cookie names set by a response (best-effort across scrapling shapes).""" + cookies = getattr(page, "cookies", None) + if isinstance(cookies, dict): + return set(cookies.keys()) + return set() + + +def _parse_json(page: Any) -> Any | None: + """Parse a scrapling response body into JSON, or ``None``. + + Prefers ``page.json()``; falls back to ``json.loads`` on the raw body when + the impersonated response hands back text. + """ + fn = getattr(page, "json", None) + if callable(fn): + with suppress(Exception): + return fn() + for attr in ("body", "text"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + with suppress(Exception): + return json.loads(val) + return None + return None + + +def _build_url(path: str, params: dict[str, Any] | None) -> str: + """Absolute URL for an instagram.com path (accepts already-absolute URLs).""" + base = path if path.startswith("http") else f"{_BASE}/{path.strip('/')}/" + if not params: + return base + qs = urlencode({k: v for k, v in params.items() if v is not None}) + sep = "&" if "?" in base else "?" + return f"{base}{sep}{qs}" if qs else base + + +class _RotatingSession: + """Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one. + + ``rotate()`` closes the current keep-alive connection and opens a new one, so + the rotating gateway hands out a different residential exit IP. Because the + warmed cookies bind to the exit IP, ``rotate()`` also drops the warmed state + — the next fetch re-warms on the new IP. Used sequentially within a single + flow (never shared across concurrent tasks), so no locking is needed. + ``session`` is ``None`` only when no proxy is configured. + """ + + def __init__(self) -> None: + self._cm: Any | None = None + self.session: Any | None = None + self.rotations = 0 + self.warmed = False + self._last_at = 0.0 + + async def _open(self) -> None: + proxy = get_proxy_url() + self.warmed = False + if proxy is None: + self._cm = self.session = None + return + self._cm = FetcherSession( + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, + ) + self.session = await self._cm.__aenter__() + + async def close(self) -> None: + if self._cm is not None: + with suppress(Exception): # best-effort teardown + await self._cm.__aexit__(None, None, None) + self._cm = self.session = None + + async def rotate(self) -> Any | None: + """Drop the current IP and connect through a fresh one. Returns new session.""" + await self.close() + self.rotations += 1 + await self._open() + logger.info( + "[instagram] rotated proxy session (rotation #%d)", self.rotations + ) + return self.session + + async def pace(self) -> None: + """Sleep to hold this sticky IP under Instagram's per-IP rate threshold.""" + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at) + if wait > 0: + await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S)) + self._last_at = time.monotonic() + + +async def open_proxy_holder() -> _RotatingSession: + """Open a warm rotate-on-block session holder (caller owns ``close()``).""" + holder = _RotatingSession() + await holder._open() + return holder + + +@asynccontextmanager +async def bind_proxy_holder(holder: _RotatingSession): + """Route this task's fetches through ``holder`` for the enclosed block. + + Does NOT close the holder — enables pooling warm sessions across sequential + jobs so each job skips the proxy handshake AND the cookie warm-up. + """ + token = _current_session.set(holder) + try: + yield holder + finally: + _current_session.reset(token) + + +@asynccontextmanager +async def proxy_session(): + """Open one reused, rotate-on-block proxy session for a continuation chain.""" + holder = await open_proxy_holder() + try: + async with bind_proxy_holder(holder): + yield holder + finally: + await holder.close() + + +async def warm_session(session: Any) -> bool: + """Mint anonymous ``csrftoken``/``mid`` cookies on a freshly opened session. + + Returns ``True`` when a ``csrftoken`` was issued (the session can now reach + the web endpoints), else ``False`` (caller rotates the IP and retries). + + Takes an already-open ``session`` (never constructs one) so tests can drive + warm/rotate deterministically with a fake session, exactly like the reddit + sibling's fetch-resilience tests. + """ + seen: set[str] = set() + with suppress(Exception): + page = await session.get(_WARM_URL, headers=_HEADERS) + seen |= _response_cookie_names(page) + return _CSRF_COOKIE in seen + + +async def _get_page(session: Any, url: str) -> Any: + """GET through the warmed sticky session, or a one-shot proxied fetch.""" + if session is not None: + return await session.get(url, headers=_HEADERS) + return await AsyncFetcher.get( + url, + headers=_HEADERS, + proxy=get_proxy_url(), + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + + +async def resolve_redirect(url: str) -> str | None: + """Follow a ``share/`` short URL to its canonical target, or ``None``. + + ``share/`` links redirect to the real post/profile URL; the resolver records + the original as ``redirectedFromUrl``. Best-effort: returns the final URL + when the session exposes it, else ``None``. + """ + holder = _current_session.get() + if holder is None: + async with proxy_session(): + return await resolve_redirect(url) + with suppress(Exception): + page = await _get_page(holder.session, url) + final = getattr(page, "url", None) + if isinstance(final, str) and final and final != url: + return final + return None + + +async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None: + """GET an Instagram web endpoint through a warmed HTTP session. + + Returns parsed JSON (dict or list), or ``None`` on 404 / non-block failure. + Warms cookies once per session; rotates the residential IP and re-warms on + 401/403; backs off on 429. Raises :class:`InstagramAccessBlockedError` only + when every rotated IP refuses anonymous access (the login-wall branch, which + we cannot satisfy). + """ + holder = _current_session.get() + if holder is None: + # No bound session (e.g. a direct call outside fan_out): open a + # short-lived warmed session for this one fetch, then tear it down. + async with proxy_session(): + return await fetch_json(path, params) + + url = _build_url(path, params) + attempt = 0 + backoffs = 0 + while True: + session = holder.session + try: + if session is not None and not holder.warmed: + warmed_ok = await warm_session(session) + holder.warmed = True # attempted; don't re-warm this IP + if not warmed_ok: + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + raise InstagramAccessBlockedError( + f"could not warm session after {attempt} IP rotations for {path}" + ) + + await holder.pace() + page = await _get_page(session, url) + status = page.status + + if status == 200: + return _parse_json(page) + if status == 404: + return None + if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: + backoffs += 1 + delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1)) + logger.warning( + "[instagram] 429 on %s; backing off %.1fs", path, delay + ) + await asyncio.sleep(delay + random.uniform(0, 1)) + continue + if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + if status in _ROTATE_STATUSES: + raise InstagramAccessBlockedError( + f"Instagram refused {path} on {attempt} rotated IPs ({status})" + ) + logger.warning("[instagram] GET %s returned %s", path, status) + return None + except InstagramAccessBlockedError: + raise + except Exception as e: + logger.warning("[instagram] GET %s failed: %s", path, e) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + return None From 6270250f74e34024da568b7046b76aac34922b04 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:38 +0530 Subject: [PATCH 044/160] feat(instagram): add fan-out orchestrator with deterministic early-stop cleanup --- .../platforms/instagram/__init__.py | 24 + .../platforms/instagram/scraper.py | 480 ++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/__init__.py b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py new file mode 100644 index 000000000..e3a2a122a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py @@ -0,0 +1,24 @@ +"""Platform-native Instagram scraper (anonymous, no browser).""" + +from .fetch import InstagramAccessBlockedError +from .schemas import ( + InstagramComment, + InstagramHashtag, + InstagramMediaItem, + InstagramPlace, + InstagramProfile, + InstagramScrapeInput, +) +from .scraper import iter_instagram, scrape_instagram + +__all__ = [ + "InstagramAccessBlockedError", + "InstagramComment", + "InstagramHashtag", + "InstagramMediaItem", + "InstagramPlace", + "InstagramProfile", + "InstagramScrapeInput", + "iter_instagram", + "scrape_instagram", +] diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py new file mode 100644 index 000000000..46569f4dd --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -0,0 +1,480 @@ +"""Orchestrator for the Instagram scraper. + +The core is the async generator :func:`iter_instagram` (unbounded); +:func:`scrape_instagram` is a thin collector with a caller-supplied ``limit`` +guard. Any cap is caller policy, never baked into flow logic. + +Independent targets (one per ``directUrl`` / discovered entity) fan out +concurrently on a pool of warm sessions (sticky IPs); each target's own paging +stays sequential. ``fan_out`` is ported from ``../reddit/scraper.py`` but bound +to *this* module's proxy holders so every worker warms its own session once and +reuses it. + +Flows are selected by ``resultsType``: +- ``posts`` / ``reels`` / ``mentions`` -> media items (profile / hashtag feeds, + or discovery search) +- ``comments`` -> comment items for post/reel URLs +- ``details`` -> profile / hashtag / place metadata (by URL or discovery search) + +ponytail: deep feed pagination (past the first web page of media) needs the +GraphQL cursor endpoint whose doc-id drifts; v1 emits the first page and stops. +The upgrade path is a ``_paginate_feed`` helper in this file plus a doc-id in +``fetch.py`` — contained to these two files, per the acquisition-seam rule. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import aclosing +from datetime import UTC, datetime, timedelta +from typing import Any + +from .fetch import ( + InstagramAccessBlockedError, + bind_proxy_holder, + fetch_json, + now_iso, + open_proxy_holder, +) +from .parsers import ( + parse_comment, + parse_hashtag, + parse_media, + parse_place, + parse_profile, +) +from .schemas import InstagramScrapeInput +from .url_resolver import ResolvedUrl, resolve_url + +logger = logging.getLogger(__name__) + +__all__ = [ + "InstagramAccessBlockedError", + "iter_instagram", + "scrape_instagram", +] + +# Independent jobs run concurrently on a pool of warm proxy sessions. Anonymous +# Instagram is the most hostile platform, so this stays low to avoid burning the +# residential pool with parallel login walls. +_FANOUT_CONCURRENCY = 8 + +# Per-post comment fetches fan across their own warm sessions; kept below the +# top-level width so N concurrent targets x this can't explode the IP count. +_COMMENT_CONCURRENCY = 4 + +_PROFILE_PATH = "api/v1/users/web_profile_info/" +_HASHTAG_PATH = "api/v1/tags/web_info/" +_LOCATION_PATH = "api/v1/locations/web_info/" +_SEARCH_PATH = "web/search/topsearch/" + + +def _parse_newer_than(value: str | None) -> datetime | None: + """Parse ``onlyPostsNewerThan`` (ISO, YYYY-MM-DD, or relative) to UTC. + + Relative forms: ``" "`` where unit is minute/hour/day/week/month/ + year (singular or plural). Anything unparseable returns ``None`` (no filter). + """ + if not value: + return None + text = value.strip().lower() + parts = text.split() + if len(parts) == 2 and parts[0].isdigit(): + n = int(parts[0]) + unit = parts[1].rstrip("s") + days = { + "minute": n / 1440, + "hour": n / 24, + "day": n, + "week": n * 7, + "month": n * 30, + "year": n * 365, + }.get(unit) + if days is None: + return None + return datetime.now(UTC) - timedelta(days=days) + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + if dt.tzinfo: + return dt + return dt.replace(tzinfo=UTC) + except ValueError: + return None + + +def _is_after(timestamp: str | None, cutoff: datetime | None) -> bool: + """True if the item ``timestamp`` (ISO) is at/after the cutoff (or no cutoff).""" + if cutoff is None: + return True + if not timestamp: + return True + try: + dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + return dt >= cutoff + except ValueError: + return True + + +async def fan_out( + jobs: list[AsyncIterator[dict[str, Any]]], *, concurrency: int = _FANOUT_CONCURRENCY +) -> AsyncIterator[dict[str, Any]]: + """Stream items from independent async-iterator jobs via a warm worker pool. + + Each worker opens ONE proxy session and reuses it across the sequential jobs + it pulls, so only the first job per worker pays the proxy handshake + the + cookie warm-up. A bad job yields nothing rather than aborting the batch; + workers are cancelled and their sessions closed if the consumer stops early. + """ + if not jobs: + return + job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue() + for job in jobs: + job_queue.put_nowait(job) + results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue() + + async def worker() -> None: + holder = None + try: + holder = await open_proxy_holder() + except Exception as e: # no session: jobs still run via one-shot fetches + logger.warning("[instagram] proxy session open failed: %s", e) + try: + while True: + try: + job = job_queue.get_nowait() + except asyncio.QueueEmpty: + return + items: list[dict[str, Any]] = [] + try: + if holder is not None: + async with bind_proxy_holder(holder): + items = [item async for item in job] + else: + items = [item async for item in job] + except InstagramAccessBlockedError: + raise # a hard login wall must abort the batch, not be swallowed + except Exception as e: # one bad target must not kill the run + logger.warning("[instagram] fan-out job failed: %s", e) + await results.put(items) + finally: + if holder is not None: + await holder.close() + + tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] + try: + for _ in range(len(jobs)): + for item in await results.get(): + yield item + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +def _emit(partial: dict[str, Any], *, input_url: str | None) -> dict[str, Any]: + """Stamp provenance and serialize (parsers return plain dicts).""" + out = {**partial, "scrapedAt": now_iso()} + if input_url is not None: + out.setdefault("inputUrl", input_url) + return out + + +async def _profile_user(username: str) -> dict[str, Any] | None: + """Fetch a profile's ``data.user`` node, or ``None``.""" + data = await fetch_json(_PROFILE_PATH, {"username": username}) + if isinstance(data, dict): + user = ( + data.get("data", {}).get("user") + if isinstance(data.get("data"), dict) + else None + ) + if isinstance(user, dict): + return user + return None + return None + + +def _media_matches(item: dict[str, Any], result_type: str) -> bool: + """Filter a media item by feed type. ``reels`` keeps clips/videos only.""" + if result_type == "reels": + return item.get("type") == "Video" or item.get("productType") == "clips" + return True + + +async def _media_flow( + resolved: ResolvedUrl, + *, + input_model: InstagramScrapeInput, + cutoff: datetime | None, + per_target: int, +) -> AsyncIterator[dict[str, Any]]: + """Emit media items for a profile / hashtag / place URL.""" + from .parsers import _edges + + result_type = input_model.resultsType + if resolved.kind == "profile": + user = await _profile_user(resolved.value) + if user is None: + return + nodes = _edges(user.get("edge_owner_to_timeline_media")) + emitted = 0 + for node in nodes: + item = parse_media(node) + if input_model.skipPinnedPosts and item.get("isPinned"): + continue + if not _media_matches(item, result_type): + continue + if not _is_after(item.get("timestamp"), cutoff): + continue + yield _emit(item, input_url=resolved.url) + emitted += 1 + if emitted >= per_target: + return + return + if resolved.kind == "hashtag": + data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) + if isinstance(data, dict): + parsed = parse_hashtag(data) + emitted = 0 + for node in [*parsed.get("topPosts", []), *parsed.get("posts", [])]: + if not _media_matches(node, result_type): + continue + if not _is_after(node.get("timestamp"), cutoff): + continue + yield _emit(node, input_url=resolved.url) + emitted += 1 + if emitted >= per_target: + return + return + if resolved.kind == "place": + data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) + if isinstance(data, dict): + parsed = parse_place(data) + emitted = 0 + for node in parsed.get("posts", []): + if not _is_after(node.get("timestamp"), cutoff): + continue + yield _emit(node, input_url=resolved.url) + emitted += 1 + if emitted >= per_target: + return + return + + +async def _comments_flow( + resolved: ResolvedUrl, + *, + input_model: InstagramScrapeInput, + per_target: int, +) -> AsyncIterator[dict[str, Any]]: + """Emit comment items for a post / reel URL. + + ponytail: the anonymous comment page uses a GraphQL cursor whose doc-id + drifts; v1 sources the comments embedded in the media info payload and caps + at the actor's 50/post ceiling. Deeper paging is the upgrade path in + ``fetch.py``. + """ + from .parsers import _edges + + path = f"p/{resolved.value}/" + data = await fetch_json(path, {"__a": 1, "__d": "dis"}) + node = None + if isinstance(data, dict): + items = data.get("items") + if isinstance(items, list) and items: + node = items[0] + else: + gql = data.get("graphql") + node = gql.get("shortcode_media") if isinstance(gql, dict) else None + if not isinstance(node, dict): + return + comment_nodes = _edges(node.get("edge_media_to_parent_comment")) or _edges( + node.get("edge_media_to_comment") + ) + cap = min(per_target, 50) + emitted = 0 + for cnode in comment_nodes: + item = parse_comment(cnode, post_url=resolved.url) + yield _emit(item, input_url=resolved.url) + emitted += 1 + if input_model.includeNestedComments: + for reply in _edges(cnode.get("edge_threaded_comments")): + if emitted >= cap: + return + yield _emit( + parse_comment(reply, post_url=resolved.url), + input_url=resolved.url, + ) + emitted += 1 + if emitted >= cap: + return + + +async def _details_flow( + resolved: ResolvedUrl, *, input_model: InstagramScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Emit one profile / hashtag / place detail item for a URL.""" + if resolved.kind == "profile": + user = await _profile_user(resolved.value) + if user is not None: + yield _emit(parse_profile(user), input_url=resolved.url) + return + if resolved.kind == "hashtag": + data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) + if isinstance(data, dict): + yield _emit(parse_hashtag(data), input_url=resolved.url) + return + if resolved.kind == "place": + data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) + if isinstance(data, dict): + yield _emit(parse_place(data), input_url=resolved.url) + return + + +async def _discover( + query: str, *, search_type: str, limit: int +) -> list[ResolvedUrl]: + """Resolve a discovery query into target URLs via topsearch.""" + data = await fetch_json(_SEARCH_PATH, {"query": query, "context": "blended"}) + if not isinstance(data, dict): + return [] + out: list[ResolvedUrl] = [] + if search_type in ("profile", "user"): + for entry in data.get("users", []): + user = entry.get("user", {}) if isinstance(entry, dict) else {} + name = user.get("username") + if not name: + continue + out.append( + ResolvedUrl("profile", name, f"https://www.instagram.com/{name}/") + ) + elif search_type == "hashtag": + for entry in data.get("hashtags", []): + tag = entry.get("hashtag", {}) if isinstance(entry, dict) else {} + name = tag.get("name") + if not name: + continue + out.append( + ResolvedUrl( + "hashtag", + name, + f"https://www.instagram.com/explore/tags/{name}/", + ) + ) + elif search_type == "place": + for entry in data.get("places", []): + place = entry.get("place", {}) if isinstance(entry, dict) else {} + loc = place.get("location", {}) if isinstance(place, dict) else {} + pk = loc.get("pk") or loc.get("id") + if not pk: + continue + out.append( + ResolvedUrl( + "place", + str(pk), + f"https://www.instagram.com/explore/locations/{pk}/", + ) + ) + return out[:limit] + + +def _resolve_inputs(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: + """Resolve ``directUrls`` (URLs take priority over ``search``).""" + resolved: list[ResolvedUrl] = [] + for url in input_model.directUrls: + r = resolve_url(url) + if r is None: + logger.warning("[instagram] unrecognized URL: %s", url) + continue + resolved.append(r) + return resolved + + +async def _targets(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: + """The resolved targets for this run: direct URLs, else discovery search.""" + if input_model.directUrls: + return _resolve_inputs(input_model) + if not input_model.search: + return [] + limit = input_model.searchLimit or 10 + queries = [q.strip() for q in input_model.search.split(",") if q.strip()] + targets: list[ResolvedUrl] = [] + for query in queries: + targets.extend( + await _discover(query, search_type=input_model.searchType, limit=limit) + ) + return targets + + +async def iter_instagram( + input_model: InstagramScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield flat Instagram items. ``directUrls`` override ``search``. + + Independent targets fan out concurrently; each target's paging stays + sequential. De-dupes media by ``id`` across targets. + """ + targets = await _targets(input_model) + if not targets: + return + result_type = input_model.resultsType + cutoff = _parse_newer_than(input_model.onlyPostsNewerThan) + per_target = input_model.resultsLimit or 10 + + if result_type == "comments": + jobs = [ + _comments_flow(r, input_model=input_model, per_target=per_target) + for r in targets + if r.kind in ("post", "reel") + ] + async with aclosing(fan_out(jobs, concurrency=_COMMENT_CONCURRENCY)) as stream: + async for item in stream: + yield item + return + + if result_type == "details": + jobs = [_details_flow(r, input_model=input_model) for r in targets] + async with aclosing(fan_out(jobs)) as stream: + async for item in stream: + yield item + return + + # posts / reels / mentions -> media feeds, de-duped by id across targets. + jobs = [ + _media_flow( + r, input_model=input_model, cutoff=cutoff, per_target=per_target + ) + for r in targets + ] + seen: set[str] = set() + async with aclosing(fan_out(jobs)) as stream: + async for item in stream: + item_id = item.get("id") + if isinstance(item_id, str): + if item_id in seen: + continue + seen.add(item_id) + yield item + + +async def scrape_instagram( + input_model: InstagramScrapeInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_instagram` into a list, honoring an optional ``limit``. + + ``limit`` is a request-time policy guard, NOT a ceiling in the streaming + core. + """ + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async with aclosing(iter_instagram(input_model)) as stream: + async for item in stream: + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results From eccc142974e1b6cb8960cd55e340ed25aadec669 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:47 +0530 Subject: [PATCH 045/160] docs(instagram): add platform scraper README --- .../proprietary/platforms/instagram/README.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/README.md diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md new file mode 100644 index 000000000..fa42e30a6 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -0,0 +1,105 @@ +# Instagram scraper (anonymous, no browser) + +Platform-native Instagram scraper (anonymous, no browser). Standalone module: it +depends only on `app.utils.proxy` + `scrapling` and exposes a stable public API. +Its input/output surface is a **reference-compatible** mirror of the public +Instagram scraper actor spec (same `resultsType` / `directUrls` / camelCase field +names, additive `extra="allow"` parity), so callers written against that surface +work unchanged. It is **not** wired into ingestion or Celery — the capability +layer under `app/capabilities/instagram/` is what turns these primitives into +REST + agent + MCP surfaces. + +## Approach + +Instagram's public web app exposes anonymous, logged-out JSON behind a handful of +`www.instagram.com` endpoints once a session carries an anonymous `csrftoken` + +`mid` cookie pair and the `x-ig-app-id` web header: + +> Warm an anonymous session with one plain GET to `www.instagram.com/` (mints +> `csrftoken` + `mid`), then GET the web JSON endpoints through that same +> Chrome-impersonated, sticky-IP session. Rotate the residential IP + re-warm on +> a login wall (401/403), back off on 429. + +Endpoints used (anonymous web tier only): + +| Flow | Endpoint | +|---|---| +| profile / posts / reels | `api/v1/users/web_profile_info/?username=…` | +| comments | `p//?__a=1&__d=dis` | +| hashtag | `api/v1/tags/web_info/?tag_name=…` | +| place | `api/v1/locations/web_info/?location_id=…` | +| discovery search | `web/search/topsearch/?query=…` | + +**No browser, no Chromium, no `solve_cloudflare`** — this stays on the cheap HTTP +tier the sibling scrapers already use. + +## Anonymous only — no authentication, ever + +No login, no `sessionid` account cookie, no app password. The only cookies held +are the anonymous `csrftoken` / `mid` minted by the warm-up. There is **no** +authenticate option in the input surface or the fetch layer, by design. A +persistent block after IP rotation surfaces as `InstagramAccessBlockedError` +(mirrors Reddit's `RedditAccessBlockedError`) rather than a silent empty result, +so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. + +## Module map + +| File | Responsibility | +|---|---| +| `__init__.py` | Public exports: `InstagramScrapeInput`, item models, `iter_instagram`, `scrape_instagram`, `InstagramAccessBlockedError`. | +| `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramComment`, `InstagramProfile`, `InstagramHashtag`, `InstagramPlace`) each with `to_output()`. | +| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json`. No browser imports. | +| `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`/`hashtag`/`place`; non-Instagram → `None`. Strips `_u/`, `profilecard/`; story → profile. | +| `parsers.py` | Pure JSON→dict mapping (`parse_media`, `parse_comment`, `parse_profile`, `parse_hashtag`, `parse_place`, `_edges`). I/O-free. | +| `scraper.py` | Orchestrator: `_media_flow`/`_comments_flow`/`_details_flow`/`_discover`, `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | + +## How it works + +1. `iter_instagram` resolves `directUrls` (or runs a discovery `search` per the + comma-split queries) into targets and fans them out on a pool of warm proxy + sessions (`fan_out`, 8-way; 4-way for comments). Each worker opens one + sticky-IP session and warms `csrftoken`/`mid` once, reusing it across the + sequential targets it pulls. +2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media feeds, + `comments` → per-post comment items, `details` → profile/hashtag/place + metadata. Media items de-dupe by `id` across targets. +3. `fetch_json` warms the session on first use, rotates the IP + re-warms on + 401/403, backs off on 429, returns `None` on 404. +4. Parsers map raw web JSON to flat dicts; the orchestrator stamps `scrapedAt` + and applies `resultsLimit` / `onlyPostsNewerThan` as request-time policy. + +## Observed limits & calibration caveats + +- Anonymous web JSON is rate-limited per IP; the sticky-session pool keeps each + IP's request rate modest but a hot pool will still hit login walls — that's the + `InstagramAccessBlockedError` path, not a bug. +- `likesCount` is frequently withheld on anonymous responses (surfaces as `-1` or + absent upstream); treat it as best-effort. +- Comments on the anonymous media page cap at ~50/post; deeper paging needs the + GraphQL cursor endpoint whose doc-id drifts (see the `ponytail:` note in + `scraper.py`/`fetch.py`). +- The `$3.50 / 1k items` default meter assumes the proxy-bytes-per-item measured + on the reference targets; re-measure with the `references/` scale harness before + high-volume production use. + +## Testing + +- Offline unit tests: `tests/unit/platforms/instagram/` — `test_skeleton.py` + (schema + URL resolver), `test_parsers.py` (fixture-pinned mapping), + `test_fetch_resilience.py` (warm / rotate / backoff loop with fake sessions, no + network), `test_budget.py` (fair-share caps + de-dup). +- Live e2e (needs network + residential proxy): `scripts/e2e_instagram_scraper.py` + — step 0 is the go/no-go cookie probe; later steps exercise the flows and dump + trimmed, PII-anonymized fixtures. + +```bash +cd surfsense_backend +.venv/bin/python -m pytest tests/unit/platforms/instagram/ +.venv/bin/python scripts/e2e_instagram_scraper.py # live; regenerates fixtures +``` + +## TODO / out of scope (v1) + +- Deep feed pagination past the first web page of media (GraphQL cursor doc-id). +- Deep comment pagination past the ~50/post embedded ceiling. +- Sticky-IP provider parity (same `__sid` caveat as the Reddit sibling). From ff55537bce0144a14fca0bc1bedc55ba80d0229e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:54 +0530 Subject: [PATCH 046/160] feat(instagram): add scrape capability verb --- .../capabilities/instagram/scrape/__init__.py | 3 + .../instagram/scrape/definition.py | 23 ++++ .../capabilities/instagram/scrape/executor.py | 54 +++++++++ .../capabilities/instagram/scrape/schemas.py | 105 ++++++++++++++++++ 4 files changed, 185 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/__init__.py create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/definition.py create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/executor.py create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/scrape/__init__.py b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py new file mode 100644 index 000000000..de8b3c7c4 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py @@ -0,0 +1,3 @@ +"""``instagram.scrape`` verb: Instagram URLs / search terms → posts, reels, mentions.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/scrape/definition.py b/surfsense_backend/app/capabilities/instagram/scrape/definition.py new file mode 100644 index 000000000..e84ca4938 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/definition.py @@ -0,0 +1,23 @@ +"""``instagram.scrape`` capability registration (billed per item; see config +``INSTAGRAM_SCRAPE_MICROS_PER_ITEM``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.instagram.scrape.executor import build_scrape_executor +from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput + +INSTAGRAM_SCRAPE = Capability( + name="instagram.scrape", + description=( + "Scrape public Instagram posts, reels, or mentions from " + "profile/post/hashtag/place URLs, or discover content via search queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.INSTAGRAM_ITEM, + docs_url="/docs/connectors/native/instagram", +) + +register_capability(INSTAGRAM_SCRAPE) diff --git a/surfsense_backend/app/capabilities/instagram/scrape/executor.py b/surfsense_backend/app/capabilities/instagram/scrape/executor.py new file mode 100644 index 000000000..a76722dcd --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/executor.py @@ -0,0 +1,54 @@ +"""``instagram.scrape`` executor: verb input → scraper → media items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, + scrape_instagram, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_instagram + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = InstagramScrapeInput( + resultsType=payload.result_type, + directUrls=payload.urls, + search=",".join(payload.search_queries), + searchType=payload.search_type, + resultsLimit=payload.max_per_target, + onlyPostsNewerThan=payload.newer_than, + skipPinnedPosts=payload.skip_pinned_posts, + addParentData=payload.add_parent_data, + ) + emit_progress( + "starting", + "Resolving Instagram targets", + total=payload.max_items, + unit="item", + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except InstagramAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"Instagram refused anonymous access: {exc}", + code="INSTAGRAM_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py new file mode 100644 index 000000000..0e5646dc7 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py @@ -0,0 +1,105 @@ +"""``instagram.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``InstagramScrapeInput`` +(``app/proprietary/platforms/instagram``). The executor maps this to the full +scraper input; the scraper's ``InstagramMediaItem`` is reused verbatim as the +output element. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.instagram import InstagramMediaItem + +MAX_INSTAGRAM_SOURCES = 20 +"""Per-call cap on urls + search_queries: bounds a synchronous request's fan-out.""" + +MAX_INSTAGRAM_ITEMS = 100 +"""Hard ceiling on items returned per call, regardless of the per-target caps.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description=( + "Instagram URLs or bare profile IDs: profile, post (/p/), reel " + "(/reel/), hashtag (/explore/tags/), or place (/explore/locations/). " + "Provide these OR search_queries (never both)." + ), + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description=( + "Discovery keywords (hashtags as plaintext, no '#'). Provide these " + "OR urls (never both)." + ), + ) + search_type: Literal["hashtag", "profile", "place", "user"] = Field( + default="hashtag", + description="What to discover from search_queries. Only used with search_queries.", + ) + result_type: Literal["posts", "reels", "mentions"] = Field( + default="posts", + description="Which feed to return. 'mentions' requires profile URLs.", + ) + newer_than: str | None = Field( + default=None, + description=( + "Only return posts newer than this: YYYY-MM-DD, ISO timestamp, or " + "relative ('1 day', '2 months'); UTC." + ), + ) + skip_pinned_posts: bool = Field( + default=False, + description="Exclude pinned posts (posts mode).", + ) + max_per_target: int = Field( + default=10, + ge=1, + description="Max results per URL or per discovered target.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max total items to return across all sources.", + ) + add_parent_data: bool = Field( + default=False, + description="Attach a dataSource block to each feed item.", + ) + + @model_validator(mode="after") + def _exactly_one_source(self) -> ScrapeInput: + if not self.urls and not self.search_queries: + raise ValueError( + "Provide at least one of 'urls' or 'search_queries'." + ) + if self.urls and self.search_queries: + raise ValueError( + "Provide 'urls' OR 'search_queries', not both (they cannot be combined)." + ) + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable items for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[InstagramMediaItem] = Field( + default_factory=list, + description="One media item per result (post/reel/mention), in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned item = one billable unit.""" + return len(self.items) From 38a26b1ccf0fad75615e25946517d756a1ec3145 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:58 +0530 Subject: [PATCH 047/160] feat(instagram): add comments capability verb --- .../instagram/comments/__init__.py | 3 + .../instagram/comments/definition.py | 22 +++++++ .../instagram/comments/executor.py | 53 ++++++++++++++++ .../instagram/comments/schemas.py | 61 +++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/comments/__init__.py create mode 100644 surfsense_backend/app/capabilities/instagram/comments/definition.py create mode 100644 surfsense_backend/app/capabilities/instagram/comments/executor.py create mode 100644 surfsense_backend/app/capabilities/instagram/comments/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/comments/__init__.py b/surfsense_backend/app/capabilities/instagram/comments/__init__.py new file mode 100644 index 000000000..84899d758 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/__init__.py @@ -0,0 +1,3 @@ +"""``instagram.comments`` verb: post/reel URLs → comments (and replies).""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/comments/definition.py b/surfsense_backend/app/capabilities/instagram/comments/definition.py new file mode 100644 index 000000000..7794228ac --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/definition.py @@ -0,0 +1,22 @@ +"""``instagram.comments`` capability registration (billed per comment; see config +``INSTAGRAM_SCRAPE_MICROS_PER_COMMENT``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.instagram.comments.executor import build_comments_executor +from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput + +INSTAGRAM_COMMENTS = Capability( + name="instagram.comments", + description=( + "Fetch comments (and optionally replies) for Instagram post or reel URLs." + ), + input_schema=CommentsInput, + output_schema=CommentsOutput, + executor=build_comments_executor(), + billing_unit=BillingUnit.INSTAGRAM_COMMENT, + docs_url="/docs/connectors/native/instagram", +) + +register_capability(INSTAGRAM_COMMENTS) diff --git a/surfsense_backend/app/capabilities/instagram/comments/executor.py b/surfsense_backend/app/capabilities/instagram/comments/executor.py new file mode 100644 index 000000000..260ee06c9 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/executor.py @@ -0,0 +1,53 @@ +"""``instagram.comments`` executor: verb input → scraper → comment items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, + scrape_instagram, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_comments_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_instagram + + async def execute(payload: CommentsInput) -> CommentsOutput: + actor_input = InstagramScrapeInput( + resultsType="comments", + directUrls=payload.urls, + resultsLimit=payload.max_comments_per_post, + isNewestComments=payload.newest_first, + includeNestedComments=payload.include_replies, + ) + emit_progress( + "starting", + "Fetching Instagram comments", + total=payload.max_items, + unit="comment", + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except InstagramAccessBlockedError as exc: + raise ForbiddenError( + f"Instagram refused anonymous access: {exc}", + code="INSTAGRAM_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", + f"Scraped {len(items)} comment(s)", + current=len(items), + unit="comment", + ) + return CommentsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/instagram/comments/schemas.py b/surfsense_backend/app/capabilities/instagram/comments/schemas.py new file mode 100644 index 000000000..fadb83215 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/schemas.py @@ -0,0 +1,61 @@ +"""``instagram.comments`` I/O contracts. + +A lean surface over ``InstagramScrapeInput`` (``resultsType="comments"``). The +scraper's ``InstagramComment`` is reused verbatim as the output element. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.instagram.scrape.schemas import ( + MAX_INSTAGRAM_ITEMS, + MAX_INSTAGRAM_SOURCES, +) +from app.proprietary.platforms.instagram import InstagramComment + +MAX_COMMENTS_PER_POST = 50 +"""Anonymous web media pages surface at most ~50 comments per post.""" + + +class CommentsInput(BaseModel): + urls: list[str] = Field( + min_length=1, + max_length=MAX_INSTAGRAM_SOURCES, + description="Post or reel URLs to fetch comments for (shortCode or numeric-ID forms).", + ) + newest_first: bool = Field( + default=False, + description="Return newest comments first.", + ) + include_replies: bool = Field( + default=False, + description="Include nested replies; each reply is a separate billable item.", + ) + max_comments_per_post: int = Field( + default=10, + ge=1, + le=MAX_COMMENTS_PER_POST, + description="Max comments per post (Instagram caps at 50).", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max total comments to return across all posts.", + ) + + @property + def estimated_units(self) -> int: + return self.max_items + + +class CommentsOutput(BaseModel): + items: list[InstagramComment] = Field( + default_factory=list, + description="One item per comment (or reply), in emission order.", + ) + + @property + def billable_units(self) -> int: + return len(self.items) From 7ea474be938da3b70d21672a41bb1d2b070ca831 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:01:38 +0530 Subject: [PATCH 048/160] feat(instagram): add details capability verb --- .../instagram/details/__init__.py | 3 + .../instagram/details/definition.py | 23 +++++ .../instagram/details/executor.py | 50 +++++++++++ .../capabilities/instagram/details/schemas.py | 86 +++++++++++++++++++ 4 files changed, 162 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/details/__init__.py create mode 100644 surfsense_backend/app/capabilities/instagram/details/definition.py create mode 100644 surfsense_backend/app/capabilities/instagram/details/executor.py create mode 100644 surfsense_backend/app/capabilities/instagram/details/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/details/__init__.py b/surfsense_backend/app/capabilities/instagram/details/__init__.py new file mode 100644 index 000000000..8e4c85258 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/__init__.py @@ -0,0 +1,3 @@ +"""``instagram.details`` verb: profile/hashtag/place URLs or search → metadata.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/details/definition.py b/surfsense_backend/app/capabilities/instagram/details/definition.py new file mode 100644 index 000000000..fa7f912d8 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/definition.py @@ -0,0 +1,23 @@ +"""``instagram.details`` capability registration (billed per item; see config +``INSTAGRAM_SCRAPE_MICROS_PER_ITEM``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.instagram.details.executor import build_details_executor +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput + +INSTAGRAM_DETAILS = Capability( + name="instagram.details", + description=( + "Fetch Instagram profile, hashtag, or place metadata by URL or discovery " + "search. Each item carries a detailKind discriminator." + ), + input_schema=DetailsInput, + output_schema=DetailsOutput, + executor=build_details_executor(), + billing_unit=BillingUnit.INSTAGRAM_ITEM, + docs_url="/docs/connectors/native/instagram", +) + +register_capability(INSTAGRAM_DETAILS) diff --git a/surfsense_backend/app/capabilities/instagram/details/executor.py b/surfsense_backend/app/capabilities/instagram/details/executor.py new file mode 100644 index 000000000..f9a152f50 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/executor.py @@ -0,0 +1,50 @@ +"""``instagram.details`` executor: verb input → scraper → detail items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, + scrape_instagram, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_details_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_instagram + + async def execute(payload: DetailsInput) -> DetailsOutput: + actor_input = InstagramScrapeInput( + resultsType="details", + directUrls=payload.urls, + search=",".join(payload.search_queries), + searchType=payload.search_type, + searchLimit=payload.search_limit, + ) + emit_progress( + "starting", + "Resolving Instagram detail targets", + total=payload.max_items, + unit="item", + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except InstagramAccessBlockedError as exc: + raise ForbiddenError( + f"Instagram refused anonymous access: {exc}", + code="INSTAGRAM_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return DetailsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/instagram/details/schemas.py b/surfsense_backend/app/capabilities/instagram/details/schemas.py new file mode 100644 index 000000000..31972b095 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/schemas.py @@ -0,0 +1,86 @@ +"""``instagram.details`` I/O contracts. + +A lean surface over ``InstagramScrapeInput`` (``resultsType="details"``). Each +output item is a profile / hashtag / place, discriminated by the synthesized +``detailKind`` field (a SurfSense addition; every other field mirrors the actor). +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import BaseModel, Field, model_validator + +from app.capabilities.instagram.scrape.schemas import ( + MAX_INSTAGRAM_ITEMS, + MAX_INSTAGRAM_SOURCES, +) +from app.proprietary.platforms.instagram import ( + InstagramHashtag, + InstagramPlace, + InstagramProfile, +) + +InstagramDetailItem = Annotated[ + InstagramProfile | InstagramHashtag | InstagramPlace, + Field(discriminator="detailKind"), +] + + +class DetailsInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description=( + "Profile / hashtag / place URLs (or bare profile IDs). The URL type " + "determines the detail kind. Provide these OR search_queries." + ), + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description="Discovery keywords. Provide these OR urls (never both).", + ) + search_type: Literal["hashtag", "profile", "place"] = Field( + default="hashtag", + description="What to discover from search_queries (no 'user' — use instagram.scrape).", + ) + search_limit: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max discovered entities per query.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max total detail items to return.", + ) + + @model_validator(mode="after") + def _exactly_one_source(self) -> DetailsInput: + if not self.urls and not self.search_queries: + raise ValueError( + "Provide at least one of 'urls' or 'search_queries'." + ) + if self.urls and self.search_queries: + raise ValueError( + "Provide 'urls' OR 'search_queries', not both (they cannot be combined)." + ) + return self + + @property + def estimated_units(self) -> int: + return self.max_items + + +class DetailsOutput(BaseModel): + items: list[InstagramDetailItem] = Field( + default_factory=list, + description="One item per profile/hashtag/place, keyed by detailKind.", + ) + + @property + def billable_units(self) -> int: + return len(self.items) From 929efe8152a83abd4cb4c7ae78aa58aa16bf8df5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:01:49 +0530 Subject: [PATCH 049/160] feat(instagram): register instagram.* capability namespace --- surfsense_backend/app/capabilities/instagram/__init__.py | 7 +++++++ surfsense_backend/app/routes/__init__.py | 1 + 2 files changed, 8 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/__init__.py diff --git a/surfsense_backend/app/capabilities/instagram/__init__.py b/surfsense_backend/app/capabilities/instagram/__init__.py new file mode 100644 index 000000000..4004f86c7 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/__init__.py @@ -0,0 +1,7 @@ +"""``instagram.*`` namespace: platform-native Instagram data verbs.""" + +from __future__ import annotations + +from app.capabilities.instagram.comments import definition as _comments # noqa: F401 +from app.capabilities.instagram.details import definition as _details # noqa: F401 +from app.capabilities.instagram.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1a5b182b8..d15cfdc40 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends # Import verb namespaces for their registration side effects before the door builds. import app.capabilities.google_maps import app.capabilities.google_search +import app.capabilities.instagram import app.capabilities.reddit import app.capabilities.web import app.capabilities.youtube # noqa: F401 From c5e3fd4006975075f603e3fa7d7581e0280e0291 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:01:55 +0530 Subject: [PATCH 050/160] feat(mcp): add Instagram scrape/comments/details tools --- .../mcp_server/features/scrapers/__init__.py | 12 +- .../features/scrapers/platforms/instagram.py | 194 ++++++++++++++++++ surfsense_mcp/mcp_server/selfcheck.py | 3 + surfsense_mcp/mcp_server/server.py | 3 +- 4 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index dfa2f3ab2..aade7a99d 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -13,9 +13,17 @@ from mcp.server.fastmcp import FastMCP from ...core.client import SurfSenseClient from ...core.workspace_context import WorkspaceContext from . import run_history -from .platforms import google_maps, google_search, reddit, web, youtube +from .platforms import google_maps, google_search, instagram, reddit, web, youtube -_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) +_REGISTRARS = ( + web, + google_search, + reddit, + youtube, + google_maps, + instagram, + run_history, +) def register( diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py new file mode 100644 index 000000000..39dfea71d --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -0,0 +1,194 @@ +"""Instagram scraper tools: posts/reels, comments, and details.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ResultType = Literal["posts", "reels", "mentions"] +SearchType = Literal["hashtag", "profile", "place", "user"] +DetailSearchType = Literal["hashtag", "profile", "place"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Instagram scrape, comments, and details tools.""" + + @mcp.tool( + name="surfsense_instagram_scrape", + title="Scrape Instagram posts or reels", + annotations=SCRAPE, + structured_output=False, + ) + async def instagram_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Instagram URLs: a profile, post (/p/), reel " + "(/reel/), hashtag (/explore/tags/), or place " + "(/explore/locations/). Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to discover content for (hashtags as plain " + "text, no '#'). Provide search_queries OR urls." + ), + ] = None, + search_type: Annotated[ + SearchType, Field(description="What to discover from search_queries.") + ] = "hashtag", + result_type: Annotated[ + ResultType, + Field(description="Which feed to return. 'mentions' needs profile URLs."), + ] = "posts", + max_items: Annotated[ + int, Field(ge=1, description="Maximum items to return across sources.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape public Instagram posts or reels from URLs or search queries. + + Use this for Instagram content research: a creator's recent posts, a + hashtag or location feed, or discovering posts by keyword. For a post's + comment section use surfsense_instagram_comments; for profile/hashtag/ + place metadata use surfsense_instagram_details. Returns per-item caption, + likes, comments count, media URLs, and owner. + Example: urls=['https://www.instagram.com/natgeo/'], result_type='reels'. + """ + return await run_scraper( + client, + context, + platform="instagram", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "search_type": search_type, + "result_type": result_type, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_instagram_comments", + title="Fetch Instagram comments", + annotations=SCRAPE, + structured_output=False, + ) + async def instagram_comments( + urls: Annotated[ + list[str], + Field( + min_length=1, + description="Instagram post or reel URLs, e.g. " + "['https://www.instagram.com/p/Cabc123/'].", + ), + ], + max_comments_per_post: Annotated[ + int, + Field(ge=1, le=50, description="Max comments per post (Instagram caps at 50)."), + ] = 10, + include_replies: Annotated[ + bool, Field(description="Include nested replies.") + ] = False, + newest_first: Annotated[ + bool, Field(description="Return newest comments first.") + ] = False, + max_items: Annotated[ + int, Field(ge=1, description="Max total comments across all posts.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch the comments (and optionally replies) on Instagram posts or reels. + + Use this when the user wants a post's discussion or audience reaction + rather than the post itself; get post URLs from surfsense_instagram_scrape + if you only have a topic or profile. Returns comment text, author, likes, + and replies. + Example: urls=['https://www.instagram.com/p/Cabc123/'], include_replies=True. + """ + return await run_scraper( + client, + context, + platform="instagram", + verb="comments", + payload={ + "urls": urls, + "max_comments_per_post": max_comments_per_post, + "include_replies": include_replies, + "newest_first": newest_first, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_instagram_details", + title="Fetch Instagram profile/hashtag/place details", + annotations=SCRAPE, + structured_output=False, + ) + async def instagram_details( + urls: Annotated[ + list[str] | None, + Field( + description="Profile, hashtag, or place URLs (or bare profile " + "IDs). Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to discover profiles/hashtags/places for. " + "Provide search_queries OR urls." + ), + ] = None, + search_type: Annotated[ + DetailSearchType, + Field(description="What to discover from search_queries."), + ] = "hashtag", + max_items: Annotated[ + int, Field(ge=1, description="Max detail items to return.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch Instagram profile, hashtag, or place metadata. + + Use this for entity lookups: a profile's follower/post counts and bio, a + hashtag's post volume and top posts, or a place's coordinates and post + count. For a feed of posts use surfsense_instagram_scrape instead. Each + item carries a detailKind field marking whether it is a profile, hashtag, + or place. + Example: urls=['https://www.instagram.com/explore/tags/crossfit/']. + """ + return await run_scraper( + client, + context, + platform="instagram", + verb="details", + payload={ + "urls": urls, + "search_queries": search_queries, + "search_type": search_type, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index 20224c173..f7b7a76c5 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -25,6 +25,9 @@ EXPECTED_TOOLS = { "surfsense_youtube_comments", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", + "surfsense_instagram_scrape", + "surfsense_instagram_comments", + "surfsense_instagram_details", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", # knowledge-base management diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 8ea9bc919..19cf966f7 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -36,7 +36,8 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " - "communities), YouTube (videos, transcripts, comments), Google " + "communities), YouTube (videos, transcripts, comments), Instagram " + "(posts, reels, comments, profile/hashtag/place details), Google " "Maps (places, reviews), Google Search results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " From 98eab8c4b5bef0fc98cf5bf7b7feb5e243687a7f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:05 +0530 Subject: [PATCH 051/160] feat(chat): add Instagram builtin subagent --- .../subagents/builtins/instagram/__init__.py | 1 + .../subagents/builtins/instagram/agent.py | 43 ++++++++++++ .../builtins/instagram/description.md | 2 + .../builtins/instagram/system_prompt.md | 68 +++++++++++++++++++ .../builtins/instagram/tools/__init__.py | 0 .../builtins/instagram/tools/index.py | 29 ++++++++ .../multi_agent_chat/subagents/registry.py | 4 ++ 7 files changed, 147 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py new file mode 100644 index 000000000..5e3857c86 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py @@ -0,0 +1 @@ +"""``instagram`` builtin subagent: structured Instagram posts, comments, and details.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py new file mode 100644 index 000000000..27a3e1bdb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py @@ -0,0 +1,43 @@ +"""``instagram`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured data from Instagram posts, reels, comments, and profiles." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md new file mode 100644 index 000000000..9efedc52a --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md @@ -0,0 +1,2 @@ +Instagram specialist: pulls structured data from Instagram — posts and reels (caption, likes, comments count, media URLs, owner, timestamp), a post's comments and replies, and profile/hashtag/place details (follower and post counts, bio, hashtag volume and top posts, place coordinates). Finds content by hashtag, profile, or place search, and compares fresh Instagram data against earlier findings in this chat. +Use whenever the task involves Instagram content or an instagram.com link. Triggers include "get this Instagram profile/post/reel", "find posts about X on Instagram", "how many followers/likes", "get the comments on this post", "what are people saying about this reel", "look up this hashtag/location", and comparisons against earlier Instagram results in this chat. Not for general web pages (use the web crawling specialist for non-Instagram URLs). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md new file mode 100644 index 000000000..7d5278afa --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -0,0 +1,68 @@ +You are the SurfSense Instagram sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live Instagram data gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it. + + + +- `instagram_scrape` +- `instagram_comments` +- `instagram_details` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Known profile/post/reel/hashtag/place links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). +- Finding content on a topic: call `instagram_scrape` with `search_queries` and the matching `search_type` (hashtag, profile, or place). +- Comments / sentiment on specific posts or reels: call `instagram_comments` with the post `urls`. +- Profile, hashtag, or place metadata (follower counts, bio, hashtag volume, coordinates): call `instagram_details`. +- Batch multiple URLs (or queries) into one call rather than many single-item calls. + +- Multi-post comment analysis: a batched comments result lists posts in order, so a truncated preview usually shows only the first post(s). Before summarizing, page the stored run (or `search_run` by post id) until you have read real comments for EVERY post in the batch — never infer one post's sentiment from another's, and never report a post as "limited data" while its comments sit unread in the run. +- Comparison requests: pull the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). + + + +- Use only tools in ``. +- An item whose `status` is not `success` returned no data — report it unavailable, never invent it. +- Anonymous Instagram access can be rate-limited or blocked; if a verb returns no data, report it unavailable and suggest a narrower retry rather than fabricating. +- Report only deltas you can point to in the evidence. Never fabricate facts, counts, quotes, or URLs. + + + +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Non-Instagram web pages belong to the web crawling specialist, not here. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable URL or search query — return `status=blocked` with the missing fields. +- Tool failure or access block: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the URLs you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw payloads. +- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. + + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py new file mode 100644 index 000000000..1bfbabad9 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py @@ -0,0 +1,29 @@ +"""``instagram`` sub-agent tools: the three Instagram capability verbs.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.instagram.comments.definition import INSTAGRAM_COMMENTS +from app.capabilities.instagram.details.definition import INSTAGRAM_DETAILS +from app.capabilities.instagram.scrape.definition import INSTAGRAM_SCRAPE + +NAME = "instagram" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [INSTAGRAM_SCRAPE, INSTAGRAM_COMMENTS, INSTAGRAM_DETAILS] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 34895a514..7a9b5b1f7 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -21,6 +21,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.google_maps.agent impor from app.agents.chat.multi_agent_chat.subagents.builtins.google_search.agent import ( build_subagent as build_google_search_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.instagram.agent import ( + build_subagent as build_instagram_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( build_subagent as build_knowledge_base_subagent, ) @@ -79,6 +82,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "google_drive": build_google_drive_subagent, "google_maps": build_google_maps_subagent, "google_search": build_google_search_subagent, + "instagram": build_instagram_subagent, "knowledge_base": build_knowledge_base_subagent, "mcp_discovery": build_mcp_discovery_subagent, "memory": build_memory_subagent, From 9da136070d638ce7df2befa81245bd957d564969 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:41 +0530 Subject: [PATCH 052/160] test(instagram): add platform-layer unit tests --- .../unit/platforms/instagram/__init__.py | 0 .../unit/platforms/instagram/test_budget.py | 97 ++++++ .../instagram/test_fetch_resilience.py | 308 ++++++++++++++++++ .../unit/platforms/instagram/test_parsers.py | 153 +++++++++ .../unit/platforms/instagram/test_skeleton.py | 106 ++++++ 5 files changed, 664 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/instagram/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_budget.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_parsers.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py diff --git a/surfsense_backend/tests/unit/platforms/instagram/__init__.py b/surfsense_backend/tests/unit/platforms/instagram/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_budget.py b/surfsense_backend/tests/unit/platforms/instagram/test_budget.py new file mode 100644 index 000000000..ea500be40 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_budget.py @@ -0,0 +1,97 @@ +"""Offline budget tests: per-target caps, cross-target de-dup, and the limit guard. + +No network. ``fetch_json`` is stubbed with a synthetic profile payload and the +fan-out proxy holders are replaced with no-ops, so the orchestrator's paging and +de-dup policy is exercised deterministically. +""" + +from __future__ import annotations + +from contextlib import asynccontextmanager + +import pytest + +from app.proprietary.platforms.instagram import scraper +from app.proprietary.platforms.instagram.schemas import InstagramScrapeInput + + +class _NoopHolder: + async def close(self) -> None: + return None + + +@pytest.fixture +def _stub_proxy(monkeypatch): + async def _open(): + return _NoopHolder() + + @asynccontextmanager + async def _bind(_holder): + yield _holder + + monkeypatch.setattr(scraper, "open_proxy_holder", _open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _bind) + + +def _profile_payload(n: int) -> dict: + return { + "data": { + "user": { + "id": "9", + "username": "acct", + "edge_owner_to_timeline_media": { + "count": n, + "edges": [ + {"node": {"id": str(i), "shortcode": f"S{i}"}} + for i in range(n) + ], + }, + } + } + } + + +async def test_per_target_cap_limits_media(_stub_proxy, monkeypatch): + async def _fetch(path, params=None): + return _profile_payload(50) + + monkeypatch.setattr(scraper, "fetch_json", _fetch) + model = InstagramScrapeInput( + resultsType="posts", + directUrls=["https://www.instagram.com/acct/"], + resultsLimit=5, + ) + items = [i async for i in scraper.iter_instagram(model)] + assert len(items) == 5 + + +async def test_cross_target_dedup_by_id(_stub_proxy, monkeypatch): + async def _fetch(path, params=None): + return _profile_payload(3) # both targets return ids 0,1,2 + + monkeypatch.setattr(scraper, "fetch_json", _fetch) + model = InstagramScrapeInput( + resultsType="posts", + directUrls=[ + "https://www.instagram.com/one/", + "https://www.instagram.com/two/", + ], + resultsLimit=10, + ) + items = [i async for i in scraper.iter_instagram(model)] + ids = sorted(i["id"] for i in items) + assert ids == ["0", "1", "2"] + + +async def test_scrape_instagram_honors_limit(_stub_proxy, monkeypatch): + async def _fetch(path, params=None): + return _profile_payload(50) + + monkeypatch.setattr(scraper, "fetch_json", _fetch) + model = InstagramScrapeInput( + resultsType="posts", + directUrls=["https://www.instagram.com/acct/"], + resultsLimit=100, + ) + items = await scraper.scrape_instagram(model, limit=7) + assert len(items) == 7 diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py new file mode 100644 index 000000000..eefae2fd9 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -0,0 +1,308 @@ +"""Offline resilience tests for the Instagram fetch seam and fan-out worker pool. + +No network. Fake sessions drive the ``csrftoken`` warm-up + rotate-on-block + +backoff paths deterministically (in live runs the first IP warms and returns +200s, so these branches rarely fire). Mirrors the reddit sibling's +``test_fetch_resilience.py`` shape, adapted to Instagram's cookie warm-up. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from app.proprietary.platforms.instagram import fetch, scraper +from app.proprietary.platforms.instagram.fetch import ( + InstagramAccessBlockedError, + _current_session, + fetch_json, +) + +_PAYLOAD = {"data": {"user": {"username": "natgeo"}}} + + +class _FakePage: + def __init__(self, status: int, *, cookies: dict | None = None, payload=None): + self.status = status + self.cookies = cookies or {} + self._payload = payload if payload is not None else _PAYLOAD + + def json(self): + return self._payload + + @property + def body(self) -> str: + return json.dumps(self._payload) + + +class _FakeSession: + """One 'IP': the warm-up GET mints csrftoken per flag; endpoint GETs return ``status``.""" + + def __init__(self, status: int = 200, *, csrftoken: bool = True, payload=None) -> None: + self.status = status + self.csrftoken = csrftoken + self.payload = payload + self.json_calls = 0 + self.warm_calls = 0 + + async def get(self, url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.instagram.com": + self.warm_calls += 1 + ck = {"csrftoken": "x", "mid": "y"} if self.csrftoken else {} + return _FakePage(200, cookies=ck) + self.json_calls += 1 + return _FakePage(self.status, payload=self.payload) + + +class _FakeHolder: + """Holder whose ``rotate()`` advances to the next fake session (a new IP).""" + + def __init__(self, sessions: list[_FakeSession]) -> None: + self._sessions = sessions + self.session = sessions[0] + self.rotations = 0 + self.warmed = False + + async def rotate(self): + self.rotations += 1 + self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] + self.warmed = False # cookies bind to the IP: re-warm on the fresh one + return self.session + + async def pace(self) -> None: + return None + + async def close(self) -> None: + return None + + +def _no_sleep(monkeypatch) -> None: + async def _noop(_seconds): + return None + + monkeypatch.setattr(fetch.asyncio, "sleep", _noop) + + +async def test_warms_then_returns_json(): + holder = _FakeHolder([_FakeSession(200, csrftoken=True)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/", {"username": "natgeo"}) + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 0 + assert holder.session.warm_calls == 1 # warmed exactly once + + +async def test_rotates_when_warm_fails_then_succeeds(): + holder = _FakeHolder( + [_FakeSession(200, csrftoken=False), _FakeSession(200, csrftoken=True)] + ) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + + +async def test_raises_when_no_ip_can_warm(): + holder = _FakeHolder( + [_FakeSession(200, csrftoken=False) for _ in range(fetch._MAX_ROTATIONS + 1)] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await fetch_json("api/v1/users/web_profile_info/") + except InstagramAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == fetch._MAX_ROTATIONS + + +async def test_rotates_and_rewarms_on_403(): + holder = _FakeHolder([_FakeSession(403), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + assert holder.session.warm_calls == 1 # re-warmed on the fresh IP + + +async def test_rotates_on_401_login_wall(): + holder = _FakeHolder([_FakeSession(401), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + + +async def test_404_returns_none_without_rotating(): + holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/tags/web_info/") + finally: + _current_session.reset(token) + assert result is None + assert holder.rotations == 0 + + +async def test_429_backs_off_without_rotating(monkeypatch): + _no_sleep(monkeypatch) + session = _FakeSession(429) + + async def _get(url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.instagram.com": + session.warm_calls += 1 + return _FakePage(200, cookies={"csrftoken": "x"}) + session.json_calls += 1 + return _FakePage(429 if session.json_calls == 1 else 200) + + session.get = _get # type: ignore[method-assign] + holder = _FakeHolder([session]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 0 + + +async def test_persistent_403_raises_blocked(monkeypatch): + _no_sleep(monkeypatch) + holder = _FakeHolder([_FakeSession(403) for _ in range(fetch._MAX_ROTATIONS + 1)]) + token = _current_session.set(holder) + try: + raised = False + try: + await fetch_json("api/v1/users/web_profile_info/") + except InstagramAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == fetch._MAX_ROTATIONS + + +class _TrackingHolder: + """Fake fan-out session holder that records whether it was closed.""" + + def __init__(self) -> None: + self.closed = False + + async def close(self) -> None: + self.closed = True + + +async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch): + holders: list[_TrackingHolder] = [] + + async def _fake_open(): + h = _TrackingHolder() + holders.append(h) + return h + + @asynccontextmanager + async def _fake_bind(_holder): + yield _holder + + monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) + + async def _job(n: int) -> AsyncIterator[dict]: + for i in range(5): + yield {"job": n, "i": i} + + jobs = [_job(n) for n in range(20)] + gen = scraper.fan_out(jobs, concurrency=4) + collected = [] + async for item in gen: + collected.append(item) + if len(collected) >= 3: + break + await gen.aclose() + + assert len(collected) >= 3 + assert holders, "workers should have opened sessions" + assert all(h.closed for h in holders), "a worker leaked its proxy session" + + +async def test_fan_out_empty_jobs_is_noop(): + out = [x async for x in scraper.fan_out([])] + assert out == [] + + +def _profile_payload(username: str, n: int) -> dict: + # IDs namespaced per target so cross-target de-dup doesn't collapse them. + return { + "data": { + "user": { + "id": f"u_{username}", + "username": username, + "edge_owner_to_timeline_media": { + "count": n, + "edges": [{"node": {"id": f"{username}:{i}"}} for i in range(n)], + }, + } + } + } + + +async def test_scrape_instagram_closes_sessions_when_limit_stops_inflight_workers( + monkeypatch, +): + """Hitting ``limit`` must tear down the whole fan-out chain deterministically. + + Regression: closing the outer ``iter_instagram`` generator does NOT + synchronously close the inner ``fan_out`` it loops over — CPython defers that + to async-gen GC — so without an explicit ``aclosing`` the collector's early + ``break`` leaked every warm proxy session that was still mid-fetch. The + ``fan_out``-direct test misses this because instant jobs self-drain before + cancellation ever runs; here each fetch yields to the loop so workers are + genuinely in-flight when the limit trips. + """ + holders: list[_TrackingHolder] = [] + + async def _fake_open(): + h = _TrackingHolder() + holders.append(h) + return h + + @asynccontextmanager + async def _fake_bind(_holder): + yield _holder + + async def _fetch(path, params=None): + await asyncio.sleep(0) # yield control: keep sibling workers in-flight + username = (params or {}).get("username", "acct") + return _profile_payload(username, 5) + + monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) + monkeypatch.setattr(scraper, "fetch_json", _fetch) + + model = scraper.InstagramScrapeInput( + resultsType="posts", + directUrls=[f"https://www.instagram.com/acct{i}/" for i in range(50)], + resultsLimit=5, + ) + items = await scraper.scrape_instagram(model, limit=3) + + assert len(items) == 3 + assert holders, "workers should have opened sessions" + assert all(h.closed for h in holders), "early stop leaked a proxy session" diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py new file mode 100644 index 000000000..6f6c1411e --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -0,0 +1,153 @@ +"""Offline parser tests: raw web JSON -> flat item dicts. + +Synthetic nodes cover the GraphQL ``edge_*`` flattening the anonymous web +payloads use. A fixture-pinned test runs only when a captured fixture is present +(the live e2e script dumps trimmed, PII-anonymized fixtures), so the suite stays +green offline. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from app.proprietary.platforms.instagram.parsers import ( + parse_comment, + parse_hashtag, + parse_media, + parse_place, + parse_profile, +) + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _edge(nodes: list[dict]) -> dict: + return {"edges": [{"node": n} for n in nodes]} + + +def test_parse_media_flattens_edges_and_extracts_tags(): + node = { + "id": "1", + "shortcode": "Cabc", + "__typename": "GraphImage", + "taken_at_timestamp": 1_600_000_000, + "edge_media_to_caption": _edge([{"text": "love #nasa shot @buzz"}]), + "edge_media_to_comment": {"count": 7}, + "edge_liked_by": {"count": 42}, + "owner": {"username": "natgeo", "id": "9"}, + } + item = parse_media(node) + assert item["shortCode"] == "Cabc" + assert item["type"] == "Image" + assert item["hashtags"] == ["nasa"] + assert item["mentions"] == ["buzz"] + assert item["commentsCount"] == 7 + assert item["likesCount"] == 42 + assert item["ownerUsername"] == "natgeo" + assert item["url"] == "https://www.instagram.com/p/Cabc/" + + +def test_parse_media_passes_through_hidden_like_count(): + # Instagram reports -1 when the creator hid likes; never coerce it away. + item = parse_media({"id": "1", "edge_liked_by": {"count": -1}}) + assert item["likesCount"] == -1 + + +def test_parse_media_marks_video_type(): + item = parse_media({"id": "1", "is_video": True, "video_view_count": 99}) + assert item["type"] == "Video" + assert item["videoViewCount"] == 99 + + +def test_parse_comment(): + node = { + "id": "c1", + "text": "nice", + "created_at": 1_600_000_000, + "shortcode": "Cabc", + "owner": {"username": "bob", "id": "5"}, + "edge_liked_by": {"count": 3}, + } + item = parse_comment(node, post_url="https://www.instagram.com/p/Cabc/") + assert item["id"] == "c1" + assert item["text"] == "nice" + assert item["ownerUsername"] == "bob" + assert item["likesCount"] == 3 + assert item["postUrl"] == "https://www.instagram.com/p/Cabc/" + + +def test_parse_profile_flattens_counts_and_latest_posts(): + user = { + "id": "9", + "username": "natgeo", + "full_name": "Nat Geo", + "edge_followed_by": {"count": 1000}, + "edge_follow": {"count": 50}, + "edge_owner_to_timeline_media": { + "count": 2, + "edges": [{"node": {"id": "p1", "shortcode": "A"}}], + }, + } + item = parse_profile(user) + assert item["detailKind"] == "profile" + assert item["username"] == "natgeo" + assert item["followersCount"] == 1000 + assert item["followsCount"] == 50 + assert item["postsCount"] == 2 + assert len(item["latestPosts"]) == 1 + + +def test_parse_hashtag(): + data = { + "data": { + "id": "h1", + "name": "crossfit", + "edge_hashtag_to_media": { + "count": 5, + "edges": [{"node": {"id": "m1", "shortcode": "A"}}], + }, + "edge_hashtag_to_top_posts": { + "edges": [{"node": {"id": "t1", "shortcode": "B"}}] + }, + } + } + item = parse_hashtag(data) + assert item["detailKind"] == "hashtag" + assert item["name"] == "crossfit" + assert item["postsCount"] == 5 + assert len(item["topPosts"]) == 1 + assert len(item["posts"]) == 1 + + +def test_parse_place(): + data = { + "location": { + "id": "7538318", + "name": "Copenhagen", + "slug": "copenhagen", + "edge_location_to_media": { + "count": 3, + "edges": [{"node": {"id": "m1", "shortcode": "A"}}], + }, + } + } + item = parse_place(data) + assert item["detailKind"] == "place" + assert item["name"] == "Copenhagen" + assert item["location_id"] == "7538318" + assert len(item["posts"]) == 1 + + +@pytest.mark.skipif( + not (_FIXTURES / "profile.json").exists(), + reason="captured fixture absent (run scripts/e2e_instagram_scraper.py to dump)", +) +def test_fixture_profile_maps(): + raw = json.loads((_FIXTURES / "profile.json").read_text()) + user = raw.get("data", {}).get("user", raw) + item = parse_profile(user) + assert item["detailKind"] == "profile" + assert item["username"] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py new file mode 100644 index 000000000..59017c81c --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py @@ -0,0 +1,106 @@ +"""Offline skeleton tests: input surface parity + URL classification. + +No network. Locks the two invariants the reference-compatible surface promises — +no auth fields ever, and additive ``extra="allow"`` parity — plus the full +``url_resolver`` classification/normalization table (``_u/`` and profilecard +stripping, story→profile, ID-only locations, numeric post-ID flagging). +""" + +from __future__ import annotations + +from app.proprietary.platforms.instagram.schemas import ( + InstagramMediaItem, + InstagramScrapeInput, +) +from app.proprietary.platforms.instagram.url_resolver import resolve_url + + +def test_input_has_no_auth_fields(): + # Anonymous-only: the input surface must never expose a login/credential seam. + forbidden = { + "sessionid", + "username", + "password", + "cookies", + "authorization", + "proxyConfiguration", + "loginCredentials", + } + assert forbidden.isdisjoint(InstagramScrapeInput.model_fields) + + +def test_input_defaults(): + model = InstagramScrapeInput() + assert model.resultsType == "posts" + assert model.searchType == "hashtag" + assert model.directUrls == [] + assert model.addParentData is False + + +def test_input_allows_extra_inert_fields(): + # A reference field the acquisition layer doesn't source is accepted, not rejected. + model = InstagramScrapeInput(enhanceUserSearchWithFacebookPage="x") + assert model.model_dump().get("enhanceUserSearchWithFacebookPage") == "x" + + +def test_media_item_to_output_keeps_none_keys(): + out = InstagramMediaItem(id="123", shortCode="abc").to_output() + assert out["id"] == "123" + assert out["shortCode"] == "abc" + # Unsourced fields stay present as None / [] for additive parity. + assert out["likesCount"] is None + assert out["requestErrorMessages"] == [] + + +def test_resolve_profile(): + r = resolve_url("https://www.instagram.com/natgeo/") + assert r.kind == "profile" + assert r.value == "natgeo" + + +def test_resolve_bare_profile_id(): + r = resolve_url("natgeo") + assert r.kind == "profile" and r.value == "natgeo" + + +def test_resolve_post_and_reel(): + r = resolve_url("https://www.instagram.com/p/Cabc123/") + assert r.kind == "post" and r.value == "Cabc123" and r.numeric_post_id is False + r = resolve_url("https://www.instagram.com/reel/Cxyz/") + assert r.kind == "reel" and r.value == "Cxyz" + + +def test_resolve_hashtag(): + r = resolve_url("https://www.instagram.com/explore/tags/crossfit/") + assert r.kind == "hashtag" and r.value == "crossfit" + + +def test_resolve_place_with_slug_and_id_only(): + with_slug = resolve_url( + "https://www.instagram.com/explore/locations/7538318/copenhagen/" + ) + assert with_slug.kind == "place" and with_slug.value == "7538318" + assert with_slug.slug == "copenhagen" + id_only = resolve_url("https://www.instagram.com/explore/locations/7538318/") + assert id_only.kind == "place" and id_only.value == "7538318" + + +def test_resolve_strips_u_and_profilecard(): + stripped_u = resolve_url("https://www.instagram.com/_u/natgeo/") + assert stripped_u.kind == "profile" and stripped_u.value == "natgeo" + card = resolve_url("https://www.instagram.com/natgeo/profilecard/") + assert card.kind == "profile" and card.value == "natgeo" + + +def test_resolve_story_reduces_to_profile(): + r = resolve_url("https://www.instagram.com/stories/natgeo/12345/") + assert r.kind == "profile" and r.value == "natgeo" + + +def test_resolve_numeric_post_id_flagged(): + r = resolve_url("https://www.instagram.com/p/12345/") + assert r.kind == "post" and r.numeric_post_id is True + + +def test_resolve_rejects_non_instagram_host(): + assert resolve_url("https://example.com/natgeo/") is None From 2e66e714e2cd3818f3224232ef9b2502722d4c3f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:48 +0530 Subject: [PATCH 053/160] test(instagram): add capability unit tests --- .../unit/capabilities/instagram/__init__.py | 0 .../capabilities/instagram/test_executor.py | 104 ++++++++++++++++++ .../capabilities/instagram/test_registry.py | 35 ++++++ .../capabilities/instagram/test_schemas.py | 80 ++++++++++++++ 4 files changed, 219 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/test_registry.py create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py diff --git a/surfsense_backend/tests/unit/capabilities/instagram/__init__.py b/surfsense_backend/tests/unit/capabilities/instagram/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py new file mode 100644 index 000000000..9087b0bcf --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py @@ -0,0 +1,104 @@ +"""Executor tests: lean verb input → ``InstagramScrapeInput`` mapping + wrapping. + +A fake scraper captures the actor input the executor built (no network), so the +snake_case→camelCase mapping and the ``InstagramAccessBlockedError`` → +``ForbiddenError`` translation are asserted deterministically. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.instagram.comments.executor import build_comments_executor +from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.instagram.details.executor import build_details_executor +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput +from app.capabilities.instagram.scrape.executor import build_scrape_executor +from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, +) + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + def __init__(self, items: list[dict]) -> None: + self.items = items + self.calls: list[tuple[InstagramScrapeInput, int | None]] = [] + + async def __call__(self, actor_input, *, limit=None): + self.calls.append((actor_input, limit)) + return self.items + + +async def test_scrape_maps_urls_and_wraps_items(): + fake = _FakeScraper([{"id": "1", "shortCode": "abc", "caption": "hi"}]) + execute = build_scrape_executor(fake) + out = await execute(ScrapeInput(urls=["https://www.instagram.com/natgeo/"])) + assert isinstance(out, ScrapeOutput) + assert out.items[0].shortCode == "abc" + actor_input, limit = fake.calls[0] + assert actor_input.resultsType == "posts" + assert actor_input.directUrls == ["https://www.instagram.com/natgeo/"] + assert actor_input.search == "" + assert limit == 10 # default max_items forwarded as the collector limit + + +async def test_scrape_joins_search_queries(): + fake = _FakeScraper([]) + execute = build_scrape_executor(fake) + await execute(ScrapeInput(search_queries=["fit", "gym"], search_type="hashtag")) + actor_input, _ = fake.calls[0] + assert actor_input.search == "fit,gym" + assert actor_input.searchType == "hashtag" + assert actor_input.directUrls == [] + + +async def test_scrape_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input, *, limit=None): + raise InstagramAccessBlockedError("login wall") + + execute = build_scrape_executor(_blocked) + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(urls=["x"])) + + +async def test_comments_maps_flags(): + fake = _FakeScraper([{"id": "c1", "text": "nice"}]) + execute = build_comments_executor(fake) + out = await execute( + CommentsInput( + urls=["https://www.instagram.com/p/Cabc/"], + newest_first=True, + include_replies=True, + max_comments_per_post=25, + ) + ) + assert isinstance(out, CommentsOutput) + assert out.items[0].text == "nice" + actor_input, _ = fake.calls[0] + assert actor_input.resultsType == "comments" + assert actor_input.isNewestComments is True + assert actor_input.includeNestedComments is True + assert actor_input.resultsLimit == 25 + + +async def test_details_maps_and_wraps_discriminated_items(): + fake = _FakeScraper( + [ + { + "detailKind": "profile", + "username": "natgeo", + "url": "https://www.instagram.com/natgeo/", + } + ] + ) + execute = build_details_executor(fake) + out = await execute(DetailsInput(urls=["https://www.instagram.com/natgeo/"])) + assert isinstance(out, DetailsOutput) + assert out.items[0].username == "natgeo" + actor_input, _ = fake.calls[0] + assert actor_input.resultsType == "details" diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py new file mode 100644 index 000000000..89257253a --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py @@ -0,0 +1,35 @@ +"""The instagram namespace registers its three verbs for the doors/agent to read. + +Unlike the stale reddit assertion (``billing_unit is None``), these assert the +real meters — the Capability definitions are the source of truth. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + instagram, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit + +pytestmark = pytest.mark.unit + + +def test_instagram_scrape_registered_with_item_meter(): + cap = get_capability("instagram.scrape") + assert cap.name == "instagram.scrape" + assert cap.billing_unit is BillingUnit.INSTAGRAM_ITEM + + +def test_instagram_comments_registered_with_comment_meter(): + cap = get_capability("instagram.comments") + assert cap.name == "instagram.comments" + assert cap.billing_unit is BillingUnit.INSTAGRAM_COMMENT + + +def test_instagram_details_registered_with_item_meter(): + cap = get_capability("instagram.details") + assert cap.name == "instagram.details" + assert cap.billing_unit is BillingUnit.INSTAGRAM_ITEM diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py new file mode 100644 index 000000000..13efb8a48 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py @@ -0,0 +1,80 @@ +"""``instagram.*`` input guards: source exclusivity and bounded batches.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.instagram.comments.schemas import CommentsInput +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput +from app.capabilities.instagram.scrape.schemas import ( + MAX_INSTAGRAM_ITEMS, + MAX_INSTAGRAM_SOURCES, + ScrapeInput, +) + +pytestmark = pytest.mark.unit + + +def test_scrape_rejects_no_source(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_scrape_rejects_both_sources(): + with pytest.raises(ValidationError): + ScrapeInput(urls=["https://www.instagram.com/natgeo/"], search_queries=["fit"]) + + +def test_scrape_accepts_urls_only(): + payload = ScrapeInput(urls=["https://www.instagram.com/natgeo/"]) + assert payload.search_queries == [] + assert payload.estimated_units == payload.max_items + + +def test_scrape_bounds(): + with pytest.raises(ValidationError): + ScrapeInput( + urls=["https://www.instagram.com/x/"], + max_items=MAX_INSTAGRAM_ITEMS + 1, + ) + with pytest.raises(ValidationError): + ScrapeInput( + urls=[ + f"https://www.instagram.com/u{i}/" + for i in range(MAX_INSTAGRAM_SOURCES + 1) + ] + ) + + +def test_comments_requires_urls_and_caps_at_50(): + with pytest.raises(ValidationError): + CommentsInput(urls=[]) + with pytest.raises(ValidationError): + CommentsInput( + urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=51 + ) + ok = CommentsInput( + urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=50 + ) + assert ok.max_comments_per_post == 50 + + +def test_details_discriminated_union_selects_by_detail_kind(): + out = DetailsOutput( + items=[ + {"detailKind": "profile", "username": "natgeo"}, + {"detailKind": "hashtag", "name": "fit"}, + {"detailKind": "place", "name": "Copenhagen"}, + ] + ) + kinds = [type(i).__name__ for i in out.items] + assert kinds == ["InstagramProfile", "InstagramHashtag", "InstagramPlace"] + assert out.billable_units == 3 + + +def test_details_rejects_both_sources(): + with pytest.raises(ValidationError): + DetailsInput( + urls=["https://www.instagram.com/natgeo/"], search_queries=["x"] + ) From 458d223ef8bfbbe534ccd24c3801ac360ce12710 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:54 +0530 Subject: [PATCH 054/160] test(billing): cover Instagram item and comment billing --- .../tests/unit/capabilities/test_billing.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/surfsense_backend/tests/unit/capabilities/test_billing.py b/surfsense_backend/tests/unit/capabilities/test_billing.py index 36d9a2978..85187b5f5 100644 --- a/surfsense_backend/tests/unit/capabilities/test_billing.py +++ b/surfsense_backend/tests/unit/capabilities/test_billing.py @@ -427,3 +427,54 @@ async def test_platform_gate_disabled_is_noop(monkeypatch): ) session.execute.assert_not_called() + + +# =================================================================== +# Instagram per-item / per-comment billing +# =================================================================== + + +async def test_instagram_item_charges_owner_per_item( + monkeypatch, record_usage, _enable_platform_billing +): + monkeypatch.setattr(config, "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", 3500) + session, user = _make_session(_OWNER, balance_micros=1_000_000) + + charged = await charge_capability( + _FakePlatformOutput(4), BillingUnit.INSTAGRAM_ITEM, _ctx(session) + ) + + assert charged == 4 * 3500 + assert user.credit_micros_balance == 1_000_000 - 4 * 3500 + kwargs = record_usage.await_args.kwargs + assert kwargs["usage_type"] == "instagram_item" + + +async def test_instagram_comment_charges_owner_per_comment( + monkeypatch, record_usage, _enable_platform_billing +): + monkeypatch.setattr(config, "INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", 1500) + session, user = _make_session(_OWNER, balance_micros=1_000_000) + + charged = await charge_capability( + _FakePlatformOutput(6), BillingUnit.INSTAGRAM_COMMENT, _ctx(session) + ) + + assert charged == 6 * 1500 + assert user.credit_micros_balance == 1_000_000 - 6 * 1500 + kwargs = record_usage.await_args.kwargs + assert kwargs["usage_type"] == "instagram_comment" + + +async def test_instagram_gate_blocks_when_worst_case_exceeds_balance( + monkeypatch, _enable_platform_billing +): + monkeypatch.setattr(config, "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", 3500) + session = _gate_session(_OWNER, balance_micros=5000) # affords 1 item, not 2 + + with pytest.raises(InsufficientCreditsError): + await gate_capability( + _FakePlatformInput(estimated_units=2), + BillingUnit.INSTAGRAM_ITEM, + _ctx(session), + ) From e151346fbed6a147c7096d3b3678a561566b894d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:03:23 +0530 Subject: [PATCH 055/160] test(chat): include Instagram in subagent composition test --- .../unit/agents/multi_agent_chat/test_subagent_composition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index c3ad04250..1d04f2ebf 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -32,6 +32,7 @@ _EXPECTED_SUBAGENTS = frozenset( "google_drive", "google_maps", "google_search", + "instagram", "knowledge_base", "mcp_discovery", "memory", From 7309ca5ca05e0a15d6c25e27634965948b948a27 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:03:29 +0530 Subject: [PATCH 056/160] test(instagram): add manual e2e scraper script --- .../scripts/e2e_instagram_scraper.py | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_instagram_scraper.py diff --git a/surfsense_backend/scripts/e2e_instagram_scraper.py b/surfsense_backend/scripts/e2e_instagram_scraper.py new file mode 100644 index 000000000..95625fd06 --- /dev/null +++ b/surfsense_backend/scripts/e2e_instagram_scraper.py @@ -0,0 +1,227 @@ +"""Manual functional e2e for the Instagram scraper (app/proprietary/platforms/instagram). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_instagram_scraper.py + # or: .venv/bin/python scripts/e2e_instagram_scraper.py + +This is NOT a pytest test (it needs live network + a residential/custom proxy). +It: + + Step 0 — go/no-go probe: open a proxy session, mint the anonymous + ``csrftoken``/``mid`` cookies, then fetch ``web_profile_info`` on the SAME + sticky IP and assert it returns a profile. If this fails the whole + approach is invalid — later steps are skipped. + Step 1 — scrape a profile's posts. + Step 2 — scrape a profile's reels. + Step 3 — fetch comments for a discovered post URL. + Step 4 — fetch profile / hashtag / place details. + Step 5 — run a discovery search. + Step 6 — dump trimmed, PII-anonymized raw fixtures into + tests/unit/platforms/instagram/fixtures/ for the offline parser tests. +""" + +import asyncio +import json +import sys +from pathlib import Path + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.instagram import ( # noqa: E402 + InstagramScrapeInput, + scrape_instagram, +) +from app.proprietary.platforms.instagram.fetch import ( # noqa: E402 + fetch_json, + proxy_session, + warm_session, +) + +_PROFILE = "natgeo" +_HASHTAG = "https://www.instagram.com/explore/tags/travel/" +_PLACE = "https://www.instagram.com/explore/locations/213385402/new-york-new-york/" +_SEARCH_TERM = "coffee" + +_FIXTURE_DIR = ( + _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures" +) + +# Fields to strip from dumped fixtures so we never commit PII / volatile tokens. +_PII_KEYS = frozenset( + {"profile_pic_url", "profile_pic_url_hd", "display_url", "video_url", "biography"} +) + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +def _anonymize(obj): + """Recursively blank PII-ish string values so fixtures are safe to commit.""" + if isinstance(obj, dict): + return { + k: ("" if k in _PII_KEYS and v else _anonymize(v)) + for k, v in obj.items() + } + if isinstance(obj, list): + return [_anonymize(x) for x in obj] + return obj + + +async def step0_probe() -> bool: + _hr("STEP 0 — go/no-go: csrftoken warm-up + sticky web_profile_info") + async with proxy_session() as holder: + if holder.session is None: + return _check( + "proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds" + ) + minted = await warm_session(holder.session) + holder.warmed = True # don't let fetch_json re-warm; we just warmed it + _check("csrftoken warm-up minted a session", minted) + data = await fetch_json( + "api/v1/users/web_profile_info/", {"username": _PROFILE} + ) + user = (data or {}).get("data", {}).get("user") if isinstance(data, dict) else None + print(f" web_profile_info({_PROFILE}) -> user={'yes' if user else 'no'}") + return _check("sticky web_profile_info", minted and bool(user)) + + +async def step1_posts() -> bool: + _hr("STEP 1 — profile posts") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="posts", + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], + resultsLimit=5, + ), + limit=5, + ) + for it in items[:5]: + print(f" - {it.get('shortCode')} | likes={it.get('likesCount')}") + return _check("profile returned posts", len(items) > 0, f"{len(items)} posts") + + +async def step2_reels() -> bool: + _hr("STEP 2 — profile reels") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="reels", + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], + resultsLimit=5, + ), + limit=5, + ) + print(f" {len(items)} reels for {_PROFILE}") + return _check("reels returned items", len(items) >= 0, f"{len(items)} reels") + + +async def step3_comments(post_url: str | None) -> bool: + _hr("STEP 3 — comments for a post") + if not post_url: + return _check("had a post URL", False, "step 1 found no post") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="comments", directUrls=[post_url], resultsLimit=10 + ), + limit=10, + ) + print(f" {len(items)} comments for {post_url}") + return _check("comments returned", len(items) >= 0, f"{len(items)} comments") + + +async def step4_details() -> bool: + _hr("STEP 4 — profile / hashtag / place details") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="details", + directUrls=[ + f"https://www.instagram.com/{_PROFILE}/", + _HASHTAG, + _PLACE, + ], + ), + limit=10, + ) + kinds = sorted({i.get("detailKind") for i in items}) + print(f" detail kinds={kinds}") + return _check("details returned", len(items) > 0, f"{len(items)} items {kinds}") + + +async def step5_search() -> bool: + _hr("STEP 5 — discovery search") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="posts", + search=_SEARCH_TERM, + searchType="hashtag", + searchLimit=3, + resultsLimit=3, + ), + limit=9, + ) + print(f" {len(items)} items for search '{_SEARCH_TERM}'") + return _check("search returned results", len(items) >= 0, f"{len(items)} items") + + +async def step6_dump_fixtures() -> bool: + _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests") + profile = await fetch_json( + "api/v1/users/web_profile_info/", {"username": _PROFILE} + ) + _FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + wrote = [] + if isinstance(profile, dict) and profile.get("data", {}).get("user"): + (_FIXTURE_DIR / "profile.json").write_text( + json.dumps(_anonymize(profile)), encoding="utf-8" + ) + wrote.append("profile.json") + return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}") + + +async def _first_post_url() -> str | None: + """Discover a live post URL from the target profile's first page.""" + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="posts", + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], + resultsLimit=1, + ), + limit=1, + ) + return items[0].get("url") if items else None + + +async def main() -> int: + results = [await step0_probe()] + if not results[-1]: + print("\ncookie probe failed — the approach is invalid on this IP/proxy.") + print("Aborting remaining steps.") + return 1 + results.append(await step1_posts()) + results.append(await step2_reels()) + post_url = await _first_post_url() + results.append(await step3_comments(post_url)) + results.append(await step4_details()) + results.append(await step5_search()) + results.append(await step6_dump_fixtures()) + _hr("SUMMARY") + print(f" {sum(results)}/{len(results)} steps passed") + return 0 if all(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) From d00030d97cac04b029e462950b5312fa1e2d0803 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:03:34 +0530 Subject: [PATCH 057/160] feat(web): add Instagram playground platform and brand icon --- surfsense_web/lib/playground/catalog.ts | 11 +++++++++++ surfsense_web/lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/instagram.svg | 1 + 3 files changed, 13 insertions(+) create mode 100644 surfsense_web/public/connectors/instagram.svg diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 9b508228a..b1287e447 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -2,6 +2,7 @@ import type { ComponentType } from "react"; import { GoogleMapsIcon, GoogleSearchIcon, + InstagramIcon, RedditIcon, WebIcon, YouTubeIcon, @@ -48,6 +49,16 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { name: "youtube.comments", verb: "comments", label: "Comments" }, ], }, + { + id: "instagram", + label: "Instagram", + icon: InstagramIcon, + verbs: [ + { name: "instagram.scrape", verb: "scrape", label: "Scrape" }, + { name: "instagram.comments", verb: "comments", label: "Comments" }, + { name: "instagram.details", verb: "details", label: "Details" }, + ], + }, { id: "google_maps", label: "Google Maps", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index e7a443c2f..460b7f7a3 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -24,6 +24,7 @@ function brandIcon(src: string, alt: string) { export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); +export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/instagram.svg b/surfsense_web/public/connectors/instagram.svg new file mode 100644 index 000000000..81233a38e --- /dev/null +++ b/surfsense_web/public/connectors/instagram.svg @@ -0,0 +1 @@ + From fa2ac406f363d9ea966668f63fde6352bba0f2d2 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:07:19 +0530 Subject: [PATCH 058/160] feat(web): add Instagram connector marketing page --- .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/instagram.tsx | 291 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 surfsense_web/lib/connectors-marketing/instagram.tsx diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 901670f8a..429fe9b59 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -1,5 +1,6 @@ import { googleMaps } from "./google-maps"; import { googleSearch } from "./google-search"; +import { instagram } from "./instagram"; import { reddit } from "./reddit"; import type { ConnectorPageContent } from "./types"; import { webCrawl } from "./web-crawl"; @@ -11,6 +12,7 @@ export type { ConnectorPageContent } from "./types"; const CONNECTOR_LIST: ConnectorPageContent[] = [ reddit, youtube, + instagram, googleMaps, googleSearch, webCrawl, diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx new file mode 100644 index 000000000..33b963eb8 --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -0,0 +1,291 @@ +import { IconBrandInstagram } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const instagram: ConnectorPageContent = { + slug: "instagram", + name: "Instagram", + icon: IconBrandInstagram, + + metaTitle: "Instagram Scraper API for Social Listening | SurfSense", + metaDescription: + "Scrape public Instagram posts, reels, comments, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", + keywords: [ + "instagram scraper", + "instagram scraper api", + "instagram api", + "instagram api alternative", + "scrape instagram", + "instagram graph api alternative", + "instagram comment scraper", + "instagram profile scraper", + "instagram hashtag scraper", + "instagram data api", + "instagram mcp server", + "instagram sentiment analysis", + "social listening", + ], + + h1: "Instagram Scraper API for Social Listening and Creator Research", + heroLede: + "The SurfSense Instagram API extracts public posts, reels, comments, and profile, hashtag, and place details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post and what their audiences say, so you spot trends and sentiment first.", + + transcript: { + prompt: "Pull recent reels from @competitor and tell me what the comments think", + toolCall: + 'instagram.scrape({ urls: ["instagram.com/competitor/"],\n result_type: "reels", max_items: 20 })', + rows: [ + { + primary: "Behind the scenes of our new launch", + secondary: "@competitor · 84.2k likes · 1,203 comments", + tag: "top reel", + }, + { + primary: "Comments skew positive on price, negative on shipping", + secondary: "1,203 comments · 0.71 positive", + tag: "sentiment", + }, + { + primary: "3 creators tagged asking for a collab", + secondary: "@a · @b · @c · buying intent", + tag: "lead signal", + }, + ], + resultSummary: "20 reels · 4,910 comments · surfaced in 2.4s", + }, + + extractIntro: + "Every call returns structured items keyed by type. Point the API at a profile, post, reel, hashtag, or place URL, or discover content with a search query.", + extractFields: [ + { + label: "Posts & Reels", + description: + "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.", + }, + { + label: "Comments", + description: "Comment text, author, like and reply counts, and nested replies for any post.", + }, + { + label: "Profiles", + description: + "Follower, following, and post counts, bio, external URL, verified and business flags.", + }, + { + label: "Hashtags", + description: "Post volume, top posts, and recent posts for any /explore/tags/ hashtag.", + }, + { + label: "Places", + description: "Name, coordinates, address, and recent posts for any /explore/locations/ place.", + }, + { + label: "Owner & Media", + description: + "Owner username and id on every item, plus image and video URLs, alt text, and view counts.", + }, + ], + + useCasesHeading: "What teams do with the Instagram API", + useCases: [ + { + title: "Creator and competitor monitoring", + description: + "Track what your competitors and target creators post, and how their audiences react. Feed the stream to an agent that flags viral formats, launches, and shifts in engagement the moment they land.", + }, + { + title: "Audience sentiment and comment mining", + description: + "Pull full comment threads on a post or reel and score them for sentiment, so you can measure how a launch, a collab, or a campaign actually resonated with real followers.", + }, + { + title: "Hashtag and trend research", + description: + "Map the volume and top content behind any hashtag before you spend on a campaign. Turn trend research into a content calendar your team can act on.", + }, + { + title: "Influencer vetting and outreach", + description: + "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership, and surface who is already mentioning your brand.", + }, + ], + + comparison: { + heading: "An Instagram API alternative built for agents", + intro: + "The official Instagram Graph API requires a Business account, app review, and access tokens, and it can't read arbitrary public profiles. Here is how SurfSense compares.", + columnLabel: "Instagram Graph API", + rows: [ + { + feature: "Access", + official: "Business/Creator account + app review + tokens", + surfsense: "Public data only, no login or account required", + }, + { + feature: "Coverage", + official: "Mostly your own or connected accounts", + surfsense: "Any public profile, post, hashtag, or place", + }, + { + feature: "Comments", + official: "Limited to accounts you manage", + surfsense: "Public comments and replies on any post, up to 50 per post", + }, + { + feature: "Setup", + official: "Register an app, pass review, manage tokens", + surfsense: "One API key, or add the MCP server to your agent", + }, + { + feature: "Agent-ready", + official: "No; you build the harness yourself", + surfsense: "MCP server exposes instagram.scrape as a native tool", + }, + ], + }, + + api: { + platform: "instagram", + verb: "scrape", + mcpTool: "instagram.scrape", + requestBody: { + urls: ["instagram.com/natgeo/"], + result_type: "reels", + max_items: 20, + }, + }, + + schema: { + requestNote: + "Provide exactly one source: urls OR search_queries (never both). Up to 20 sources per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "Instagram URLs or bare profile IDs: profile, post (/p/), reel (/reel/), hashtag (/explore/tags/), or place (/explore/locations/). Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Discovery keywords (hashtags as plaintext, no '#'). Provide these OR urls, not both. Max 20.", + }, + { + name: "search_type", + type: "string", + defaultValue: '"hashtag"', + description: + "What to discover from search_queries: hashtag, profile, place, or user. Only used with search_queries.", + }, + { + name: "result_type", + type: "string", + defaultValue: '"posts"', + description: "Which feed to return: posts, reels, or mentions. 'mentions' requires profile URLs.", + }, + { + name: "newer_than", + type: "string", + description: + "Only return posts newer than this: YYYY-MM-DD, ISO timestamp, or relative ('1 day', '2 months'), UTC.", + }, + { + name: "skip_pinned_posts", + type: "boolean", + defaultValue: "false", + description: "Exclude pinned posts in posts mode.", + }, + { + name: "max_per_target", + type: "integer", + defaultValue: "10", + description: "Max results per URL or per discovered target.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "10", + description: "Max total items to return across all sources. 1 to 100.", + }, + { + name: "add_parent_data", + type: "boolean", + defaultValue: "false", + description: "Attach a dataSource block to each feed item describing its source.", + }, + ], + responseNote: + "The response is { items: [...] } with one flat media item per result. Fields the anonymous endpoints do not expose are null. One returned item is one billable unit.", + response: [ + { + name: "id / shortCode / url", + type: "string", + description: "Identity and provenance: the media id, shortcode, and canonical post URL.", + }, + { + name: "type", + type: "string", + description: "Media type: Image, Video, or Sidecar (carousel).", + }, + { + name: "caption / hashtags / mentions", + type: "string / string[]", + description: "Caption text plus the hashtags and @-mentions parsed from it.", + }, + { + name: "likesCount / commentsCount", + type: "integer", + description: "Engagement counts. likesCount is -1 when the creator hides likes.", + }, + { + name: "displayUrl / videoUrl / videoViewCount", + type: "string / integer", + description: "Media URLs and, for videos, the public view count.", + }, + { + name: "ownerUsername / ownerId / ownerFullName", + type: "string", + description: "The account that posted the item.", + }, + { + name: "timestamp", + type: "string", + description: "ISO timestamp for when the post was published.", + }, + ], + }, + + faq: [ + { + question: "Is scraping Instagram legal?", + answer: + "SurfSense reads only public Instagram data, the same posts, reels, and comments any logged-out visitor can see. It never logs in and cannot access private accounts or stories. As always, review Instagram's terms and your own compliance needs before you run at scale.", + }, + { + question: "Do I need an Instagram account or the Graph API?", + answer: + "No. This is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured data back.", + }, + { + question: "What are the rate limits?", + answer: + "Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request, and up to 50 comments per post. SurfSense manages the underlying request budget and proxy rotation for you, so you scale reads without managing tokens.", + }, + { + question: "Can I get comments and replies?", + answer: + "Yes. The instagram.comments verb returns public comments on any post or reel, with author, like and reply counts, and optionally the nested replies. Get post URLs first from instagram.scrape if you only have a topic or a profile.", + }, + ], + + related: [ + { label: "Reddit API", href: "/reddit" }, + { label: "YouTube API", href: "/youtube" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "SERP API", href: "/google-search" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + { label: "Read the docs", href: "/docs" }, + ], +}; From 4e86710b26646bfa64bba06542c25964459a3d5b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:07:34 +0530 Subject: [PATCH 059/160] feat(web): cross-link Instagram from sibling connectors --- surfsense_web/lib/connectors-marketing/google-maps.tsx | 1 + surfsense_web/lib/connectors-marketing/google-search.tsx | 1 + surfsense_web/lib/connectors-marketing/reddit.tsx | 1 + surfsense_web/lib/connectors-marketing/web-crawl.tsx | 1 + surfsense_web/lib/connectors-marketing/youtube.tsx | 1 + 5 files changed, 5 insertions(+) diff --git a/surfsense_web/lib/connectors-marketing/google-maps.tsx b/surfsense_web/lib/connectors-marketing/google-maps.tsx index 97103ddac..03b1d247b 100644 --- a/surfsense_web/lib/connectors-marketing/google-maps.tsx +++ b/surfsense_web/lib/connectors-marketing/google-maps.tsx @@ -303,6 +303,7 @@ export const googleMaps: ConnectorPageContent = { related: [ { label: "Reddit API", href: "/reddit" }, { label: "YouTube API", href: "/youtube" }, + { label: "Instagram API", href: "/instagram" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, diff --git a/surfsense_web/lib/connectors-marketing/google-search.tsx b/surfsense_web/lib/connectors-marketing/google-search.tsx index 8cab4cd82..7f436f9b0 100644 --- a/surfsense_web/lib/connectors-marketing/google-search.tsx +++ b/surfsense_web/lib/connectors-marketing/google-search.tsx @@ -260,6 +260,7 @@ export const googleSearch: ConnectorPageContent = { { label: "Web Crawl API", href: "/web-crawl" }, { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/reddit.tsx b/surfsense_web/lib/connectors-marketing/reddit.tsx index db85a969c..93a8618ef 100644 --- a/surfsense_web/lib/connectors-marketing/reddit.tsx +++ b/surfsense_web/lib/connectors-marketing/reddit.tsx @@ -310,6 +310,7 @@ export const reddit: ConnectorPageContent = { related: [ { label: "YouTube API", href: "/youtube" }, + { label: "Instagram API", href: "/instagram" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, diff --git a/surfsense_web/lib/connectors-marketing/web-crawl.tsx b/surfsense_web/lib/connectors-marketing/web-crawl.tsx index fe25a85fe..1cb4d5dac 100644 --- a/surfsense_web/lib/connectors-marketing/web-crawl.tsx +++ b/surfsense_web/lib/connectors-marketing/web-crawl.tsx @@ -282,6 +282,7 @@ export const webCrawl: ConnectorPageContent = { { label: "SERP API", href: "/google-search" }, { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/youtube.tsx b/surfsense_web/lib/connectors-marketing/youtube.tsx index 1850cbe86..f5f3215ef 100644 --- a/surfsense_web/lib/connectors-marketing/youtube.tsx +++ b/surfsense_web/lib/connectors-marketing/youtube.tsx @@ -270,6 +270,7 @@ export const youtube: ConnectorPageContent = { related: [ { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, From f5aad487ac549e047246cd8fa72396af4f3fb3b4 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:07:42 +0530 Subject: [PATCH 060/160] docs(web): document Instagram native connector --- .../content/docs/connectors/index.mdx | 2 +- .../content/docs/connectors/native/index.mdx | 7 +- .../docs/connectors/native/instagram.mdx | 77 +++++++++++++++++++ .../content/docs/connectors/native/meta.json | 2 +- 4 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/instagram.mdx diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 1ee9209e6..747d278e2 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> + Date: Thu, 9 Jul 2026 16:07:50 +0530 Subject: [PATCH 061/160] docs(web): add Instagram to MCP server tool docs --- surfsense_web/app/(home)/mcp-server/page.tsx | 14 ++++++++++---- surfsense_web/content/docs/how-to/mcp-server.mdx | 8 ++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 5e4961b1e..a5557e11e 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -93,6 +93,9 @@ const TOOL_GROUPS = [ "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_instagram_scrape", + "surfsense_instagram_comments", + "surfsense_instagram_details", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", @@ -127,7 +130,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -215,8 +218,8 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Google Maps, Google Search, and the open web, and search, - read, and write your knowledge base. One API key, typed tools, pay as you go. + scrape Reddit, YouTube, Instagram, Google Maps, Google Search, and the open web, and + search, read, and write your knowledge base. One API key, typed tools, pay as you go.

+ diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index f5f419219..632dc5a4d 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 21 native, typed tools: every scraper (Reddit, YouTube, Instagram, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -133,7 +133,7 @@ Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a pro } ``` -Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 18 tools should appear with a green dot. +Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 21 tools should appear with a green dot. @@ -257,14 +257,14 @@ For self-host (stdio), all settings are environment variables passed by the clie - **401 errors** — the API key is wrong or expired; create a new one. - **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. - **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. -- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 18 tools register without needing a backend. +- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 21 tools register without needing a backend. ## Tools reference | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_comments`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From d800ca33a5198715d53ce16c8040d35d392c1280 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 18:29:17 +0530 Subject: [PATCH 062/160] feat(instagram): handle soft login wall in fetch logic Added detection for Instagram's soft login wall, which returns a 200 status with login HTML. Implemented a new function to identify login redirects and adjusted the fetch logic to treat these cases similarly to 401/403 responses. --- .../proprietary/platforms/instagram/fetch.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index 82b0bcc14..e734a0dc4 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -101,6 +101,13 @@ _WARM_URL = "https://www.instagram.com/" _BASE = "https://www.instagram.com" _CSRF_COOKIE = "csrftoken" +# Soft login wall: instead of a 401/403, IG answers an api/v1/* request with a +# 302 to /accounts/login/ that the impersonated client follows to a 200 login +# page. The status is 200 but the body is login HTML, so this evades the +# status-code rotate check — detect it via the response's final URL and treat +# it exactly like a 403. +_LOGIN_PATH = "/accounts/login" + def now_iso() -> str: """UTC timestamp in the millisecond ISO shape used by scraper output.""" @@ -136,6 +143,17 @@ def _parse_json(page: Any) -> Any | None: return None +def _is_login_redirect(page: Any) -> bool: + """True if IG redirected this request to the anonymous login wall. + + A soft block: the final URL lands on ``/accounts/login/`` (served 200), so + the status check never fires. Best-effort — returns ``False`` when the + response exposes no URL. + """ + final = getattr(page, "url", None) + return isinstance(final, str) and _LOGIN_PATH in final + + def _build_url(path: str, params: dict[str, Any] | None) -> str: """Absolute URL for an instagram.com path (accepts already-absolute URLs).""" base = path if path.startswith("http") else f"{_BASE}/{path.strip('/')}/" @@ -320,8 +338,11 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N await holder.pace() page = await _get_page(session, url) status = page.status + # A 302 -> /accounts/login/ soft block presents as 200 login HTML; + # fold it into the same rotate path as a 401/403. + blocked = status in _ROTATE_STATUSES or _is_login_redirect(page) - if status == 200: + if status == 200 and not blocked: return _parse_json(page) if status == 404: return None @@ -333,13 +354,14 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await asyncio.sleep(delay + random.uniform(0, 1)) continue - if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: + if blocked and attempt < _MAX_ROTATIONS: attempt += 1 await holder.rotate() continue - if status in _ROTATE_STATUSES: + if blocked: raise InstagramAccessBlockedError( - f"Instagram refused {path} on {attempt} rotated IPs ({status})" + f"Instagram refused {path} on {attempt} rotated IPs " + f"(status={status}, login_wall={_is_login_redirect(page)})" ) logger.warning("[instagram] GET %s returned %s", path, status) return None From 6384409b1d97bc3013d5726dab7156093ea4082f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 18:29:29 +0530 Subject: [PATCH 063/160] test(instagram): enhance fetch resilience tests for login redirects Added tests to handle scenarios where a 200 status response is redirected to a login page, ensuring proper rotation of IPs and raising errors when persistent login redirects occur. Updated the _FakeSession class to support login wall detection. --- .../instagram/test_fetch_resilience.py | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index eefae2fd9..037268db2 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -24,9 +24,12 @@ _PAYLOAD = {"data": {"user": {"username": "natgeo"}}} class _FakePage: - def __init__(self, status: int, *, cookies: dict | None = None, payload=None): + def __init__( + self, status: int, *, cookies: dict | None = None, payload=None, url=None + ): self.status = status self.cookies = cookies or {} + self.url = url self._payload = payload if payload is not None else _PAYLOAD def json(self): @@ -40,10 +43,18 @@ class _FakePage: class _FakeSession: """One 'IP': the warm-up GET mints csrftoken per flag; endpoint GETs return ``status``.""" - def __init__(self, status: int = 200, *, csrftoken: bool = True, payload=None) -> None: + def __init__( + self, + status: int = 200, + *, + csrftoken: bool = True, + payload=None, + login_wall: bool = False, + ) -> None: self.status = status self.csrftoken = csrftoken self.payload = payload + self.login_wall = login_wall self.json_calls = 0 self.warm_calls = 0 @@ -53,7 +64,9 @@ class _FakeSession: ck = {"csrftoken": "x", "mid": "y"} if self.csrftoken else {} return _FakePage(200, cookies=ck) self.json_calls += 1 - return _FakePage(self.status, payload=self.payload) + # A soft login wall: 200, but the final URL is the login page. + final = "https://www.instagram.com/accounts/login/" if self.login_wall else url + return _FakePage(self.status, payload=self.payload, url=final) class _FakeHolder: @@ -150,6 +163,41 @@ async def test_rotates_on_401_login_wall(): assert holder.rotations == 1 +async def test_rotates_on_login_redirect_then_succeeds(): + # 200 status but redirected to /accounts/login/: a soft block that must + # rotate to a fresh IP, not be mistaken for an empty result. + holder = _FakeHolder( + [_FakeSession(200, login_wall=True), _FakeSession(200)] + ) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + + +async def test_persistent_login_redirect_raises_blocked(): + holder = _FakeHolder( + [ + _FakeSession(200, login_wall=True) + for _ in range(fetch._MAX_ROTATIONS + 1) + ] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) + except InstagramAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == fetch._MAX_ROTATIONS + + async def test_404_returns_none_without_rotating(): holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) token = _current_session.set(holder) From 5551ebbda74ad1d8ae7f712b03a8448745536524 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:08:00 +0530 Subject: [PATCH 064/160] chore(instagram): modify instagram svg --- surfsense_web/public/connectors/instagram.svg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_web/public/connectors/instagram.svg b/surfsense_web/public/connectors/instagram.svg index 81233a38e..edba1b251 100644 --- a/surfsense_web/public/connectors/instagram.svg +++ b/surfsense_web/public/connectors/instagram.svg @@ -1 +1,2 @@ - + + \ No newline at end of file From e8f8eeab270b65c5b51fadc548df6065f8a96942 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:08:18 +0530 Subject: [PATCH 065/160] fix(instagram): improve error handling for Instagram access blocks in scraper Updated the fan_out function to handle InstagramAccessBlockedError more gracefully. Instead of raising the error directly, it now puts the error into the results queue to prevent deadlocks. This change ensures that the consumer can handle access block scenarios without interrupting the processing of other jobs. --- .../platforms/instagram/scraper.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 46569f4dd..34f6658a3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -132,7 +132,13 @@ async def fan_out( job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue() for job in jobs: job_queue.put_nowait(job) - results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue() + # A batch of items on success, or a hard-block exception to re-raise on the + # consumer side. The consumer reads exactly one entry per job, so a worker + # MUST put something for every job it pulls — raising instead would strand + # the error on a dead task and deadlock the consumer on ``results.get()``. + results: asyncio.Queue[list[dict[str, Any]] | InstagramAccessBlockedError] = ( + asyncio.Queue() + ) async def worker() -> None: holder = None @@ -153,8 +159,11 @@ async def fan_out( items = [item async for item in job] else: items = [item async for item in job] - except InstagramAccessBlockedError: - raise # a hard login wall must abort the batch, not be swallowed + except InstagramAccessBlockedError as e: + # A hard login wall aborts the batch. Hand it to the consumer + # via the queue (not ``raise``) so it never deadlocks waiting. + await results.put(e) + return except Exception as e: # one bad target must not kill the run logger.warning("[instagram] fan-out job failed: %s", e) await results.put(items) @@ -165,7 +174,10 @@ async def fan_out( tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] try: for _ in range(len(jobs)): - for item in await results.get(): + batch = await results.get() + if isinstance(batch, InstagramAccessBlockedError): + raise batch + for item in batch: yield item finally: for task in tasks: From 4af0b9dbbd99b6982ed364ed2eeef6d28e2df7ba Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:09:06 +0530 Subject: [PATCH 066/160] test(instagram): add regression test for InstagramAccessBlockedError handling Introduced a new test to ensure that InstagramAccessBlockedError is properly propagated without causing deadlocks in the scraper's fan_out function. This regression test verifies that the error surfaces correctly when a blocked job is encountered, enhancing the resilience of the fetch process. --- .../instagram/test_fetch_resilience.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index 037268db2..0ded704f2 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -295,6 +295,34 @@ async def test_fan_out_empty_jobs_is_noop(): assert out == [] +async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): + # Regression: a worker that raises InstagramAccessBlockedError used to strand + # the exception on its task and deadlock the consumer on results.get(). It + # must surface as InstagramAccessBlockedError, not hang. + async def _fake_open(): + return _TrackingHolder() + + @asynccontextmanager + async def _fake_bind(_holder): + yield _holder + + monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) + + async def _blocked_job() -> AsyncIterator[dict]: + raise InstagramAccessBlockedError("login wall") + yield {} # unreachable; makes this an async generator + + raised = False + try: + async with asyncio.timeout(5): # fail fast if the deadlock regresses + async for _ in scraper.fan_out([_blocked_job()], concurrency=1): + pass + except InstagramAccessBlockedError: + raised = True + assert raised, "hard block must propagate, not deadlock" + + def _profile_payload(username: str, n: int) -> dict: # IDs namespaced per target so cross-target de-dup doesn't collapse them. return { From be26f00d20a3f18e62ca7f36ccaf8c14ca753631 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:56:29 +0530 Subject: [PATCH 067/160] refactor(instagram): enhance fetch logic for login wall detection Updated the fetch_json function to raise InstagramAccessBlockedError immediately upon detecting a login redirect (302 -> /accounts/login/). This change prevents unnecessary IP rotations when encountering endpoint-level access blocks, improving the efficiency of the scraper's handling of Instagram's login wall. --- .../proprietary/platforms/instagram/fetch.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index e734a0dc4..3e9955943 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -338,11 +338,19 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N await holder.pace() page = await _get_page(session, url) status = page.status - # A 302 -> /accounts/login/ soft block presents as 200 login HTML; - # fold it into the same rotate path as a 401/403. - blocked = status in _ROTATE_STATUSES or _is_login_redirect(page) - if status == 200 and not blocked: + # Endpoint-level login wall (302 -> /accounts/login/, served as 200 + # login HTML): fail fast, do NOT rotate. Unlike the per-IP 401/403 + # below — which recovers on a fresh exit IP, so it still rotates — + # every rotated IP hits this same wall (observed live), so rotating + # only burns the pool and re-warms for an unwinnable block. Raising + # (vs returning None) keeps a blocked target distinguishable from an + # empty one; fan_out swallows it per-target for partial results. + if _is_login_redirect(page): + raise InstagramAccessBlockedError( + f"Instagram login wall on {path} (endpoint requires auth)" + ) + if status == 200: return _parse_json(page) if status == 404: return None @@ -354,14 +362,13 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await asyncio.sleep(delay + random.uniform(0, 1)) continue - if blocked and attempt < _MAX_ROTATIONS: + if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: attempt += 1 await holder.rotate() continue - if blocked: + if status in _ROTATE_STATUSES: raise InstagramAccessBlockedError( - f"Instagram refused {path} on {attempt} rotated IPs " - f"(status={status}, login_wall={_is_login_redirect(page)})" + f"Instagram refused {path} on {attempt} rotated IPs ({status})" ) logger.warning("[instagram] GET %s returned %s", path, status) return None From a0b388a5c5cdda81b594d0b2caadb0daa26398dc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:56:41 +0530 Subject: [PATCH 068/160] refactor(instagram): enhance batch processing for access block handling Updated the fan_out function to allow partial results when encountering blocked targets. Instead of aborting the entire batch on a hard login wall, the function now tracks blocked statuses and raises InstagramAccessBlockedError only if all targets are blocked. This change improves the scraper's resilience and efficiency in handling Instagram's access restrictions. --- .../platforms/instagram/scraper.py | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 34f6658a3..7d25e8c83 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -124,23 +124,24 @@ async def fan_out( Each worker opens ONE proxy session and reuses it across the sequential jobs it pulls, so only the first job per worker pays the proxy handshake + the - cookie warm-up. A bad job yields nothing rather than aborting the batch; - workers are cancelled and their sessions closed if the consumer stops early. + cookie warm-up. Partial results (matches the reddit sibling): one blocked or + failed target yields nothing rather than aborting the batch — Instagram is + an aggregation, not an atomic transaction, so 4/5 good targets beat 0/5. But + if EVERY target was refused (zero items AND a hard block seen), the whole run + couldn't reach anonymous data, so we surface ``InstagramAccessBlockedError`` + (-> 403) instead of a misleading empty success. Workers are cancelled and + their sessions closed if the consumer stops early. """ if not jobs: return job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue() for job in jobs: job_queue.put_nowait(job) - # A batch of items on success, or a hard-block exception to re-raise on the - # consumer side. The consumer reads exactly one entry per job, so a worker - # MUST put something for every job it pulls — raising instead would strand - # the error on a dead task and deadlock the consumer on ``results.get()``. - results: asyncio.Queue[list[dict[str, Any]] | InstagramAccessBlockedError] = ( - asyncio.Queue() - ) + results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue() + blocked = False # set if any target hit a hard login/auth wall async def worker() -> None: + nonlocal blocked holder = None try: holder = await open_proxy_holder() @@ -160,10 +161,10 @@ async def fan_out( else: items = [item async for item in job] except InstagramAccessBlockedError as e: - # A hard login wall aborts the batch. Hand it to the consumer - # via the queue (not ``raise``) so it never deadlocks waiting. - await results.put(e) - return + # Partial results: a blocked target must not kill the batch. + # Record it so a fully-blocked run can still surface the 403. + blocked = True + logger.warning("[instagram] target blocked: %s", e) except Exception as e: # one bad target must not kill the run logger.warning("[instagram] fan-out job failed: %s", e) await results.put(items) @@ -172,18 +173,24 @@ async def fan_out( await holder.close() tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] + emitted = 0 try: for _ in range(len(jobs)): - batch = await results.get() - if isinstance(batch, InstagramAccessBlockedError): - raise batch - for item in batch: + for item in await results.get(): + emitted += 1 yield item finally: for task in tasks: if not task.done(): task.cancel() await asyncio.gather(*tasks, return_exceptions=True) + # Reached only on natural exhaustion (an early-stop close raises GeneratorExit + # inside the loop and skips this). Nothing came back AND a wall was hit -> + # the run was fully refused, so fail loud rather than return empty. + if emitted == 0 and blocked: + raise InstagramAccessBlockedError( + "Instagram refused anonymous access to every target" + ) def _emit(partial: dict[str, Any], *, input_url: str | None) -> dict[str, Any]: From 0309e1d28ac8a71382113ddfe0a9c5c65122475a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:56:55 +0530 Subject: [PATCH 069/160] refactor(instagram): streamline login redirect handling in tests Revised tests for handling login redirects to improve clarity and efficiency. The test for immediate failure on endpoint-level walls was updated to prevent unnecessary IP rotations. Additionally, new tests were added to ensure that partial results are returned when encountering blocked targets, enhancing the resilience of the scraper's batch processing. --- .../instagram/test_fetch_resilience.py | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index 0ded704f2..29d92fad8 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -163,28 +163,11 @@ async def test_rotates_on_401_login_wall(): assert holder.rotations == 1 -async def test_rotates_on_login_redirect_then_succeeds(): - # 200 status but redirected to /accounts/login/: a soft block that must - # rotate to a fresh IP, not be mistaken for an empty result. - holder = _FakeHolder( - [_FakeSession(200, login_wall=True), _FakeSession(200)] - ) - token = _current_session.set(holder) - try: - result = await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) - finally: - _current_session.reset(token) - assert result == _PAYLOAD - assert holder.rotations == 1 - - -async def test_persistent_login_redirect_raises_blocked(): - holder = _FakeHolder( - [ - _FakeSession(200, login_wall=True) - for _ in range(fetch._MAX_ROTATIONS + 1) - ] - ) +async def test_login_redirect_fails_fast_without_rotating(): + # A 302 -> /accounts/login/ (served 200) is an endpoint-level wall: rotating + # never clears it, so we raise immediately instead of burning IP rotations. + # A second healthy session is present to prove we do NOT fall through to it. + holder = _FakeHolder([_FakeSession(200, login_wall=True), _FakeSession(200)]) token = _current_session.set(holder) try: raised = False @@ -195,7 +178,7 @@ async def test_persistent_login_redirect_raises_blocked(): finally: _current_session.reset(token) assert raised - assert holder.rotations == fetch._MAX_ROTATIONS + assert holder.rotations == 0 # fail fast: no rotation burned async def test_404_returns_none_without_rotating(): @@ -295,10 +278,7 @@ async def test_fan_out_empty_jobs_is_noop(): assert out == [] -async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): - # Regression: a worker that raises InstagramAccessBlockedError used to strand - # the exception on its task and deadlock the consumer on results.get(). It - # must surface as InstagramAccessBlockedError, not hang. +async def _install_fake_holders(monkeypatch) -> None: async def _fake_open(): return _TrackingHolder() @@ -309,9 +289,17 @@ async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) - async def _blocked_job() -> AsyncIterator[dict]: - raise InstagramAccessBlockedError("login wall") - yield {} # unreachable; makes this an async generator + +async def _blocked_job() -> AsyncIterator[dict]: + raise InstagramAccessBlockedError("login wall") + yield {} # unreachable; makes this an async generator + + +async def test_fan_out_all_blocked_raises_without_deadlock(monkeypatch): + # Regression: a blocked worker used to strand its exception on a dead task + # and deadlock the consumer on results.get(). When EVERY target is blocked + # (zero items), it must surface InstagramAccessBlockedError, not hang. + await _install_fake_holders(monkeypatch) raised = False try: @@ -320,7 +308,26 @@ async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): pass except InstagramAccessBlockedError: raised = True - assert raised, "hard block must propagate, not deadlock" + assert raised, "fully-blocked run must surface the 403, not deadlock" + + +async def test_fan_out_partial_results_survive_one_blocked_target(monkeypatch): + # Q2: one blocked target must NOT abort the batch — the good target's items + # come through and no exception is raised (aggregation, not a transaction). + await _install_fake_holders(monkeypatch) + + async def _good_job() -> AsyncIterator[dict]: + for i in range(3): + yield {"id": f"good:{i}"} + + async with asyncio.timeout(5): + items = [ + item + async for item in scraper.fan_out( + [_blocked_job(), _good_job()], concurrency=2 + ) + ] + assert [it["id"] for it in items] == ["good:0", "good:1", "good:2"] def _profile_payload(username: str, n: int) -> dict: From f66ab730445629a6f5cf3ce558db1448e7691fd6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:22:46 +0530 Subject: [PATCH 070/160] refactor(instagram): improve handling of login-gated endpoints in fetch logic Enhanced the fetch_json function to immediately raise InstagramAccessBlockedError for login-gated endpoints, preventing unnecessary IP rotations. Introduced a new constant for authentication-walled paths to streamline the detection of access blocks, improving the scraper's efficiency in handling Instagram's restrictions. --- .../proprietary/platforms/instagram/fetch.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index 3e9955943..73320eddb 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -75,6 +75,12 @@ _MAX_ROTATIONS = 3 _MAX_BACKOFFS = 4 _BACKOFF_BASE_S = 5.0 +# Endpoints Instagram serves only to logged-in clients (confirmed live). A bare +# 401/403 here is an endpoint auth wall, not a per-IP block, so every rotated IP +# hits the same wall — fail fast instead of burning the pool, exactly like the +# /accounts/login/ redirect branch. Content endpoints (profiles) still rotate. +_AUTH_WALLED_PATHS = ("web/search/topsearch/", "api/v1/tags/web_info/") + # Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't # burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits; # a pool with a stricter per-IP cap may need it raised — watch for 429 log spam. @@ -362,11 +368,18 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await asyncio.sleep(delay + random.uniform(0, 1)) continue - if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: - attempt += 1 - await holder.rotate() - continue if status in _ROTATE_STATUSES: + # Bare 401/403 on a login-gated endpoint: rotating never clears an + # endpoint auth wall, so fail fast (mirrors the login-redirect + # branch above). Other endpoints rotate — a per-IP 401 recovers. + if any(p in path for p in _AUTH_WALLED_PATHS): + raise InstagramAccessBlockedError( + f"Instagram login wall on {path} (endpoint requires auth)" + ) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue raise InstagramAccessBlockedError( f"Instagram refused {path} on {attempt} rotated IPs ({status})" ) From 82f1d0b4e5b7ac3f403ecdba22e9d8764f2eddd1 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:23:09 +0530 Subject: [PATCH 071/160] refactor(instagram): improve anonymous scraping logic and error messaging Enhanced the Instagram scraper to clarify the requirements for accessing user profiles and hashtags. Updated the error message for blocked access to provide detailed guidance on necessary credentials. Introduced a regex for validating Instagram usernames and refined the discovery function to handle profile queries directly, improving user experience and error handling in anonymous mode. --- .../capabilities/instagram/scrape/executor.py | 5 +- .../platforms/instagram/scraper.py | 64 +++++++------------ 2 files changed, 27 insertions(+), 42 deletions(-) diff --git a/surfsense_backend/app/capabilities/instagram/scrape/executor.py b/surfsense_backend/app/capabilities/instagram/scrape/executor.py index a76722dcd..c65e2ccb7 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/executor.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/executor.py @@ -43,7 +43,10 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: except InstagramAccessBlockedError as exc: # Anonymous-only scraper; a hard block can't be retried with creds. raise ForbiddenError( - f"Instagram refused anonymous access: {exc}", + "Instagram requires a login for this request and SurfSense scrapes " + "anonymously. Provide a profile URL or handle via directUrls; " + "keyword/hashtag search needs an account and is unavailable. " + f"Details: {exc}", code="INSTAGRAM_ACCESS_BLOCKED", ) from exc emit_progress( diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 7d25e8c83..f748d5026 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -26,6 +26,7 @@ from __future__ import annotations import asyncio import logging +import re from collections.abc import AsyncIterator from contextlib import aclosing from datetime import UTC, datetime, timedelta @@ -70,6 +71,10 @@ _HASHTAG_PATH = "api/v1/tags/web_info/" _LOCATION_PATH = "api/v1/locations/web_info/" _SEARCH_PATH = "web/search/topsearch/" +# Instagram usernames: 1-30 chars of letters/digits/period/underscore. Used to +# treat a profile/user discovery query as a direct (anonymous) handle lookup. +_HANDLE_RE = re.compile(r"[A-Za-z0-9._]{1,30}\Z") + def _parse_newer_than(value: str | None) -> datetime | None: """Parse ``onlyPostsNewerThan`` (ISO, YYYY-MM-DD, or relative) to UTC. @@ -356,48 +361,25 @@ async def _details_flow( async def _discover( query: str, *, search_type: str, limit: int ) -> list[ResolvedUrl]: - """Resolve a discovery query into target URLs via topsearch.""" - data = await fetch_json(_SEARCH_PATH, {"query": query, "context": "blended"}) - if not isinstance(data, dict): - return [] - out: list[ResolvedUrl] = [] + """Resolve a discovery query into targets - anonymously. + + Instagram's keyword search (``topsearch``) is login-walled, so we never call + it. A profile/user query is resolved as a direct handle lookup against the + anonymous profile endpoint ("messi" -> instagram.com/messi/). Hashtag/place + keyword discovery has NO anonymous endpoint (topsearch and the tag/location + feeds all require a session), so we surface a clear block instead of a + misleading empty success. + """ if search_type in ("profile", "user"): - for entry in data.get("users", []): - user = entry.get("user", {}) if isinstance(entry, dict) else {} - name = user.get("username") - if not name: - continue - out.append( - ResolvedUrl("profile", name, f"https://www.instagram.com/{name}/") - ) - elif search_type == "hashtag": - for entry in data.get("hashtags", []): - tag = entry.get("hashtag", {}) if isinstance(entry, dict) else {} - name = tag.get("name") - if not name: - continue - out.append( - ResolvedUrl( - "hashtag", - name, - f"https://www.instagram.com/explore/tags/{name}/", - ) - ) - elif search_type == "place": - for entry in data.get("places", []): - place = entry.get("place", {}) if isinstance(entry, dict) else {} - loc = place.get("location", {}) if isinstance(place, dict) else {} - pk = loc.get("pk") or loc.get("id") - if not pk: - continue - out.append( - ResolvedUrl( - "place", - str(pk), - f"https://www.instagram.com/explore/locations/{pk}/", - ) - ) - return out[:limit] + handle = query.strip().lstrip("@") + if _HANDLE_RE.match(handle): + url = f"https://www.instagram.com/{handle}/" + return [ResolvedUrl("profile", handle, url)][:limit] + return [] # not a handle, and no anonymous fuzzy search -> nothing to do + raise InstagramAccessBlockedError( + f"Instagram {search_type} search requires login and is unavailable in " + "anonymous mode - pass a profile URL or handle via directUrls instead" + ) def _resolve_inputs(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: From 2c251f9e0780464b783b083a574faa235f82fdfc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:26:18 +0530 Subject: [PATCH 072/160] test(instagram): add tests for anonymous profile and hashtag discovery Introduced new tests to validate the behavior of the Instagram scraper when discovering anonymous profiles and handling hashtag searches. The tests ensure that profile lookups succeed without requiring authentication, while hashtag searches correctly raise an error when access is blocked, enhancing the robustness of the scraper's functionality. --- .../instagram/test_fetch_resilience.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index 29d92fad8..e142a33f7 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -389,3 +389,22 @@ async def test_scrape_instagram_closes_sessions_when_limit_stops_inflight_worker assert len(items) == 3 assert holders, "workers should have opened sessions" assert all(h.closed for h in holders), "early stop leaked a proxy session" + + +async def test_discover_profile_is_anonymous_handle_lookup(): + # keyword search (topsearch) is login-walled, so a profile/user query resolves + # as a DIRECT handle lookup against the anonymous profile endpoint — no network + # here, just the URL resolution, so no session/monkeypatch needed. + targets = await scraper._discover("messi", search_type="user", limit=10) + assert [(t.kind, t.value) for t in targets] == [("profile", "messi")] + + +async def test_discover_hashtag_search_blocks_anonymously(): + # hashtag/place keyword discovery has no anonymous endpoint at all, so it must + # fail loud (clear message) rather than return a misleading empty success. + raised = False + try: + await scraper._discover("travel", search_type="hashtag", limit=10) + except InstagramAccessBlockedError: + raised = True + assert raised From 6652efd0358e131bb8ca1c3696ef687718a8ff3e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 17:48:19 +0200 Subject: [PATCH 073/160] feat(tiktok): return profile metadata even when video listing is gated A profile's account data (name, followers, bio, verification) lives in the page's rehydration blob and loads over plain HTTP without a signed request, so emit it first and always. The video listing needs a signed item_list XHR that TikTok withholds from anonymous sessions, so it stays best-effort and degrades to an ErrorItem. A blocked profile now yields its metadata instead of only an ErrorItem. --- .../platforms/tiktok/extraction/__init__.py | 3 +- .../platforms/tiktok/extraction/author.py | 7 ++ .../platforms/tiktok/flows/profile.py | 37 +++++++++ .../platforms/tiktok/orchestrator.py | 3 + .../platforms/tiktok/schemas/__init__.py | 2 + .../platforms/tiktok/schemas/items.py | 15 ++++ .../platforms/tiktok/test_orchestrator.py | 75 ++++++++++++++++++- 7 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index d70a494a5..86aefc2d7 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from .author import parse_author +from .author import parse_author, parse_profile from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct @@ -12,6 +12,7 @@ __all__ = [ "extract_rehydration_data", "items_from_response", "parse_author", + "parse_profile", "parse_video", "user_info", "video_item_struct", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py index 414de39b6..7dc1783e4 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/author.py @@ -29,3 +29,10 @@ def build_author_meta(author: dict[str, Any], stats: dict[str, Any]) -> dict[str def parse_author(user_info: dict[str, Any]) -> dict[str, Any]: """Map a ``webapp.user-detail`` ``userInfo`` (``{user, stats}``) to authorMeta.""" return build_author_meta(user_info.get("user") or {}, user_info.get("stats") or {}) + + +def parse_profile(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a ``userInfo`` to a standalone :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + return TikTokProfileItem(**parse_author(user_info)).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py new file mode 100644 index 000000000..feebd3c0d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/profile.py @@ -0,0 +1,37 @@ +"""Profile flow: reliable blob metadata first, then the (gated) video listing. + +A profile's account data (name, followers, bio, verification) lives in the page's +rehydration blob and loads over plain HTTP without a signed request, so we emit it +first and always. The video listing needs a signed ``item_list`` XHR that TikTok +withholds from anonymous sessions, so it is best-effort: it streams videos when it +loads and degrades to an ErrorItem (via :func:`iter_listing`) when withheld. The +metadata item therefore survives even when the videos are blocked. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import extract_rehydration_data, parse_profile, user_info +from ..extraction.timestamps import now_iso +from ..targets.types import TikTokTarget +from . import FetchFn, FetchListingFn +from .listing import iter_listing + + +async def iter_profile( + target: TikTokTarget, + *, + cap: int, + fetch: FetchFn, + fetch_listing: FetchListingFn, +) -> AsyncIterator[dict[str, Any]]: + html = await fetch(target.url) + info = user_info(extract_rehydration_data(html) or {}) if html else None + if info: + item = parse_profile(info) + item["scrapedAt"] = now_iso() + yield item + async for out in iter_listing(target, cap=cap, fetch_listing=fetch_listing): + yield out diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 95bc10fb9..3903a1088 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -14,6 +14,7 @@ from urllib.parse import quote from .flows import FetchFn, FetchListingFn from .flows.listing import iter_listing +from .flows.profile import iter_profile from .flows.video import iter_video from .schemas import TikTokScrapeInput from .session import fetch_html, fetch_item_list @@ -57,6 +58,8 @@ def _dispatch( ) -> AsyncIterator[dict[str, Any]]: if target.kind == "video": return iter_video(target, fetch=fetch) + if target.kind == "profile": + return iter_profile(target, cap=cap, fetch=fetch, fetch_listing=fetch_listing) return iter_listing(target, cap=cap, fetch_listing=fetch_listing) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py index 3593f0e28..a52400a58 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/__init__.py @@ -8,6 +8,7 @@ from .items import ( CommentItem, ErrorItem, MusicMeta, + TikTokProfileItem, TikTokVideoItem, VideoMeta, ) @@ -18,6 +19,7 @@ __all__ = [ "ErrorItem", "MusicMeta", "StartUrl", + "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "VideoMeta", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py index c4ff87618..50a5c4293 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/schemas/items.py @@ -30,6 +30,21 @@ class AuthorMeta(BaseModel): video: int | None = None +class TikTokProfileItem(AuthorMeta): + """A profile's public metadata, read from the page's rehydration blob. + + Emitted even when the video listing is withheld from anonymous access, so a + blocked profile still yields its account data (name, followers, bio, + verification) instead of only an ErrorItem. Distinguishable from a video item + by the absence of ``webVideoUrl``/``text``. + """ + + scrapedAt: str | None = None + + def to_output(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) + + class MusicMeta(BaseModel): model_config = ConfigDict(extra="allow") diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py index 777ed49de..3a4e42776 100644 --- a/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_orchestrator.py @@ -10,6 +10,33 @@ import json from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok +async def _no_html(_url: str) -> str: + """Fetch stub that yields no rehydration blob (skips profile metadata).""" + return "" + + +def _profile_page(username: str, followers: int, videos: int) -> str: + blob = { + "__DEFAULT_SCOPE__": { + "webapp.user-detail": { + "userInfo": { + "user": { + "id": "u1", + "uniqueId": username, + "nickname": "Nick", + "verified": True, + }, + "stats": {"followerCount": followers, "videoCount": videos}, + } + } + } + } + return ( + '' + ) + + def _video_page(video_id: str, username: str) -> str: blob = { "__DEFAULT_SCOPE__": { @@ -81,12 +108,57 @@ async def test_scrape_profile_returns_listing_items(): ] items = await scrape_tiktok( - TikTokScrapeInput(profiles=["a"], resultsPerPage=5), fetch_listing=fake_listing + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=_no_html, + fetch_listing=fake_listing, ) assert [i["id"] for i in items] == ["1", "2"] assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" +async def test_profile_emits_metadata_then_videos(): + # The blob metadata item comes first and is billable; videos follow. + async def fake_fetch(_url: str) -> str: + return _profile_page("a", followers=100, videos=2) + + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [{"id": "1", "author": {"uniqueId": "a"}}] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=fake_fetch, + fetch_listing=fake_listing, + ) + assert len(items) == 2 + profile, video = items + assert "webVideoUrl" not in profile # metadata item, not a video + assert profile["name"] == "a" + assert profile["fans"] == 100 + assert profile["verified"] is True + assert profile["scrapedAt"] is not None + assert video["id"] == "1" + + +async def test_profile_metadata_survives_blocked_listing(): + # Videos withheld from anonymous access: we still return the profile metadata + # (not just an ErrorItem), so a blocked profile isn't a total loss. + async def fake_fetch(_url: str) -> str: + return _profile_page("a", followers=100, videos=9) + + async def fake_listing(_url: str, _count: int) -> list[dict]: + return [] + + items = await scrape_tiktok( + TikTokScrapeInput(profiles=["a"], resultsPerPage=5), + fetch=fake_fetch, + fetch_listing=fake_listing, + ) + assert len(items) == 2 + assert items[0]["name"] == "a" + assert items[0]["fans"] == 100 + assert items[1]["errorCode"] == "no_items" + + async def test_listing_dedupes_then_caps_per_target(): async def fake_listing(_url: str, _count: int) -> list[dict]: return [{"id": "1"}, {"id": "1"}, {"id": "2"}, {"id": "3"}] @@ -105,6 +177,7 @@ async def test_empty_listing_emits_error_item(): items = await scrape_tiktok( TikTokScrapeInput(profiles=["nasa"], resultsPerPage=5), + fetch=_no_html, fetch_listing=fake_listing, ) assert len(items) == 1 From 192b6dc31a39ead462f0bf4cd1ebac1da01f18e6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 18:00:40 +0200 Subject: [PATCH 074/160] feat(tiktok): add tiktok.user_search verb for account discovery Video/general search is login-walled for anonymous sessions, but the Users tab (/api/search/user) returns public account records without a redirect, so this exposes the one reliably-unblocked search path. A keyword yields TikTokProfileItems (name, followers, bio, verification), deduped per query, capped, and degraded to an ErrorItem when a query is empty/withheld. Reuses the browser capture (generalized over XHR markers + extractor) and the shared profile item shape. Billed per account on a new TIKTOK_USER meter (TIKTOK_MICROS_PER_USER), surfaced on the chat subagent alongside tiktok.scrape. --- docker/.env.example | 2 + surfsense_backend/.env.example | 2 + .../subagents/builtins/tiktok/tools/index.py | 5 +- .../app/capabilities/core/billing.py | 2 + .../app/capabilities/core/types.py | 1 + .../app/capabilities/tiktok/__init__.py | 1 + .../tiktok/user_search/__init__.py | 3 + .../tiktok/user_search/definition.py | 26 +++++++ .../tiktok/user_search/executor.py | 39 +++++++++++ .../tiktok/user_search/schemas.py | 56 +++++++++++++++ surfsense_backend/app/config/__init__.py | 3 + .../proprietary/platforms/tiktok/__init__.py | 6 +- .../platforms/tiktok/extraction/__init__.py | 3 + .../tiktok/extraction/user_search.py | 56 +++++++++++++++ .../platforms/tiktok/flows/__init__.py | 3 + .../platforms/tiktok/flows/user_search.py | 54 ++++++++++++++ .../platforms/tiktok/orchestrator.py | 27 ++++++- .../platforms/tiktok/session/__init__.py | 3 +- .../platforms/tiktok/session/listing.py | 47 ++++++++++--- .../unit/capabilities/tiktok/test_registry.py | 13 ++++ .../tiktok/user_search/__init__.py | 0 .../tiktok/user_search/test_executor.py | 49 +++++++++++++ .../tiktok/user_search/test_schemas.py | 49 +++++++++++++ .../unit/platforms/tiktok/test_user_search.py | 70 +++++++++++++++++++ 24 files changed, 502 insertions(+), 18 deletions(-) create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/user_search/schemas.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py diff --git a/docker/.env.example b/docker/.env.example index 5daa0f3e8..86536e94c 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -449,6 +449,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 7e87c186d..eeb346a1b 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -286,6 +286,8 @@ MICROS_PER_PAGE=1000 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# TIKTOK_MICROS_PER_VIDEO=3500 +# TIKTOK_MICROS_PER_USER=2500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index e790d44f0..d826a8a85 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape capability verb.""" +"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs.""" from __future__ import annotations @@ -9,12 +9,13 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE +from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH] def load_tools( diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 048797473..8bc9fbd14 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -36,6 +36,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", + BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", } @@ -53,6 +54,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.TIKTOK_VIDEO: "video", + BillingUnit.TIKTOK_USER: "profile", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index b45641a2d..cbb01c2a6 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -26,6 +26,7 @@ class BillingUnit(StrEnum): YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" TIKTOK_VIDEO = "tiktok_video" + TIKTOK_USER = "tiktok_user" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index ed3f32682..067c91c0a 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -3,3 +3,4 @@ from __future__ import annotations from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 +from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..d4344968c --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.user_search``: find public TikTok accounts by keyword.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/definition.py b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py new file mode 100644 index 000000000..6a37f879f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/definition.py @@ -0,0 +1,26 @@ +"""``tiktok.user_search`` capability registration (billed per account; see config +``TIKTOK_MICROS_PER_USER``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +TIKTOK_USER_SEARCH = Capability( + name="tiktok.user_search", + description=( + "Find public TikTok accounts by keyword. Returns profile metadata " + "(name, followers, bio, verification) per matching account." + ), + input_schema=UserSearchInput, + output_schema=UserSearchOutput, + executor=build_user_search_executor(), + billing_unit=BillingUnit.TIKTOK_USER, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_USER_SEARCH) diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/executor.py b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py new file mode 100644 index 000000000..e83f396bd --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/executor.py @@ -0,0 +1,39 @@ +"""``tiktok.user_search`` executor: queries -> scraper -> TikTok profile items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) +from app.proprietary.platforms.tiktok import search_tiktok_users + +SearchFn = Callable[..., Awaitable[list[dict]]] + + +def build_user_search_executor(search_fn: SearchFn | None = None) -> Executor: + """Bind the executor to a search fn (defaults to the proprietary actor).""" + search_fn = search_fn or search_tiktok_users + + async def execute(payload: UserSearchInput) -> UserSearchOutput: + emit_progress( + "starting", + "Searching TikTok accounts", + total=payload.max_items, + unit="item", + ) + items = await search_fn( + payload.queries, + per_query=payload.results_per_query, + limit=payload.max_items, + ) + emit_progress( + "done", f"Found {len(items)} account(s)", current=len(items), unit="item" + ) + return UserSearchOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py new file mode 100644 index 000000000..7cb2aecbb --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/user_search/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.user_search`` I/O contracts. + +Account discovery over ``TikTok``'s Users tab. Where video/general search is +login-walled for anonymous sessions, ``/api/search/user`` returns public account +records, so this verb exposes the one reliably-unblocked search path. Each result +is a :class:`TikTokProfileItem` (the same shape the profile verb emits). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import TikTokProfileItem + + +class UserSearchInput(BaseModel): + queries: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="Keywords to search for TikTok accounts (e.g. names, brands).", + ) + results_per_query: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max accounts to return per query.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total accounts to return across all queries.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable accounts for the pre-flight gate: ``max_items`` is a + hard cross-query ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class UserSearchOutput(BaseModel): + items: list[TikTokProfileItem] = Field( + default_factory=list, + description="One item per account found, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned account = one billable unit; ErrorItems (``errorCode`` set, + for empty/withheld queries) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index c03ac3598..97abb5eda 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -717,6 +717,9 @@ class Config: # Browser-driven listings make TikTok heavier per item than the API-backed # video meter, so it sits a touch above YouTube's video rate. TIKTOK_MICROS_PER_VIDEO = int(os.getenv("TIKTOK_MICROS_PER_VIDEO", "3500")) + # User search returns lighter account records (name/followers/bio), priced + # below the video meter to mirror the cheaper account-discovery market. + TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 27867df73..2947498ec 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -6,14 +6,16 @@ schema, the collector/generator, the video item shape, and the hard-block error. from __future__ import annotations -from .orchestrator import iter_tiktok, scrape_tiktok -from .schemas import TikTokScrapeInput, TikTokVideoItem +from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users +from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem from .session import TikTokAccessBlockedError __all__ = [ "TikTokAccessBlockedError", + "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "iter_tiktok", "scrape_tiktok", + "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index 86aefc2d7..f4712e724 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -6,6 +6,7 @@ from .author import parse_author, parse_profile from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct +from .user_search import parse_search_user, users_from_response from .video import parse_video __all__ = [ @@ -13,7 +14,9 @@ __all__ = [ "items_from_response", "parse_author", "parse_profile", + "parse_search_user", "parse_video", "user_info", + "users_from_response", "video_item_struct", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py new file mode 100644 index 000000000..5ffc2cd4d --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/user_search.py @@ -0,0 +1,56 @@ +"""Parse the ``/api/search/user`` response into profile items. + +User search returns ``{"user_list": [{"user_info": {...}}, ...]}`` where each +``user_info`` uses the mobile-API snake_case shape (``uid``, ``unique_id``, +``follower_count``, ``total_favorited``, ``avatar_thumb.url_list``) — distinct +from the camelCase ``webapp.user-detail`` blob the profile flow reads, so it gets +its own mapping into the shared :class:`TikTokProfileItem` output contract. +""" + +from __future__ import annotations + +from typing import Any + +_PROFILE_URL = "https://www.tiktok.com/@{username}" + + +def users_from_response(body: Any) -> list[dict[str, Any]]: + """Return the ``user_info`` objects carried by one search response, or ``[]``.""" + if not isinstance(body, dict): + return [] + user_list = body.get("user_list") + if not isinstance(user_list, list): + return [] + return [ + entry["user_info"] + for entry in user_list + if isinstance(entry, dict) and isinstance(entry.get("user_info"), dict) + ] + + +def _avatar(user_info: dict[str, Any]) -> str | None: + thumb = user_info.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_search_user(user_info: dict[str, Any]) -> dict[str, Any]: + """Map a search ``user_info`` to a :class:`TikTokProfileItem` output dict.""" + from ..schemas.items import TikTokProfileItem + + username = user_info.get("unique_id") + return TikTokProfileItem( + id=user_info.get("uid"), + name=username, + nickName=user_info.get("nickname"), + profileUrl=_PROFILE_URL.format(username=username) if username else None, + verified=bool(user_info.get("enterprise_verify_reason")), + signature=user_info.get("signature"), + avatar=_avatar(user_info), + fans=user_info.get("follower_count"), + heart=user_info.get("total_favorited"), + secUid=user_info.get("sec_uid"), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index a7809567d..1a8aeaacc 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -10,4 +10,7 @@ FetchFn = Callable[[str], Awaitable[str | None]] FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] """Load a listing page and return up to ``count`` captured itemStructs.""" +FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a user-search page and return up to ``count`` captured ``user_info`` records.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py new file mode 100644 index 000000000..872212e80 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/user_search.py @@ -0,0 +1,54 @@ +"""User-search flow: keyword -> public account records. + +Unlike video/general search (login-walled for anonymous sessions), the Users tab +hits ``/api/search/user`` and returns account records without a redirect. Each +query's results are deduped by uid, capped, and — when a query returns nothing — +degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import quote + +from ..extraction import parse_search_user +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from . import FetchUsersFn + +_USER_SEARCH_URL = "https://www.tiktok.com/search/user?q={query}" +_EMPTY_MESSAGE = ( + "No accounts returned for this query. It may have no matches, or TikTok " + "withheld the results from anonymous access." +) + + +async def iter_user_search( + query: str, *, cap: int, fetch_users: FetchUsersFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + url = _USER_SEARCH_URL.format(query=quote(query)) + seen: set[str] = set() + emitted = 0 + for user_info in await fetch_users(url, cap): + out = parse_search_user(user_info) + uid = out.get("id") + if uid is not None: + if uid in seen: + continue + seen.add(uid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=url, + input=query, + error=_EMPTY_MESSAGE, + errorCode="no_users", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 3903a1088..485a8dfff 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,12 +12,13 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn, FetchListingFn +from .flows import FetchFn, FetchListingFn, FetchUsersFn from .flows.listing import iter_listing from .flows.profile import iter_profile +from .flows.user_search import iter_user_search from .flows.video import iter_video from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list +from .session import fetch_html, fetch_item_list, fetch_user_search from .targets import resolve_target from .targets.types import TikTokTarget @@ -100,3 +101,25 @@ async def scrape_tiktok( if limit is not None and len(results) >= limit: break return results + + +async def search_tiktok_users( + queries: list[str], + *, + per_query: int, + limit: int | None = None, + fetch_users: FetchUsersFn = fetch_user_search, +) -> list[dict[str, Any]]: + """Collect user-search account records across queries, honoring ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for query in queries: + async for item in iter_user_search( + query, cap=per_query, fetch_users=fetch_users + ): + results.append(item) + emit_progress("searching", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 409aa5987..5e29403a0 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,7 +4,7 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_item_list +from .listing import fetch_item_list, fetch_user_search from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ @@ -12,6 +12,7 @@ __all__ = [ "bind_proxy_holder", "fetch_html", "fetch_item_list", + "fetch_user_search", "open_proxy_holder", "proxy_session", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index cc84a48b5..e0bdd9255 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio import logging +from collections.abc import Callable from typing import Any from scrapling.fetchers import StealthyFetcher @@ -29,16 +30,20 @@ from app.proprietary.web_crawler.stealth import ( ) from app.utils.proxy import get_proxy_url -from ..extraction import items_from_response +from ..extraction import items_from_response, users_from_response logger = logging.getLogger(__name__) +ExtractFn = Callable[[Any], list[dict[str, Any]]] + # XHR paths that carry itemStructs for the three listing kinds. _ITEM_LIST_MARKERS = ( "/api/post/item_list", "/api/challenge/item_list", "/api/search/", ) +# The user-search XHR carries account records (user_list), not itemStructs. +_USER_SEARCH_MARKERS = ("/api/search/user",) _HOME_URL = "https://www.tiktok.com/" _MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early @@ -57,24 +62,31 @@ def _has_mstoken(page: Any) -> bool: return False -def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: int): - """A sync ``page_action`` that warms the session then captures item_list XHRs. +def _build_page_action( + collected: list[dict[str, Any]], + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, +): + """A sync ``page_action`` that warms the session then captures matching XHRs. - A cold context returns an empty ``item_list`` body, so we first mint the - anonymous ``msToken`` (homepage hit), then navigate to the target with the - listener already attached so page-one fires into it; scrolling pages the rest. + A cold context returns an empty body, so we first mint the anonymous + ``msToken`` (homepage hit), then navigate to the target with the listener + already attached so page-one fires into it; scrolling pages the rest. + ``markers``/``extract`` select which XHRs to keep and how to unwrap them. """ def _on_response(response: Any) -> None: response_url = getattr(response, "url", "") - if not any(marker in response_url for marker in _ITEM_LIST_MARKERS): + if not any(marker in response_url for marker in markers): return try: body = response.json() except Exception: # An empty 200 (TikTok soft-block) or a body evicted before read. return - collected.extend(items_from_response(body)) + collected.extend(extract(body)) def _warm(page: Any) -> None: if _has_mstoken(page): @@ -110,7 +122,9 @@ def _build_page_action(collected: list[dict[str, Any]], url: str, target_count: return page_action -def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: +def _fetch_sync( + url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn +) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) StealthyFetcher.fetch( @@ -118,7 +132,9 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: headless=True, network_idle=False, proxy=get_proxy_url(), - page_action=_build_page_action(collected, url, target_count), + page_action=_build_page_action( + collected, url, target_count, markers, extract + ), **kwargs, ) return collected[:target_count] @@ -126,4 +142,13 @@ def _fetch_sync(url: str, target_count: int) -> list[dict[str, Any]]: async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" - return await asyncio.to_thread(_fetch_sync, page_url, target_count) + return await asyncio.to_thread( + _fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response + ) + + +async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` ``user_info`` records from a user-search page.""" + return await asyncio.to_thread( + _fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response + ) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index 3a85fe87f..893f3c2ae 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -10,6 +10,10 @@ from app.capabilities import ( from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) pytestmark = pytest.mark.unit @@ -21,3 +25,12 @@ def test_tiktok_scrape_is_registered_and_billed_per_video(): assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO + + +def test_tiktok_user_search_is_registered_and_billed_per_profile(): + cap = get_capability("tiktok.user_search") + + assert cap.name == "tiktok.user_search" + assert cap.input_schema is UserSearchInput + assert cap.output_schema is UserSearchOutput + assert cap.billing_unit is BillingUnit.TIKTOK_USER diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py new file mode 100644 index 000000000..866968808 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_executor.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` executor: verb input → search args → typed profile items. + +Boundary mocked: the proprietary search actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→TikTokProfileItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.user_search.executor import build_user_search_executor +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +class _FakeSearch: + """Records the queries + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, queries: list[str], *, per_query: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((queries, per_query, limit)) + return self._items + + +async def test_forwards_queries_and_limits_and_wraps_items(): + search = _FakeSearch([{"id": "1", "name": "nasa"}]) + execute = build_user_search_executor(search_fn=search) + + out = await execute( + UserSearchInput(queries=["nasa"], results_per_query=7, max_items=25) + ) + + assert isinstance(out, UserSearchOutput) + assert len(out.items) == 1 + assert out.items[0].name == "nasa" + + (queries, per_query, limit) = search.calls[0] + assert queries == ["nasa"] + assert per_query == 7 + assert limit == 25 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py new file mode 100644 index 000000000..575db3fb5 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/user_search/test_schemas.py @@ -0,0 +1,49 @@ +"""``tiktok.user_search`` input guards and billing: a query is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES +from app.capabilities.tiktok.user_search.schemas import ( + UserSearchInput, + UserSearchOutput, +) + +pytestmark = pytest.mark.unit + + +def test_rejects_input_with_no_query(): + with pytest.raises(ValidationError): + UserSearchInput(queries=[]) + + +def test_defaults_and_bounds(): + payload = UserSearchInput(queries=["nasa"]) + assert payload.max_items == 10 + assert payload.results_per_query == 10 + assert payload.estimated_units == 10 + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=0) + with pytest.raises(ValidationError): + UserSearchInput(queries=["nasa"], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_queries_than_the_cap(): + too_many = [f"q{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + UserSearchInput(queries=too_many) + + +def test_error_items_are_not_billed(): + # Real accounts count; ErrorItems (empty/withheld queries) are surfaced free. + out = UserSearchOutput( + items=[ + {"id": "1", "name": "nasa"}, + {"errorCode": "no_users", "input": "ghost", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py new file mode 100644 index 000000000..845b5e22d --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_user_search.py @@ -0,0 +1,70 @@ +"""User-search orchestration over a fake fetch (no network). + +Drives ``search_tiktok_users``: queries -> captured ``user_info`` -> profile items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import search_tiktok_users + + +def _user(uid: str, unique_id: str, followers: int = 10) -> dict[str, Any]: + return { + "uid": uid, + "unique_id": unique_id, + "nickname": unique_id.upper(), + "signature": "bio", + "follower_count": followers, + "total_favorited": 999, + "sec_uid": f"sec-{uid}", + "enterprise_verify_reason": "official" if uid == "1" else "", + "avatar_thumb": {"url_list": [f"https://cdn/{uid}.webp"]}, + } + + +async def test_user_search_parses_dedupes_and_caps(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "nasa"), _user("1", "nasa"), _user("2", "nasa2")] + + items = await search_tiktok_users( + ["nasa"], per_query=2, fetch_users=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["name"] == "nasa" + assert first["nickName"] == "NASA" + assert first["profileUrl"] == "https://www.tiktok.com/@nasa" + assert first["verified"] is True + assert first["fans"] == 10 + assert first["avatar"] == "https://cdn/1.webp" + assert first["secUid"] == "sec-1" + assert first["scrapedAt"] is not None + assert items[1]["verified"] is False + + +async def test_user_search_empty_query_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await search_tiktok_users( + ["ghost"], per_query=5, fetch_users=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_users" + assert items[0]["input"] == "ghost" + + +async def test_user_search_honors_limit_across_queries(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_user("1", "a"), _user("2", "b")] + + items = await search_tiktok_users( + ["q1", "q2"], per_query=5, limit=3, fetch_users=fake_fetch + ) + + # 2 from q1 + 1 from q2, then the cross-query limit stops it. + assert len(items) == 3 From 7723b5b8b6c0a4fc516735b6f4a59ecf67faf9fd Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 18:35:26 +0200 Subject: [PATCH 075/160] feat(tiktok): add tiktok.comments verb for public comment scraping Comments load over a signed /api/comment/list XHR that TikTok serves to anonymous sessions once the comments panel is opened (unlike profile-video and general-search feeds), so this is a reliable verb. Given video URLs it returns CommentItems (text, author, likes, reply counts; replies carry repliesToId), deduped per video, capped, and degraded to an ErrorItem for empty/withheld videos or a bad_url ErrorItem for non-video inputs. Generalizes the browser capture over a pluggable interaction step so the comments flow (open panel, scroll the panel to paginate) reuses the same warm+capture scaffolding as listing/user-search. Billed per comment on a new TIKTOK_COMMENT meter (TIKTOK_MICROS_PER_COMMENT, matching the per-comment market), surfaced on the chat subagent alongside tiktok.scrape/user_search. --- docker/.env.example | 1 + surfsense_backend/.env.example | 1 + .../subagents/builtins/tiktok/tools/index.py | 5 +- .../app/capabilities/core/billing.py | 2 + .../app/capabilities/core/types.py | 1 + .../app/capabilities/tiktok/__init__.py | 1 + .../capabilities/tiktok/comments/__init__.py | 3 + .../tiktok/comments/definition.py | 23 +++ .../capabilities/tiktok/comments/executor.py | 36 +++++ .../capabilities/tiktok/comments/schemas.py | 56 +++++++ surfsense_backend/app/config/__init__.py | 3 + .../proprietary/platforms/tiktok/__init__.py | 16 +- .../platforms/tiktok/extraction/__init__.py | 3 + .../platforms/tiktok/extraction/comments.py | 55 +++++++ .../platforms/tiktok/flows/__init__.py | 3 + .../platforms/tiktok/flows/comments.py | 52 +++++++ .../platforms/tiktok/orchestrator.py | 54 ++++++- .../platforms/tiktok/session/__init__.py | 3 +- .../platforms/tiktok/session/listing.py | 139 +++++++++++++++--- .../capabilities/tiktok/comments/__init__.py | 0 .../tiktok/comments/test_executor.py | 48 ++++++ .../tiktok/comments/test_schemas.py | 48 ++++++ .../unit/capabilities/tiktok/test_registry.py | 10 ++ .../unit/platforms/tiktok/test_comments.py | 89 +++++++++++ 24 files changed, 626 insertions(+), 26 deletions(-) create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/comments/schemas.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py create mode 100644 surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_comments.py diff --git a/docker/.env.example b/docker/.env.example index 86536e94c..19e1a47fd 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -451,6 +451,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # YOUTUBE_MICROS_PER_COMMENT=1500 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index eeb346a1b..20d811839 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -288,6 +288,7 @@ MICROS_PER_PAGE=1000 # YOUTUBE_MICROS_PER_COMMENT=1500 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 +# TIKTOK_MICROS_PER_COMMENT=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index d826a8a85..44ebc863d 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape and user-search capability verbs.""" +"""``tiktok`` sub-agent tools: the TikTok scrape, comments, and user-search verbs.""" from __future__ import annotations @@ -8,6 +8,7 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH @@ -15,7 +16,7 @@ NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_USER_SEARCH] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH] def load_tools( diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 8bc9fbd14..de2e48af3 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -37,6 +37,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", + BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", } @@ -55,6 +56,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.YOUTUBE_COMMENT: "comment", BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", + BillingUnit.TIKTOK_COMMENT: "comment", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index cbb01c2a6..d41c11ffb 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -27,6 +27,7 @@ class BillingUnit(StrEnum): YOUTUBE_COMMENT = "youtube_comment" TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" + TIKTOK_COMMENT = "tiktok_comment" class BillableInput(Protocol): diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index 067c91c0a..1ff1738fa 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -2,5 +2,6 @@ from __future__ import annotations +from app.capabilities.tiktok.comments import definition as _comments # noqa: F401 from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/comments/__init__.py b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..f3bd1db88 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.comments``: scrape the public comment thread of TikTok videos.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/comments/definition.py b/surfsense_backend/app/capabilities/tiktok/comments/definition.py new file mode 100644 index 000000000..1e5917fa2 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.comments`` capability registration (billed per comment; see config +``TIKTOK_MICROS_PER_COMMENT``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +TIKTOK_COMMENTS = Capability( + name="tiktok.comments", + description=( + "Scrape the public comments of TikTok videos. Provide video URLs; " + "returns comment text, author, likes, and reply counts." + ), + input_schema=CommentsInput, + output_schema=CommentsOutput, + executor=build_comments_executor(), + billing_unit=BillingUnit.TIKTOK_COMMENT, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_COMMENTS) diff --git a/surfsense_backend/app/capabilities/tiktok/comments/executor.py b/surfsense_backend/app/capabilities/tiktok/comments/executor.py new file mode 100644 index 000000000..c8bbc776f --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/executor.py @@ -0,0 +1,36 @@ +"""``tiktok.comments`` executor: video URLs -> scraper -> TikTok comment items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +CommentsFn = Callable[..., Awaitable[list[dict]]] + + +def build_comments_executor(comments_fn: CommentsFn | None = None) -> Executor: + """Bind the executor to a comments fn (defaults to the proprietary actor).""" + comments_fn = comments_fn or scrape_tiktok_comments + + async def execute(payload: CommentsInput) -> CommentsOutput: + emit_progress( + "starting", + "Scraping TikTok comments", + total=payload.max_items, + unit="item", + ) + items = await comments_fn( + payload.video_urls, + per_video=payload.comments_per_video, + limit=payload.max_items, + ) + emit_progress( + "done", f"Scraped {len(items)} comment(s)", current=len(items), unit="item" + ) + return CommentsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py new file mode 100644 index 000000000..f3c68d445 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py @@ -0,0 +1,56 @@ +"""``tiktok.comments`` I/O contracts. + +URL-only: given TikTok video URLs, return each video's public comment thread. +Unlike profile-video/general-search feeds, ``/api/comment/list`` is served to +anonymous sessions once the comments panel opens, so this verb is reliable. Each +result is a :class:`CommentItem` (top-level comments; replies carry ``repliesToId``). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import ( + MAX_TIKTOK_ITEMS, + MAX_TIKTOK_SOURCES, +) +from app.proprietary.platforms.tiktok import CommentItem + + +class CommentsInput(BaseModel): + video_urls: list[str] = Field( + min_length=1, + max_length=MAX_TIKTOK_SOURCES, + description="TikTok video URLs (/@/video/) to pull comments from.", + ) + comments_per_video: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max comments to return per video.", + ) + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max total comments to return across all videos.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable comments for the pre-flight gate: ``max_items`` is a + hard cross-video ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class CommentsOutput(BaseModel): + items: list[CommentItem] = Field( + default_factory=list, + description="One item per comment returned, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned comment = one billable unit; ErrorItems (``errorCode`` set, + for bad URLs or empty/withheld videos) are surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 97abb5eda..21ba1d1c6 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -720,6 +720,9 @@ class Config: # User search returns lighter account records (name/followers/bio), priced # below the video meter to mirror the cheaper account-discovery market. TIKTOK_MICROS_PER_USER = int(os.getenv("TIKTOK_MICROS_PER_USER", "2500")) + # Comments are the cheapest per-item TikTok data, matching the per-comment + # market (and YouTube's comment meter). + TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 2947498ec..00d345937 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -6,16 +6,28 @@ schema, the collector/generator, the video item shape, and the hard-block error. from __future__ import annotations -from .orchestrator import iter_tiktok, scrape_tiktok, search_tiktok_users -from .schemas import TikTokProfileItem, TikTokScrapeInput, TikTokVideoItem +from .orchestrator import ( + iter_tiktok, + scrape_tiktok, + scrape_tiktok_comments, + search_tiktok_users, +) +from .schemas import ( + CommentItem, + TikTokProfileItem, + TikTokScrapeInput, + TikTokVideoItem, +) from .session import TikTokAccessBlockedError __all__ = [ + "CommentItem", "TikTokAccessBlockedError", "TikTokProfileItem", "TikTokScrapeInput", "TikTokVideoItem", "iter_tiktok", "scrape_tiktok", + "scrape_tiktok_comments", "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py index f4712e724..3afd536f2 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations from .author import parse_author, parse_profile +from .comments import comments_from_response, parse_comment from .hydration import extract_rehydration_data from .item_list import items_from_response from .scopes import user_info, video_item_struct @@ -10,9 +11,11 @@ from .user_search import parse_search_user, users_from_response from .video import parse_video __all__ = [ + "comments_from_response", "extract_rehydration_data", "items_from_response", "parse_author", + "parse_comment", "parse_profile", "parse_search_user", "parse_video", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py new file mode 100644 index 000000000..8ca96cad3 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/extraction/comments.py @@ -0,0 +1,55 @@ +"""Parse a captured ``/api/comment/list`` response into comment items. + +The comment API returns ``{"comments": [...]}`` where each entry uses the +mobile-API snake_case shape (``cid``, ``digg_count``, ``reply_comment_total``, +``create_time``, and a nested ``user`` with ``uid``/``unique_id``/``nickname``/ +``avatar_thumb``). ``reply_id != "0"`` marks a reply to a parent comment. +""" + +from __future__ import annotations + +from typing import Any + +from .timestamps import epoch_to_iso + + +def comments_from_response(body: Any) -> list[dict[str, Any]]: + """Return the raw comment records carried by one API response, or ``[]``.""" + if not isinstance(body, dict): + return [] + comments = body.get("comments") + if not isinstance(comments, list): + return [] + return [c for c in comments if isinstance(c, dict)] + + +def _avatar(user: dict[str, Any]) -> str | None: + thumb = user.get("avatar_thumb") + if isinstance(thumb, dict): + urls = thumb.get("url_list") + if isinstance(urls, list) and urls: + return urls[0] + return None + + +def parse_comment(raw: dict[str, Any], video_url: str) -> dict[str, Any]: + """Map a raw comment record to a :class:`CommentItem` output dict.""" + from ..schemas.items import CommentItem + + user = raw.get("user") if isinstance(raw.get("user"), dict) else {} + reply_id = raw.get("reply_id") + create_time = raw.get("create_time") + return CommentItem( + id=raw.get("cid"), + text=raw.get("text"), + videoWebUrl=video_url, + diggCount=raw.get("digg_count"), + replyCommentTotal=raw.get("reply_comment_total"), + createTime=create_time, + createTimeISO=epoch_to_iso(create_time), + uid=user.get("uid"), + uniqueId=user.get("unique_id"), + nickName=user.get("nickname"), + avatar=_avatar(user), + repliesToId=reply_id if reply_id and reply_id != "0" else None, + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py index 1a8aeaacc..a7e38eba1 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/__init__.py @@ -13,4 +13,7 @@ FetchListingFn = Callable[[str, int], Awaitable[list[dict]]] FetchUsersFn = Callable[[str, int], Awaitable[list[dict]]] """Load a user-search page and return up to ``count`` captured ``user_info`` records.""" +FetchCommentsFn = Callable[[str, int], Awaitable[list[dict]]] +"""Load a video page and return up to ``count`` captured raw comment records.""" + FlowResult = AsyncIterator[dict] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py new file mode 100644 index 000000000..28a7772a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/tiktok/flows/comments.py @@ -0,0 +1,52 @@ +"""Comments flow: a video URL -> its public comment thread. + +Comments load over a signed ``/api/comment/list`` XHR that TikTok *does* serve to +anonymous sessions once the comments panel is opened (unlike profile-video/search +feeds). Records are deduped by comment id, capped, and — when a video has none or +withholds them — degraded to one ErrorItem, mirroring the listing flow. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from ..extraction import parse_comment +from ..extraction.timestamps import now_iso +from ..schemas import ErrorItem +from ..targets.types import TikTokTarget +from . import FetchCommentsFn + +_EMPTY_MESSAGE = ( + "No comments returned. The video may have none, comments disabled, or TikTok " + "withheld them from anonymous access." +) + + +async def iter_comments( + target: TikTokTarget, *, cap: int, fetch_comments: FetchCommentsFn +) -> AsyncIterator[dict[str, Any]]: + if cap <= 0: + return + seen: set[str] = set() + emitted = 0 + for raw in await fetch_comments(target.url, cap): + out = parse_comment(raw, target.url) + cid = out.get("id") + if cid is not None: + if cid in seen: + continue + seen.add(cid) + out["scrapedAt"] = now_iso() + yield out + emitted += 1 + if emitted >= cap: + return + if emitted == 0: + yield ErrorItem( + url=target.url, + input=target.value, + error=_EMPTY_MESSAGE, + errorCode="no_comments", + scrapedAt=now_iso(), + ).to_output() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 485a8dfff..dbe0d5ce9 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,13 +12,20 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote -from .flows import FetchFn, FetchListingFn, FetchUsersFn +from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn +from .flows.comments import iter_comments from .flows.listing import iter_listing from .flows.profile import iter_profile from .flows.user_search import iter_user_search from .flows.video import iter_video -from .schemas import TikTokScrapeInput -from .session import fetch_html, fetch_item_list, fetch_user_search +from .extraction.timestamps import now_iso +from .schemas import ErrorItem, TikTokScrapeInput +from .session import ( + fetch_comments, + fetch_html, + fetch_item_list, + fetch_user_search, +) from .targets import resolve_target from .targets.types import TikTokTarget @@ -123,3 +130,44 @@ async def search_tiktok_users( if limit is not None and len(results) >= limit: return results return results + + +async def scrape_tiktok_comments( + video_urls: list[str], + *, + per_video: int, + limit: int | None = None, + fetch_comments_fn: FetchCommentsFn = fetch_comments, +) -> list[dict[str, Any]]: + """Collect comments across video URLs, honoring ``limit``. + + A non-video URL yields one ``bad_url`` ErrorItem (never a silent drop) so the + caller can tell "wrong input" from "video has no comments". + """ + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + for url in video_urls: + target = resolve_target(url) + if target is None or target.kind != "video": + results.append( + ErrorItem( + url=url, + input=url, + error="Not a TikTok video URL.", + errorCode="bad_url", + scrapedAt=now_iso(), + ).to_output() + ) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + continue + async for item in iter_comments( + target, cap=per_video, fetch_comments=fetch_comments_fn + ): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + return results + return results diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 5e29403a0..437fc9643 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,12 +4,13 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_item_list, fetch_user_search +from .listing import fetch_comments, fetch_item_list, fetch_user_search from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ "TikTokAccessBlockedError", "bind_proxy_holder", + "fetch_comments", "fetch_html", "fetch_item_list", "fetch_user_search", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index e0bdd9255..d00e20711 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -30,11 +30,18 @@ from app.proprietary.web_crawler.stealth import ( ) from app.utils.proxy import get_proxy_url -from ..extraction import items_from_response, users_from_response +from ..extraction import ( + comments_from_response, + items_from_response, + users_from_response, +) logger = logging.getLogger(__name__) ExtractFn = Callable[[Any], list[dict[str, Any]]] +# Drives the page after navigation to trigger/paginate the target XHRs, filling +# ``collected`` until it reaches ``target_count`` (or the interaction gives up). +InteractFn = Callable[[Any, list[dict[str, Any]], int], None] # XHR paths that carry itemStructs for the three listing kinds. _ITEM_LIST_MARKERS = ( @@ -44,6 +51,16 @@ _ITEM_LIST_MARKERS = ( ) # The user-search XHR carries account records (user_list), not itemStructs. _USER_SEARCH_MARKERS = ("/api/search/user",) +# The comment feed fires only after the comments panel is opened. +_COMMENT_MARKERS = ("/api/comment/list",) +_COMMENT_ICON_SELECTORS = ( + '[data-e2e="comment-icon"]', + '[data-e2e="browse-comment"]', +) +# The comment icon hydrates a beat after DOM-ready; wait for it before clicking. +_COMMENT_ICON_WAIT_MS = 8000 +# First comment page lands shortly after the click — don't declare "empty" early. +_COMMENT_FIRST_PAGE_MS = 3500 _HOME_URL = "https://www.tiktok.com/" _MSTOKEN_COOKIE = "msToken" # Bounded scroll: a dead page can't loop forever, and a live one stops early @@ -62,18 +79,87 @@ def _has_mstoken(page: Any) -> bool: return False +def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: + """Page down a listing feed until enough items are captured or it stops growing.""" + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + + +def _open_comments(page: Any) -> None: + """Click the comment icon so the first ``/api/comment/list`` XHR fires. + + The icon must be present and interactive first (the SPA hydrates it a beat + after DOM-ready), so we wait for it, then fall back to a JS click if the + normal click is intercepted (cookie banner / overlay). + """ + for selector in _COMMENT_ICON_SELECTORS: + try: + page.wait_for_selector(selector, timeout=_COMMENT_ICON_WAIT_MS) + except Exception: + continue + try: + page.click(selector, timeout=_COMMENT_ICON_WAIT_MS) + return + except Exception: + try: + page.eval_on_selector(selector, "el => el.click()") + return + except Exception: + continue + + +def _scroll_comments( + page: Any, collected: list[dict[str, Any]], target_count: int +) -> None: + """Open the comments panel, then scroll its last comment into view to paginate. + + Comment XHRs fire only after the panel is opened, and paging must scroll the + panel (not the page, which would advance the video feed), so we anchor on the + last ``comment-level-1`` element. ponytail: naive scroll-to-last paging, + bounded by ``_SCROLL_MAX_ROUNDS``; upgrade to container-height tracking if + deep threads under-fetch. + """ + _open_comments(page) + # The panel's first page lands a beat after the click; give it room before + # we decide there are no comments to page through. + page.wait_for_timeout(_COMMENT_FIRST_PAGE_MS) + for _ in range(_SCROLL_MAX_ROUNDS): + if len(collected) >= target_count: + break + moved = page.evaluate( + """() => { + const items = document.querySelectorAll('[data-e2e="comment-level-1"]'); + if (!items.length) return false; + items[items.length - 1].scrollIntoView({block: 'end'}); + return true; + }""" + ) + page.wait_for_timeout(_SCROLL_SETTLE_MS) + if not moved: + break + + def _build_page_action( collected: list[dict[str, Any]], url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn, + interact: InteractFn, ): """A sync ``page_action`` that warms the session then captures matching XHRs. A cold context returns an empty body, so we first mint the anonymous ``msToken`` (homepage hit), then navigate to the target with the listener - already attached so page-one fires into it; scrolling pages the rest. + already attached so page-one fires into it; ``interact`` pages the rest. ``markers``/``extract`` select which XHRs to keep and how to unwrap them. """ @@ -102,28 +188,23 @@ def _build_page_action( try: _warm(page) # Navigate (back) to the target with the listener attached and a - # token in hand, so the page-one item_list fires into the capture. + # token in hand, so the page-one XHR fires into the capture. page.goto(url, wait_until="domcontentloaded") page.wait_for_timeout(_SCROLL_SETTLE_MS) - last_height = 0 - for _ in range(_SCROLL_MAX_ROUNDS): - if len(collected) >= target_count: - break - page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - page.wait_for_timeout(_SCROLL_SETTLE_MS) - height = page.evaluate("document.body.scrollHeight") - if not height or height <= last_height: - break - last_height = height + interact(page, collected, target_count) except Exception as exc: - logger.debug("[tiktok] listing scroll aborted: %s", exc) + logger.debug("[tiktok] capture interaction aborted: %s", exc) return page return page_action def _fetch_sync( - url: str, target_count: int, markers: tuple[str, ...], extract: ExtractFn + url: str, + target_count: int, + markers: tuple[str, ...], + extract: ExtractFn, + interact: InteractFn, ) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) @@ -133,7 +214,7 @@ def _fetch_sync( network_idle=False, proxy=get_proxy_url(), page_action=_build_page_action( - collected, url, target_count, markers, extract + collected, url, target_count, markers, extract, interact ), **kwargs, ) @@ -143,12 +224,34 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" return await asyncio.to_thread( - _fetch_sync, page_url, target_count, _ITEM_LIST_MARKERS, items_from_response + _fetch_sync, + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, ) async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` ``user_info`` records from a user-search page.""" return await asyncio.to_thread( - _fetch_sync, page_url, target_count, _USER_SEARCH_MARKERS, users_from_response + _fetch_sync, + page_url, + target_count, + _USER_SEARCH_MARKERS, + users_from_response, + _scroll_page, + ) + + +async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` raw comment records from a video page's XHRs.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _COMMENT_MARKERS, + comments_from_response, + _scroll_comments, ) diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py new file mode 100644 index 000000000..b23149420 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_executor.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` executor: verb input → scraper args → typed comment items. + +Boundary mocked: the proprietary comments actor (injected fake). NOT mocked: the +verb's own payload→args forwarding and the dict→CommentItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.comments.executor import build_comments_executor +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +class _FakeComments: + """Records the urls + kwargs it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[list[str], int, int | None]] = [] + + async def __call__( + self, video_urls: list[str], *, per_video: int, limit: int | None = None + ) -> list[dict]: + self.calls.append((video_urls, per_video, limit)) + return self._items + + +async def test_forwards_urls_and_limits_and_wraps_items(): + comments = _FakeComments([{"id": "1", "text": "hi"}]) + execute = build_comments_executor(comments_fn=comments) + + out = await execute( + CommentsInput(video_urls=[_VIDEO], comments_per_video=15, max_items=40) + ) + + assert isinstance(out, CommentsOutput) + assert len(out.items) == 1 + assert out.items[0].text == "hi" + + (urls, per_video, limit) = comments.calls[0] + assert urls == [_VIDEO] + assert per_video == 15 + assert limit == 40 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py new file mode 100644 index 000000000..cdcbc201e --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/comments/test_schemas.py @@ -0,0 +1,48 @@ +"""``tiktok.comments`` input guards and billing: a video URL is required, bounded, +and ErrorItems are surfaced free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES + +pytestmark = pytest.mark.unit + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def test_rejects_input_with_no_url(): + with pytest.raises(ValidationError): + CommentsInput(video_urls=[]) + + +def test_defaults_and_bounds(): + payload = CommentsInput(video_urls=[_VIDEO]) + assert payload.max_items == 20 + assert payload.comments_per_video == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=0) + with pytest.raises(ValidationError): + CommentsInput(video_urls=[_VIDEO], max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_rejects_more_urls_than_the_cap(): + too_many = [f"{_VIDEO}{i}" for i in range(MAX_TIKTOK_SOURCES + 1)] + with pytest.raises(ValidationError): + CommentsInput(video_urls=too_many) + + +def test_error_items_are_not_billed(): + # Real comments count; ErrorItems (bad URL / empty video) are surfaced free. + out = CommentsOutput( + items=[ + {"id": "1", "text": "hi"}, + {"errorCode": "no_comments", "input": "123", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index 893f3c2ae..c688cf198 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -9,6 +9,7 @@ from app.capabilities import ( ) from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability +from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput from app.capabilities.tiktok.user_search.schemas import ( UserSearchInput, @@ -34,3 +35,12 @@ def test_tiktok_user_search_is_registered_and_billed_per_profile(): assert cap.input_schema is UserSearchInput assert cap.output_schema is UserSearchOutput assert cap.billing_unit is BillingUnit.TIKTOK_USER + + +def test_tiktok_comments_is_registered_and_billed_per_comment(): + cap = get_capability("tiktok.comments") + + assert cap.name == "tiktok.comments" + assert cap.input_schema is CommentsInput + assert cap.output_schema is CommentsOutput + assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py new file mode 100644 index 000000000..e118f3c01 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_comments.py @@ -0,0 +1,89 @@ +"""Comments orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_comments``: video URLs -> captured raw comments -> items. +""" + +from __future__ import annotations + +from typing import Any + +from app.proprietary.platforms.tiktok import scrape_tiktok_comments + +_VIDEO = "https://www.tiktok.com/@bob/video/123" + + +def _comment(cid: str, reply_id: str = "0") -> dict[str, Any]: + return { + "cid": cid, + "text": f"comment {cid}", + "digg_count": 7, + "reply_comment_total": 2, + "create_time": 1700000000, + "reply_id": reply_id, + "user": { + "uid": "u1", + "unique_id": "alice", + "nickname": "Alice", + "avatar_thumb": {"url_list": ["https://cdn/a.webp"]}, + }, + } + + +async def test_comments_parse_dedupe_and_cap(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("1"), _comment("2", reply_id="1")] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=2, fetch_comments_fn=fake_fetch + ) + + assert [i["id"] for i in items] == ["1", "2"] + first = items[0] + assert first["text"] == "comment 1" + assert first["videoWebUrl"] == _VIDEO + assert first["diggCount"] == 7 + assert first["uniqueId"] == "alice" + assert first["avatar"] == "https://cdn/a.webp" + assert first["createTimeISO"] is not None + assert first["repliesToId"] is None # reply_id "0" == top-level + assert first["scrapedAt"] is not None + assert items[1]["repliesToId"] == "1" # a reply carries its parent id + + +async def test_empty_video_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_comments( + [_VIDEO], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_comments" + assert items[0]["input"] == "123" + + +async def test_non_video_url_emits_bad_url_error(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + raise AssertionError("should not fetch for a non-video URL") + + items = await scrape_tiktok_comments( + ["https://www.tiktok.com/@bob"], per_video=5, fetch_comments_fn=fake_fetch + ) + + assert len(items) == 1 + assert items[0]["errorCode"] == "bad_url" + + +async def test_comments_honor_limit_across_videos(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [_comment("1"), _comment("2")] + + items = await scrape_tiktok_comments( + [_VIDEO, "https://www.tiktok.com/@bob/video/456"], + per_video=5, + limit=3, + fetch_comments_fn=fake_fetch, + ) + + assert len(items) == 3 From 67b5472b9fc2775727af94273aab88816e303c75 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 18:52:56 +0200 Subject: [PATCH 076/160] feat(tiktok): add tiktok.trending verb for the Explore feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Explore feed (/api/explore/item_list) is a global trending-video feed served to anonymous sessions, and it returns the same itemStruct shape as the other listings — so the verb reuses parse_video, the listing flow, the TikTokVideoItem output, and the per-video billing meter wholesale. Adds a browser-capture marker + fetch_trending, a synthetic-target orchestrator entry, and the tiktok.trending capability, surfaced on the chat subagent. --- .../subagents/builtins/tiktok/tools/index.py | 5 ++- .../app/capabilities/tiktok/__init__.py | 1 + .../capabilities/tiktok/trending/__init__.py | 3 ++ .../tiktok/trending/definition.py | 23 +++++++++++ .../capabilities/tiktok/trending/executor.py | 32 +++++++++++++++ .../capabilities/tiktok/trending/schemas.py | 41 +++++++++++++++++++ .../proprietary/platforms/tiktok/__init__.py | 2 + .../platforms/tiktok/orchestrator.py | 24 ++++++++++- .../platforms/tiktok/session/__init__.py | 8 +++- .../platforms/tiktok/session/listing.py | 14 +++++++ .../platforms/tiktok/targets/types.py | 2 +- .../unit/capabilities/tiktok/test_registry.py | 11 +++++ .../capabilities/tiktok/trending/__init__.py | 0 .../tiktok/trending/test_executor.py | 38 +++++++++++++++++ .../tiktok/trending/test_schemas.py | 33 +++++++++++++++ .../unit/platforms/tiktok/test_trending.py | 35 ++++++++++++++++ 16 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/__init__.py create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/definition.py create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/executor.py create mode 100644 surfsense_backend/app/capabilities/tiktok/trending/schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_trending.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py index 44ebc863d..f1bec7fc2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/tools/index.py @@ -1,4 +1,4 @@ -"""``tiktok`` sub-agent tools: the TikTok scrape, comments, and user-search verbs.""" +"""``tiktok`` sub-agent tools: scrape, comments, user-search, and trending verbs.""" from __future__ import annotations @@ -10,13 +10,14 @@ from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools from app.capabilities.tiktok.comments.definition import TIKTOK_COMMENTS from app.capabilities.tiktok.scrape.definition import TIKTOK_SCRAPE +from app.capabilities.tiktok.trending.definition import TIKTOK_TRENDING from app.capabilities.tiktok.user_search.definition import TIKTOK_USER_SEARCH NAME = "tiktok" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH] +_CI_VERBS = [TIKTOK_SCRAPE, TIKTOK_COMMENTS, TIKTOK_USER_SEARCH, TIKTOK_TRENDING] def load_tools( diff --git a/surfsense_backend/app/capabilities/tiktok/__init__.py b/surfsense_backend/app/capabilities/tiktok/__init__.py index 1ff1738fa..4962d3162 100644 --- a/surfsense_backend/app/capabilities/tiktok/__init__.py +++ b/surfsense_backend/app/capabilities/tiktok/__init__.py @@ -4,4 +4,5 @@ from __future__ import annotations from app.capabilities.tiktok.comments import definition as _comments # noqa: F401 from app.capabilities.tiktok.scrape import definition as _scrape # noqa: F401 +from app.capabilities.tiktok.trending import definition as _trending # noqa: F401 from app.capabilities.tiktok.user_search import definition as _user_search # noqa: F401 diff --git a/surfsense_backend/app/capabilities/tiktok/trending/__init__.py b/surfsense_backend/app/capabilities/tiktok/trending/__init__.py new file mode 100644 index 000000000..c335258b3 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/__init__.py @@ -0,0 +1,3 @@ +"""``tiktok.trending``: pull the current trending videos from the Explore feed.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/tiktok/trending/definition.py b/surfsense_backend/app/capabilities/tiktok/trending/definition.py new file mode 100644 index 000000000..7f1b006ce --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/definition.py @@ -0,0 +1,23 @@ +"""``tiktok.trending`` capability registration (billed per video on the shared +``TIKTOK_MICROS_PER_VIDEO`` meter).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.tiktok.trending.executor import build_trending_executor +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +TIKTOK_TRENDING = Capability( + name="tiktok.trending", + description=( + "Get the current trending TikTok videos from the Explore feed. No input " + "needed beyond how many to return." + ), + input_schema=TrendingInput, + output_schema=TrendingOutput, + executor=build_trending_executor(), + billing_unit=BillingUnit.TIKTOK_VIDEO, + docs_url="/docs/connectors/native/tiktok", +) + +register_capability(TIKTOK_TRENDING) diff --git a/surfsense_backend/app/capabilities/tiktok/trending/executor.py b/surfsense_backend/app/capabilities/tiktok/trending/executor.py new file mode 100644 index 000000000..9551f4b6a --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/executor.py @@ -0,0 +1,32 @@ +"""``tiktok.trending`` executor: Explore feed -> scraper -> TikTok video items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput +from app.proprietary.platforms.tiktok import scrape_tiktok_trending + +TrendingFn = Callable[..., Awaitable[list[dict]]] + + +def build_trending_executor(trending_fn: TrendingFn | None = None) -> Executor: + """Bind the executor to a trending fn (defaults to the proprietary actor).""" + trending_fn = trending_fn or scrape_tiktok_trending + + async def execute(payload: TrendingInput) -> TrendingOutput: + emit_progress( + "starting", + "Fetching TikTok trending videos", + total=payload.max_items, + unit="item", + ) + items = await trending_fn(count=payload.max_items) + emit_progress( + "done", f"Fetched {len(items)} video(s)", current=len(items), unit="item" + ) + return TrendingOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/tiktok/trending/schemas.py b/surfsense_backend/app/capabilities/tiktok/trending/schemas.py new file mode 100644 index 000000000..879206907 --- /dev/null +++ b/surfsense_backend/app/capabilities/tiktok/trending/schemas.py @@ -0,0 +1,41 @@ +"""``tiktok.trending`` I/O contracts. + +The Explore feed (``/api/explore/item_list``) is a single global feed of trending +videos, served to anonymous sessions. No source input is needed — just how many +to return. Each result reuses :class:`TikTokVideoItem`, so trending videos bill on +the same per-video meter as ``tiktok.scrape``. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS +from app.proprietary.platforms.tiktok import TikTokVideoItem + + +class TrendingInput(BaseModel): + max_items: int = Field( + default=20, + ge=1, + le=MAX_TIKTOK_ITEMS, + description="Max trending videos to return from the Explore feed.", + ) + + @property + def estimated_units(self) -> int: + """Worst-case billable videos for the pre-flight gate (le=100 ceiling).""" + return self.max_items + + +class TrendingOutput(BaseModel): + items: list[TikTokVideoItem] = Field( + default_factory=list, + description="One item per trending video returned, in feed order.", + ) + + @property + def billable_units(self) -> int: + """One returned video = one billable unit; an ErrorItem (``errorCode`` set, + for an empty/withheld feed) is surfaced but never charged.""" + return sum(1 for item in self.items if not getattr(item, "errorCode", None)) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py index 00d345937..91f716d1f 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/__init__.py @@ -10,6 +10,7 @@ from .orchestrator import ( iter_tiktok, scrape_tiktok, scrape_tiktok_comments, + scrape_tiktok_trending, search_tiktok_users, ) from .schemas import ( @@ -29,5 +30,6 @@ __all__ = [ "iter_tiktok", "scrape_tiktok", "scrape_tiktok_comments", + "scrape_tiktok_trending", "search_tiktok_users", ] diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index dbe0d5ce9..5d6a24545 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,18 +12,19 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote +from .extraction.timestamps import now_iso from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn from .flows.comments import iter_comments from .flows.listing import iter_listing from .flows.profile import iter_profile from .flows.user_search import iter_user_search from .flows.video import iter_video -from .extraction.timestamps import now_iso from .schemas import ErrorItem, TikTokScrapeInput from .session import ( fetch_comments, fetch_html, fetch_item_list, + fetch_trending, fetch_user_search, ) from .targets import resolve_target @@ -32,6 +33,7 @@ from .targets.types import TikTokTarget _PROFILE_URL = "https://www.tiktok.com/@{name}" _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" _SEARCH_URL = "https://www.tiktok.com/search?q={query}" +_EXPLORE_URL = "https://www.tiktok.com/explore" def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: @@ -132,6 +134,26 @@ async def search_tiktok_users( return results +async def scrape_tiktok_trending( + *, + count: int, + fetch_trending_fn: FetchListingFn = fetch_trending, +) -> list[dict[str, Any]]: + """Collect up to ``count`` trending videos from the Explore feed. + + A single global feed, so it reuses the listing flow (parse/dedupe/cap/empty- + ErrorItem) over a synthetic target — no user input to resolve. + """ + from app.capabilities.core.progress import emit_progress + + target = TikTokTarget(kind="trending", value="explore", url=_EXPLORE_URL) + results: list[dict[str, Any]] = [] + async for item in iter_listing(target, cap=count, fetch_listing=fetch_trending_fn): + results.append(item) + emit_progress("scraping", current=len(results), total=count, unit="item") + return results + + async def scrape_tiktok_comments( video_urls: list[str], *, diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py index 437fc9643..6fa1c1b45 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/__init__.py @@ -4,7 +4,12 @@ from __future__ import annotations from .client import fetch_html from .errors import TikTokAccessBlockedError -from .listing import fetch_comments, fetch_item_list, fetch_user_search +from .listing import ( + fetch_comments, + fetch_item_list, + fetch_trending, + fetch_user_search, +) from .proxy import bind_proxy_holder, open_proxy_holder, proxy_session __all__ = [ @@ -13,6 +18,7 @@ __all__ = [ "fetch_comments", "fetch_html", "fetch_item_list", + "fetch_trending", "fetch_user_search", "open_proxy_holder", "proxy_session", diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index d00e20711..e94984440 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -51,6 +51,8 @@ _ITEM_LIST_MARKERS = ( ) # The user-search XHR carries account records (user_list), not itemStructs. _USER_SEARCH_MARKERS = ("/api/search/user",) +# The Explore feed's trending videos arrive as ordinary itemStructs. +_EXPLORE_MARKERS = ("/api/explore/item_list",) # The comment feed fires only after the comments panel is opened. _COMMENT_MARKERS = ("/api/comment/list",) _COMMENT_ICON_SELECTORS = ( @@ -255,3 +257,15 @@ async def fetch_comments(page_url: str, target_count: int) -> list[dict[str, Any comments_from_response, _scroll_comments, ) + + +async def fetch_trending(page_url: str, target_count: int) -> list[dict[str, Any]]: + """Return up to ``target_count`` trending itemStructs from the Explore feed.""" + return await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _EXPLORE_MARKERS, + items_from_response, + _scroll_page, + ) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py index c53d5239d..417d27741 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/targets/types.py @@ -5,7 +5,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Literal -TargetKind = Literal["video", "profile", "hashtag", "search"] +TargetKind = Literal["video", "profile", "hashtag", "search", "trending"] SearchSection = Literal["video", "user"] diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py index c688cf198..c6314ec10 100644 --- a/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/tiktok/test_registry.py @@ -11,6 +11,7 @@ from app.capabilities.core import BillingUnit from app.capabilities.core.store import get_capability from app.capabilities.tiktok.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.tiktok.scrape.schemas import ScrapeInput, ScrapeOutput +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput from app.capabilities.tiktok.user_search.schemas import ( UserSearchInput, UserSearchOutput, @@ -44,3 +45,13 @@ def test_tiktok_comments_is_registered_and_billed_per_comment(): assert cap.input_schema is CommentsInput assert cap.output_schema is CommentsOutput assert cap.billing_unit is BillingUnit.TIKTOK_COMMENT + + +def test_tiktok_trending_is_registered_and_billed_per_video(): + cap = get_capability("tiktok.trending") + + assert cap.name == "tiktok.trending" + assert cap.input_schema is TrendingInput + assert cap.output_schema is TrendingOutput + # Trending returns videos, so it shares the per-video meter. + assert cap.billing_unit is BillingUnit.TIKTOK_VIDEO diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py new file mode 100644 index 000000000..3539ca595 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_executor.py @@ -0,0 +1,38 @@ +"""``tiktok.trending`` executor: verb input → scraper count → typed video items. + +Boundary mocked: the proprietary trending actor (injected fake). NOT mocked: the +verb's own count forwarding and the dict→TikTokVideoItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.tiktok.trending.executor import build_trending_executor +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +pytestmark = pytest.mark.unit + + +class _FakeTrending: + """Records the count it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[int] = [] + + async def __call__(self, *, count: int) -> list[dict]: + self.calls.append(count) + return self._items + + +async def test_forwards_count_and_wraps_items(): + trending = _FakeTrending([{"id": "1", "text": "viral"}]) + execute = build_trending_executor(trending_fn=trending) + + out = await execute(TrendingInput(max_items=30)) + + assert isinstance(out, TrendingOutput) + assert len(out.items) == 1 + assert out.items[0].id == "1" + assert trending.calls == [30] diff --git a/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py new file mode 100644 index 000000000..f27fec71a --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/tiktok/trending/test_schemas.py @@ -0,0 +1,33 @@ +"""``tiktok.trending`` input guards and billing: bounded count, ErrorItems free.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.tiktok.scrape.schemas import MAX_TIKTOK_ITEMS +from app.capabilities.tiktok.trending.schemas import TrendingInput, TrendingOutput + +pytestmark = pytest.mark.unit + + +def test_defaults_and_bounds(): + payload = TrendingInput() + assert payload.max_items == 20 + assert payload.estimated_units == 20 + with pytest.raises(ValidationError): + TrendingInput(max_items=0) + with pytest.raises(ValidationError): + TrendingInput(max_items=MAX_TIKTOK_ITEMS + 1) + + +def test_error_items_are_not_billed(): + # Real videos count; an ErrorItem (empty/withheld feed) is surfaced free. + out = TrendingOutput( + items=[ + {"id": "1", "webVideoUrl": "https://tiktok.com/@a/video/1"}, + {"errorCode": "no_items", "input": "explore", "error": "empty"}, + ] + ) + assert len(out.items) == 2 + assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py b/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py new file mode 100644 index 000000000..d00d9c09f --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_trending.py @@ -0,0 +1,35 @@ +"""Trending orchestration over a fake fetch (no network). + +Drives ``scrape_tiktok_trending``: the Explore feed -> captured itemStructs -> +video items, reusing the listing flow (parse/dedupe/cap/empty-ErrorItem). +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok import scrape_tiktok_trending + + +async def test_trending_parses_dedupes_and_caps(): + async def fake_fetch(url: str, _cap: int) -> list[dict]: + assert url == "https://www.tiktok.com/explore" + return [ + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "1", "author": {"uniqueId": "a"}}, + {"id": "2", "author": {"uniqueId": "b"}}, + ] + + items = await scrape_tiktok_trending(count=2, fetch_trending_fn=fake_fetch) + + assert [i["id"] for i in items] == ["1", "2"] + assert items[0]["webVideoUrl"] == "https://www.tiktok.com/@a/video/1" + assert items[0]["scrapedAt"] is not None + + +async def test_trending_empty_feed_emits_error_item(): + async def fake_fetch(_url: str, _cap: int) -> list[dict]: + return [] + + items = await scrape_tiktok_trending(count=5, fetch_trending_fn=fake_fetch) + + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" From 6b15d610d92fdcb51cfd5d89b54a512eab4daa3c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 19:09:00 +0200 Subject: [PATCH 077/160] feat(tiktok): surface comments, user_search, trending across all product surfaces Brings the three newer verbs to parity with tiktok.scrape everywhere it was wired: MCP tools (+ selfcheck manifest), the chat subagent prompt/description, the playground catalog, the native docs page, and the SEO marketing page. Also adds live e2e stages for comments, user search, and trending. Docs/FAQ now state the real contract: profile metadata is reliable while its video list can be withheld, and keyword video search is walled (use user search for accounts). --- .../subagents/builtins/tiktok/description.md | 4 +- .../builtins/tiktok/system_prompt.md | 18 ++- .../scripts/e2e_tiktok_scrape.py | 70 ++++++++++- .../features/scrapers/platforms/tiktok.py | 119 +++++++++++++++++- surfsense_mcp/mcp_server/selfcheck.py | 3 + .../content/docs/connectors/native/tiktok.mdx | 84 ++++++++++--- .../lib/connectors-marketing/tiktok.tsx | 18 ++- surfsense_web/lib/playground/catalog.ts | 11 +- 8 files changed, 294 insertions(+), 33 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md index fc13d864d..d57c551d4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -1,2 +1,2 @@ -TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL. Also compares fresh TikTok results against earlier findings in this chat. -Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, or scrape a specific video URL. Triggers include "search TikTok for X", "trending TikTok videos about X", "videos with #X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. +Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, read a video's comments, discover accounts by keyword, or scrape a specific video URL. Triggers include "search TikTok for X", "what's trending on TikTok", "videos with #X", "comments on this TikTok", "find TikTok accounts about X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md index 49c20164e..5b6de993c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -6,17 +6,23 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- `tiktok_scrape` +- `tiktok_scrape` — videos from urls / profiles / hashtags / search_queries +- `tiktok_comments` — a video's comment thread, from `video_urls` +- `tiktok_user_search` — find accounts by keyword, from `queries` +- `tiktok_trending` — the current Explore trending-video feed - `read_run` / `search_run` (free readers for stored scrape output) - Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. -- Profiles: a creator's `profiles` feed can come back empty — TikTok restricts the profile video endpoint. Prefer `hashtags`, `search_queries`, or a direct video URL, and treat an empty profile result as a known limit, not a failure to retry endlessly. -- Controlling volume: use `max_items` for the total cap and `results_per_page` per target. -- Requested counts: `max_items` defaults to only 10 — when the task asks for N videos, set `max_items` and `results_per_page` above N. A call that caps below the target can never satisfy it. -- Batch multiple hashtags or search terms into one call rather than many single-term calls. +- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags`, `search_queries`, or a direct video URL for videos. +- Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. +- Finding accounts (not videos): call `tiktok_user_search` with `queries` — this is the reliable path for account discovery (keyword *video* search is login-walled). +- "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. +- Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). +- Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. +- Batch multiple hashtags, search terms, queries, or video URLs into one call rather than many single-item calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). @@ -59,6 +65,6 @@ Return **only** one JSON object (no markdown/prose): } Route-specific rules: -- `evidence.findings`: one entry per distinct video or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.findings`: one entry per distinct result (video, comment, or account) or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). - `evidence.sources`: one TikTok URL per finding when applicable, same cap as findings. List each URL once. diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index eedeaacac..cb5a996eb 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -15,6 +15,12 @@ What it exercises (everything REAL — live network, live proxy, live browser): Stage 5 — full scrape_tiktok() pipeline on a hashtag. Stage 6 — search via the full pipeline: same graceful-degrade contract as profile (results feed doesn't load for anonymous sessions). + Stage 7 — comments on a real video URL (served anonymously once the panel + opens): real comments OR a single honest ErrorItem. + Stage 8 — user search: the account-discovery XHR that DOES serve anonymous + headless sessions — asserts real account records come back. + Stage 9 — trending: the Explore feed of trending videos — asserts real, + normalized video items come back. On success it writes raw itemStructs under tests/fixtures/tiktok/ so the parser suites can pin against real-shaped data without network. @@ -199,6 +205,61 @@ async def stage_search_listing() -> tuple[bool, list[dict[str, Any]]]: return ok, items +async def stage_comments(video_url: str) -> tuple[bool, list[dict[str, Any]]]: + _hr("STAGE 7 — comments graceful-degrade") + print(f" target: {video_url}") + from app.proprietary.platforms.tiktok import scrape_tiktok_comments + + # Comments load over a signed /api/comment/list XHR that TikTok serves to + # anonymous sessions once the panel opens. Pass if real comments come back + # OR a graceful ErrorItem (video has none / disabled / withheld). + items = await scrape_tiktok_comments( + [video_url], per_video=_COUNT, limit=_COUNT + ) + has_comment = any(it.get("id") and not it.get("errorCode") for it in items) + has_error = any(it.get("errorCode") == "no_comments" for it in items) + ok = _check( + "comments yield records or a graceful ErrorItem (never silent empty)", + has_comment or has_error, + f"{len(items)} item(s); comment={has_comment} error={has_error}", + ) + return ok, items + + +async def stage_user_search() -> tuple[bool, list[dict[str, Any]]]: + _hr(f"STAGE 8 — user search (browser): {_PROFILE!r}") + from app.proprietary.platforms.tiktok import search_tiktok_users + + # Unlike keyword *video* search, the account-search XHR serves anonymous + # headless sessions — so this asserts real records, not just degradation. + items = await search_tiktok_users([_PROFILE], per_query=_COUNT, limit=_COUNT) + real = [it for it in items if not it.get("errorCode")] + ok = _check( + "user search returns account records", + bool(real) and bool(real[0].get("uniqueId") or real[0].get("name")), + f"{len(items)} item(s); accounts={len(real)}", + ) + if real: + print(f" sample: @{real[0].get('uniqueId') or real[0].get('name')}") + return ok, items + + +async def stage_trending() -> tuple[bool, list[dict[str, Any]]]: + _hr("STAGE 9 — trending (browser): Explore feed") + from app.proprietary.platforms.tiktok import scrape_tiktok_trending + + items = await scrape_tiktok_trending(count=_COUNT) + real = [it for it in items if not it.get("errorCode")] + ok = _check( + "trending returns normalized video items", + bool(real) and bool(real[0].get("id")) and bool(real[0].get("webVideoUrl")), + f"{len(items)} item(s); videos={len(real)}", + ) + if real: + print(f" sample: {real[0].get('webVideoUrl')} — {real[0].get('text', '')[:60]!r}") + return ok, items + + async def main() -> int: print("TikTok scraper functional e2e — live network + proxy + browser") results: dict[str, bool] = {} @@ -214,8 +275,10 @@ async def main() -> int: if video_url: ok_video, _ = await stage_blob_video(video_url) results["Stage 3 blob video"] = ok_video + ok_comments, _ = await stage_comments(video_url) + results["Stage 7 comments"] = ok_comments else: - print("\n [SKIP] Stage 3 — no captured struct to build a video URL") + print("\n [SKIP] Stage 3/7 — no captured struct to build a video URL") ok_search, _ = await stage_search_listing() results["Stage 6 search listing"] = ok_search @@ -224,6 +287,11 @@ async def main() -> int: results["Stage 2 profile listing"] = ok_profile results["Stage 5 pipeline"] = await stage_pipeline() + ok_users, _ = await stage_user_search() + results["Stage 8 user search"] = ok_users + ok_trending, _ = await stage_trending() + results["Stage 9 trending"] = ok_trending + _hr("SUMMARY") for name, ok in results.items(): print(f" {'PASS' if ok else 'FAIL/SKIP'} — {name}") diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py index 1e5f472ee..aac8efc0e 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -1,4 +1,4 @@ -"""TikTok scraper tool.""" +"""TikTok scraper tools: scrape (videos), comments, user search, and trending.""" from __future__ import annotations @@ -17,7 +17,7 @@ from ..capability import run_scraper def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the TikTok tool.""" + """Register the TikTok tools.""" @mcp.tool( name="surfsense_tiktok_scrape", @@ -86,3 +86,118 @@ def register( workspace=workspace, response_format=response_format, ) + + @mcp.tool( + name="surfsense_tiktok_comments", + title="Scrape TikTok comments", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_comments( + video_urls: Annotated[ + list[str], + Field( + description="TikTok video URLs " + "('https://www.tiktok.com/@user/video/123') to pull comments from." + ), + ], + comments_per_video: Annotated[ + int, Field(ge=1, description="Max comments to return per video.") + ] = 20, + max_items: Annotated[ + int, Field(ge=1, description="Maximum comments to return in total.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape the public comments of TikTok videos. + + Returns each comment's text, author, like count, and reply count (replies + carry the parent comment id). Example: video_urls=['https://www.tiktok.com/ + @nasa/video/123'], max_items=50. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="comments", + payload={ + "video_urls": video_urls, + "comments_per_video": comments_per_video, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_user_search", + title="Search TikTok accounts", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_user_search( + queries: Annotated[ + list[str], + Field( + description="Keywords to find TikTok accounts by, e.g. " + "['nasa', 'cooking']." + ), + ], + results_per_query: Annotated[ + int, Field(ge=1, description="Max accounts to return per query.") + ] = 10, + max_items: Annotated[ + int, Field(ge=1, description="Maximum accounts to return in total.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Find public TikTok accounts by keyword. + + Returns matching profiles with name, followers, bio, and verification — + the reliable account-discovery path (video search is login-walled). + Example: queries=['space agency'], max_items=20. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="user_search", + payload={ + "queries": queries, + "results_per_query": results_per_query, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_tiktok_trending", + title="Get trending TikTok videos", + annotations=SCRAPE, + structured_output=False, + ) + async def tiktok_trending( + max_items: Annotated[ + int, + Field(ge=1, description="Max trending videos to return from Explore."), + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Get the current trending TikTok videos from the Explore feed. + + No input needed beyond how many to return; each video comes with caption, + author, stats, music, and its web URL. Example: max_items=30. + """ + return await run_scraper( + client, + context, + platform="tiktok", + verb="trending", + payload={"max_items": max_items}, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index e5ac6bd2b..126981e3c 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -24,6 +24,9 @@ EXPECTED_TOOLS = { "surfsense_youtube_scrape", "surfsense_youtube_comments", "surfsense_tiktok_scrape", + "surfsense_tiktok_comments", + "surfsense_tiktok_user_search", + "surfsense_tiktok_trending", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_list_scraper_runs", diff --git a/surfsense_web/content/docs/connectors/native/tiktok.mdx b/surfsense_web/content/docs/connectors/native/tiktok.mdx index 409df58bd..fc4cf99c3 100644 --- a/surfsense_web/content/docs/connectors/native/tiktok.mdx +++ b/surfsense_web/content/docs/connectors/native/tiktok.mdx @@ -1,19 +1,17 @@ --- title: TikTok -description: Scrape public TikTok videos by URL, profile, hashtag, or search +description: Scrape public TikTok videos, comments, accounts, and trending feeds --- -The TikTok scraper pulls structured public data from TikTok. Give it URLs (a video, a profile, a hashtag, or a search page) and/or profiles, hashtags, or search terms, and it returns videos (caption, author, play/like/comment/share counts, music, hashtags, timestamps, and the web URL). +The TikTok connector pulls structured public data from TikTok across four verbs: **scrape** (videos), **comments**, **user search** (accounts), and **trending**. Every verb returns `{ "items": [...] }` and is billed per returned item; surfaced errors (an `errorCode` field, e.g. a withheld feed) are never charged. -## Endpoint +## Scrape videos ```bash POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/scrape ``` -## Inputs - -At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required. +Give it URLs (a video, a profile, a hashtag, or a search page) and/or profiles, hashtags, or search terms; returns videos (caption, author, play/like/comment/share counts, music, hashtags, timestamps, and the web URL). At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required. | Field | Default | Description | |-------|---------|-------------| @@ -24,22 +22,76 @@ At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required. | `results_per_page` | `10` | Max videos per profile/hashtag/search target | | `max_items` | `10` | Max total videos returned across all sources (hard cap 100) | -## Example - ```bash curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \ -H "Authorization: Bearer $SURFSENSE_API_KEY" \ -H "Content-Type: application/json" \ - -d '{ - "hashtags": ["food"], - "max_items": 25 - }' + -d '{ "hashtags": ["food"], "max_items": 25 }' ``` -The response is `{ "items": [...] }` — one item per video. Billing is per returned item. - -Video, hashtag, and search targets are the reliable paths. TikTok restricts the profile video endpoint for automated clients, so a `profiles` target can return no items even when the account is public — prefer hashtags, search, or a direct video URL. +Video and hashtag targets are the reliable video paths. A `profiles` target returns the account's **metadata** (name, followers, bio, verification) reliably, but TikTok often withholds its **video list** from automated clients — so a profile can return metadata with no videos. Keyword **video** search is login-walled and returns a surfaced error; to find accounts by keyword use **user search** below. -For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok → Scrape** in your workspace. +## Comments + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/comments +``` + +Given TikTok video URLs, returns each video's public comment thread: comment text, author, like count, and reply count (replies carry `repliesToId`, the parent comment id). + +| Field | Default | Description | +|-------|---------|-------------| +| `video_urls` | — | TikTok video URLs (`/@/video/`), max 20 per call | +| `comments_per_video` | `20` | Max comments per video | +| `max_items` | `20` | Max total comments returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/comments" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "video_urls": ["https://www.tiktok.com/@nasa/video/123"], "max_items": 50 }' +``` + +## User search + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/user_search +``` + +Finds public TikTok accounts by keyword — the reliable account-discovery path. Returns matching profiles with name, followers, bio, and verification. + +| Field | Default | Description | +|-------|---------|-------------| +| `queries` | — | Keywords to find accounts by, e.g. `["nasa", "cooking"]` (max 20) | +| `results_per_query` | `10` | Max accounts per query | +| `max_items` | `10` | Max total accounts returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/user_search" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "queries": ["space agency"], "max_items": 20 }' +``` + +## Trending + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/trending +``` + +Returns the current trending videos from TikTok's Explore feed — no input needed beyond how many to return. Items use the same video shape as **scrape** and bill on the same per-video meter. + +| Field | Default | Description | +|-------|---------|-------------| +| `max_items` | `20` | Max trending videos returned (hard cap 100) | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/trending" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "max_items": 30 }' +``` + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → TikTok** in your workspace. diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index a25b71d61..f23b73865 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -8,7 +8,7 @@ export const tiktok: ConnectorPageContent = { metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", metaDescription: - "Scrape public TikTok videos by hashtag, search, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, search, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", keywords: [ "tiktok scraper", "tiktok scraper api", @@ -17,6 +17,9 @@ export const tiktok: ConnectorPageContent = { "scrape tiktok", "tiktok data api", "tiktok hashtag scraper", + "tiktok comments scraper", + "tiktok trending scraper", + "tiktok user search", "tiktok trend tracking", "tiktok mcp", "social listening", @@ -52,7 +55,7 @@ export const tiktok: ConnectorPageContent = { }, extractIntro: - "Every call returns structured video items. Point the API at a hashtag, a search query, a creator profile, or a specific video URL.", + "Every call returns structured items. Scrape videos from a hashtag, search query, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", extractFields: [ { label: "Videos", @@ -101,7 +104,7 @@ export const tiktok: ConnectorPageContent = { { title: "Campaign and sentiment tracking", description: - "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — and report the momentum, not a vanity view count.", + "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — then pull the comments on top videos to read how the audience actually reacts, not just a vanity view count.", }, ], @@ -134,7 +137,7 @@ export const tiktok: ConnectorPageContent = { { feature: "Agent-ready", official: "No; you build the harness yourself", - surfsense: "MCP server exposes tiktok.scrape as a native tool", + surfsense: "MCP server exposes scrape, comments, user search, and trending as native tools", }, ], }, @@ -262,7 +265,12 @@ export const tiktok: ConnectorPageContent = { { question: "Can I scrape a specific creator's videos?", answer: - "You can pass profiles or a profile URL, but TikTok restricts its profile video endpoint for automated clients, so a profile target can return no videos even for a public account. For reliable results, scrape by hashtag, by search query, or by a direct video URL.", + "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag, by search query, or by a direct video URL.", + }, + { + question: "What TikTok data can I scrape?", + answer: + "Four verbs: scrape (videos by hashtag, search, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", }, ], diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 4503662f8..d739b9dc7 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -53,7 +53,16 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ id: "tiktok", label: "TikTok", icon: TikTokIcon, - verbs: [{ name: "tiktok.scrape", verb: "scrape", label: "Scrape" }], + verbs: [ + { name: "tiktok.scrape", verb: "scrape", label: "Scrape" }, + { name: "tiktok.comments", verb: "comments", label: "Comments" }, + { + name: "tiktok.user_search", + verb: "user_search", + label: "User Search", + }, + { name: "tiktok.trending", verb: "trending", label: "Trending" }, + ], }, { id: "google_maps", From bcfc3a5c3a79bd1688a476cd9d51da5c6702fb2a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 9 Jul 2026 20:04:30 +0200 Subject: [PATCH 078/160] test(capabilities): expect export_run in the agent tool manifest The run-reader set gained export_run (CSV export of a stored run), but this manifest assertion still expected only read_run/search_run. Add export_run so the agent-door test reflects the shipped readers. --- .../tests/unit/capabilities/access/test_agent_tools.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py index b2173292f..cdab68bc0 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py @@ -78,8 +78,14 @@ async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate): tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=caps) by_name = {t.name: t for t in tools} - # One tool per verb, plus the two shared run-reader tools. - assert set(by_name) == {"web_scrape", "web_discover", "read_run", "search_run"} + # One tool per verb, plus the shared run-reader tools. + assert set(by_name) == { + "web_scrape", + "web_discover", + "read_run", + "search_run", + "export_run", + } assert by_name["web_scrape"].description == "web.scrape does a thing." assert by_name["web_scrape"].args_schema is _EchoInput From 2a6715e2a362ceda5f0baebd7c5a5d4963f65633 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:23:53 +0530 Subject: [PATCH 079/160] feat(homepage): update hero section description and add Instagram use case Enhanced the hero section by including Instagram in the list of platforms monitored by SurfSense. Additionally, introduced a new use case for "Social sentiment mining" that details the capabilities of pulling public posts and analyzing audience reactions on Instagram. --- surfsense_web/components/homepage/hero-section.tsx | 2 +- surfsense_web/components/homepage/use-cases.tsx | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 89d1e46f5..ddcddf9cc 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -719,7 +719,7 @@ export function HeroSection() { > SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market with live data from platforms - like Reddit, YouTube, Google Maps, Google Search, and the open web. + like Reddit, Instagram, YouTube, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/homepage/use-cases.tsx b/surfsense_web/components/homepage/use-cases.tsx index 44609351d..6163f7481 100644 --- a/surfsense_web/components/homepage/use-cases.tsx +++ b/surfsense_web/components/homepage/use-cases.tsx @@ -28,6 +28,14 @@ const USE_CASES: { anchor: "Reddit API", art: "brand", }, + { + title: "Social sentiment mining", + description: + "Pull public posts, reels, and full comment threads from any creator or competitor, then score how audiences actually react to launches and campaigns.", + href: "/instagram", + anchor: "Instagram API", + art: "chat", + }, { title: "B2B lead generation", description: From abb2ea2d3b45b673ef5facdf6a3f4ebc4a5d1054 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:24:06 +0530 Subject: [PATCH 080/160] feat(auth-utils): add Instagram to public route prefixes for monitoring --- surfsense_web/lib/auth-utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 979b09605..8c8d919c5 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -39,6 +39,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/mcp-server", "/external-mcp-connectors", "/reddit", + "/instagram", "/youtube", "/google-maps", "/google-search", From d3d8f10940282d5e5fa16c3c5f6d8089e573cb85 Mon Sep 17 00:00:00 2001 From: moduvoice Date: Fri, 10 Jul 2026 11:22:54 +0700 Subject: [PATCH 081/160] feat(i18n): add Korean (ko) translation Adds a complete Korean locale (surfsense_web/messages/ko.json) with full key parity with en.json, and wires ko into every place locales are currently enumerated: next-intl routing config, the client-side locale context/loader, and the two language-switcher UIs (main app + sidebar user profile). --- surfsense_web/components/LanguageSwitcher.tsx | 3 +- .../layout/ui/sidebar/SidebarUserProfile.tsx | 3 +- surfsense_web/contexts/LocaleContext.tsx | 6 +- surfsense_web/i18n/routing.ts | 2 +- surfsense_web/messages/ko.json | 821 ++++++++++++++++++ 5 files changed, 830 insertions(+), 5 deletions(-) create mode 100644 surfsense_web/messages/ko.json diff --git a/surfsense_web/components/LanguageSwitcher.tsx b/surfsense_web/components/LanguageSwitcher.tsx index 6fdef4ca0..13cf2fe95 100644 --- a/surfsense_web/components/LanguageSwitcher.tsx +++ b/surfsense_web/components/LanguageSwitcher.tsx @@ -25,6 +25,7 @@ export function LanguageSwitcher() { { code: "pt" as const, name: "Português", flag: "🇧🇷" }, { code: "hi" as const, name: "हिन्दी", flag: "🇮🇳" }, { code: "zh" as const, name: "简体中文", flag: "🇨🇳" }, + { code: "ko" as const, name: "한국어", flag: "🇰🇷" }, ]; /** @@ -32,7 +33,7 @@ export function LanguageSwitcher() { * Updates locale in context and localStorage */ const handleLanguageChange = (newLocale: string) => { - setLocale(newLocale as "en" | "es" | "pt" | "hi" | "zh"); + setLocale(newLocale as "en" | "es" | "pt" | "hi" | "zh" | "ko"); }; return ( diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx index d5e72174e..cfb76e8dc 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx @@ -52,6 +52,7 @@ const LANGUAGES = [ { code: "pt" as const, name: "Português", flag: "🇧🇷" }, { code: "hi" as const, name: "हिन्दी", flag: "🇮🇳" }, { code: "zh" as const, name: "简体中文", flag: "🇨🇳" }, + { code: "ko" as const, name: "한국어", flag: "🇰🇷" }, ]; // Supported themes configuration @@ -157,7 +158,7 @@ export function SidebarUserProfile({ const showDownloadCta = !isDesktop && !isMobileOS && isDesktopViewport; const useMobileSubmenus = !isDesktopViewport; - const handleLanguageChange = (newLocale: "en" | "es" | "pt" | "hi" | "zh") => { + const handleLanguageChange = (newLocale: "en" | "es" | "pt" | "hi" | "zh" | "ko") => { setLocale(newLocale); }; diff --git a/surfsense_web/contexts/LocaleContext.tsx b/surfsense_web/contexts/LocaleContext.tsx index ec336ce8c..5780f374f 100644 --- a/surfsense_web/contexts/LocaleContext.tsx +++ b/surfsense_web/contexts/LocaleContext.tsx @@ -4,7 +4,7 @@ import type React from "react"; import { createContext, useCallback, useContext, useEffect, useMemo, useState } from "react"; import enMessages from "../messages/en.json"; -type Locale = "en" | "es" | "pt" | "hi" | "zh"; +type Locale = "en" | "es" | "pt" | "hi" | "zh" | "ko"; /** * Dynamically load locale messages on demand. @@ -20,6 +20,8 @@ const loadMessages = async (locale: Locale): Promise => { return (await import("../messages/pt.json")).default; case "zh": return (await import("../messages/zh.json")).default; + case "ko": + return (await import("../messages/ko.json")).default; default: return enMessages; } @@ -47,7 +49,7 @@ export function LocaleProvider({ children }: { children: React.ReactNode }) { setMounted(true); if (typeof window !== "undefined") { const stored = localStorage.getItem(LOCALE_STORAGE_KEY); - if (stored && (["en", "es", "pt", "hi", "zh"] as const).includes(stored as Locale)) { + if (stored && (["en", "es", "pt", "hi", "zh", "ko"] as const).includes(stored as Locale)) { const storedLocale = stored as Locale; setLocaleState(storedLocale); // Load messages for non-English locale diff --git a/surfsense_web/i18n/routing.ts b/surfsense_web/i18n/routing.ts index d709ff62a..a943ccfa9 100644 --- a/surfsense_web/i18n/routing.ts +++ b/surfsense_web/i18n/routing.ts @@ -7,7 +7,7 @@ import { defineRouting } from "next-intl/routing"; */ export const routing = defineRouting({ // A list of all locales that are supported - locales: ["en", "es", "pt", "hi", "zh"], + locales: ["en", "es", "pt", "hi", "zh", "ko"], // Used when no locale matches defaultLocale: "en", diff --git a/surfsense_web/messages/ko.json b/surfsense_web/messages/ko.json new file mode 100644 index 000000000..3b5d88852 --- /dev/null +++ b/surfsense_web/messages/ko.json @@ -0,0 +1,821 @@ +{ + "common": { + "app_name": "SurfSense", + "welcome": "환영합니다", + "save": "저장", + "cancel": "취소", + "delete": "삭제", + "edit": "편집", + "create": "생성", + "update": "업데이트", + "search": "검색", + "close": "닫기", + "confirm": "확인", + "back": "뒤로", + "next": "다음", + "submit": "제출", + "yes": "예", + "no": "아니오", + "add": "추가", + "remove": "제거", + "select": "선택", + "all": "전체", + "none": "없음", + "error": "오류", + "success": "성공", + "warning": "경고", + "info": "정보", + "required": "필수", + "optional": "선택 사항", + "retry": "다시 시도", + "owner": "소유자", + "shared": "공유됨", + "settings": "설정" + }, + "auth": { + "login": "로그인", + "register": "회원가입", + "logout": "로그아웃", + "email": "이메일", + "password": "비밀번호", + "confirm_password": "비밀번호 확인", + "forgot_password": "비밀번호를 잊으셨나요?", + "show_password": "비밀번호 표시", + "hide_password": "비밀번호 숨기기", + "remember_me": "로그인 상태 유지", + "sign_in": "로그인", + "sign_up": "회원가입", + "sign_in_with": "{provider}(으)로 로그인", + "dont_have_account": "계정이 없으신가요?", + "already_have_account": "이미 계정이 있으신가요?", + "reset_password": "비밀번호 재설정", + "email_required": "이메일을 입력해주세요", + "password_required": "비밀번호를 입력해주세요", + "invalid_email": "유효하지 않은 이메일 주소입니다", + "password_too_short": "비밀번호는 8자 이상이어야 합니다", + "welcome_back": "다시 오신 것을 환영합니다", + "create_account": "계정 생성", + "login_subtitle": "계정에 접속하려면 인증 정보를 입력하세요", + "register_subtitle": "SurfSense를 시작하려면 가입하세요", + "or_continue_with": "또는 다음으로 계속하기", + "by_continuing": "계속 진행하면 다음에 동의하는 것입니다", + "terms_of_service": "서비스 약관", + "and": "및", + "privacy_policy": "개인정보 처리방침", + "full_name": "이름", + "username": "사용자 이름", + "continue": "계속", + "back_to_login": "로그인으로 돌아가기", + "login_success": "로그인에 성공했습니다", + "register_success": "계정이 성공적으로 생성되었습니다", + "continue_with_google": "Google로 계속하기", + "cloud_dev_notice": "SurfSense Cloud는 현재 개발 중입니다. 자세한 내용은", + "docs": "문서", + "cloud_dev_self_hosted": "를 참고하여 자체 호스팅 버전을 확인해보세요.", + "passwords_no_match": "비밀번호가 일치하지 않습니다", + "password_mismatch": "비밀번호 불일치", + "passwords_no_match_desc": "입력하신 비밀번호가 서로 일치하지 않습니다", + "creating_account": "계정을 생성하는 중입니다", + "creating_account_btn": "계정 생성 중", + "redirecting_login": "로그인 페이지로 이동 중입니다" + }, + "workspace": { + "create_title": "워크스페이스 생성", + "create_description": "지식을 정리할 새 워크스페이스를 생성하세요", + "name_label": "이름", + "name_placeholder": "워크스페이스 이름을 입력하세요", + "description_label": "설명", + "description_placeholder": "이 워크스페이스의 용도는 무엇인가요?", + "create_button": "생성", + "creating": "생성 중", + "all_workspaces": "모든 워크스페이스", + "workspaces_count": "{count, plural, =0 {워크스페이스 없음} other {워크스페이스 #개}}", + "no_workspaces": "아직 워크스페이스가 없습니다", + "create_first_workspace": "첫 워크스페이스를 생성하여 시작해보세요", + "members_count": "{count, plural, other {멤버 #명}}", + "create_new_workspace": "새 워크스페이스 생성", + "delete_title": "워크스페이스 삭제", + "delete_confirm": "\"{name}\"을(를) 삭제하시겠습니까? 이 작업은 되돌릴 수 없으며 모든 데이터가 영구적으로 삭제됩니다.", + "leave": "나가기", + "leave_title": "워크스페이스 나가기", + "leave_confirm": "\"{name}\"에서 나가시겠습니까? 이 워크스페이스의 모든 문서와 채팅에 대한 접근 권한을 잃게 됩니다.", + "leaving": "나가는 중...", + "welcome_title": "SurfSense에 오신 것을 환영합니다", + "welcome_description": "첫 워크스페이스를 생성하여 지식을 정리하고, 소스를 연결하고, AI와 대화를 시작해보세요.", + "create_first_button": "첫 워크스페이스 생성하기" + }, + "userSettings": { + "title": "사용자 설정", + "description": "계정 설정 및 API 접근을 관리하세요", + "back_to_app": "앱으로 돌아가기", + "profile_nav_label": "프로필", + "profile_nav_description": "표시 이름과 아바타를 관리하세요", + "profile_title": "프로필", + "profile_description": "개인 정보를 업데이트하세요", + "profile_avatar": "프로필 사진", + "profile_display_name": "표시 이름", + "profile_display_name_hint": "앱 전체에서 표시되는 이름입니다", + "profile_email": "이메일", + "profile_save": "변경 사항 저장", + "profile_saved": "프로필이 성공적으로 업데이트되었습니다", + "profile_save_error": "프로필 업데이트에 실패했습니다", + "api_key_nav_label": "API 액세스", + "api_key_nav_description": "API 액세스 토큰을 관리하세요", + "api_key_title": "API 액세스", + "api_key_description": "API 요청을 인증하려면 이 키를 사용하세요", + "api_key_warning_description": "API 키는 계정에 대한 전체 접근 권한을 부여합니다. 공개적으로 공유하거나 버전 관리에 커밋하지 마세요.", + "your_api_key": "내 API 키", + "copied": "복사되었습니다!", + "copy": "클립보드에 복사", + "no_api_key": "API 키를 찾을 수 없습니다", + "usage_title": "사용 방법", + "usage_description": "Authorization 헤더에 API 키를 포함하세요:" + }, + "dashboard": { + "title": "대시보드", + "workspaces": "워크스페이스", + "documents": "문서", + "connectors": "커넥터", + "settings": "설정", + "chat": "채팅", + "api_keys": "API 키", + "profile": "프로필", + "loading_dashboard": "대시보드 로딩 중", + "loading_config": "설정 로딩 중", + "config_error": "설정 오류", + "failed_load_llm_config": "LLM 설정을 불러오지 못했습니다", + "error_loading_chats": "채팅을 불러오는 중 오류가 발생했습니다", + "loading_chat": "채팅 로딩 중", + "loading_document": "문서 로딩 중", + "no_recent_chats": "최근 채팅이 없습니다", + "error_loading_space": "워크스페이스를 불러오는 중 오류가 발생했습니다", + "unknown_workspace": "알 수 없는 워크스페이스", + "delete_chat": "채팅 삭제", + "delete_chat_confirm": "삭제하시겠습니까", + "delete_note": "노트 삭제", + "delete_note_confirm": "삭제하시겠습니까", + "action_cannot_undone": "이 작업은 되돌릴 수 없습니다.", + "deleting": "삭제 중", + "surfsense_dashboard": "SurfSense 대시보드", + "welcome_message": "SurfSense 대시보드에 오신 것을 환영합니다.", + "your_workspaces": "내 워크스페이스", + "shared": "공유됨", + "create_workspace": "워크스페이스 생성", + "add_new_workspace": "새 워크스페이스 추가", + "loading": "로딩 중", + "may_take_moment": "잠시 시간이 걸릴 수 있습니다", + "error": "오류", + "something_wrong": "문제가 발생했습니다", + "error_details": "오류 세부 정보", + "try_again": "다시 시도", + "go_home": "홈으로 이동", + "delete_workspace": "워크스페이스 삭제", + "delete_space_confirm": "\"{name}\"을(를) 삭제하시겠습니까? 이 작업은 되돌릴 수 없으며 이 워크스페이스의 모든 문서와 채팅이 영구적으로 삭제됩니다.", + "leave": "나가기", + "leave_title": "워크스페이스 나가기", + "leave_confirm": "\"{name}\"에서 나가시겠습니까? 이 워크스페이스의 모든 문서와 채팅에 대한 접근 권한을 잃게 됩니다.", + "leaving": "나가는 중...", + "no_spaces_found": "워크스페이스를 찾을 수 없습니다", + "create_first_space": "첫 워크스페이스를 생성하여 시작해보세요", + "created": "생성됨" + }, + "navigation": { + "home": "홈", + "docs": "문서", + "pricing": "가격", + "contact": "문의", + "login": "로그인", + "register": "회원가입", + "dashboard": "대시보드", + "sign_in": "로그인", + "book_a_call": "상담 예약" + }, + "nav_menu": { + "settings": "설정", + "platform": "플랫폼", + "chat": "채팅", + "manage_llms": "LLM 관리", + "sources": "소스", + "add_sources": "소스 추가", + "documents": "문서", + "upload_documents": "문서 업로드", + "add_webpages": "웹페이지 추가", + "add_youtube": "유튜브 동영상 추가", + "add_youtube_videos": "유튜브 동영상 추가", + "manage_documents": "문서 관리", + "connectors": "커넥터", + "add_connector": "커넥터 추가", + "manage_connectors": "외부 MCP 커넥터 관리", + "logs": "로그", + "all_workspaces": "모든 워크스페이스", + "team": "팀" + }, + "pricing": { + "title": "SurfSense 가격", + "subtitle": "나에게 맞는 플랜을 선택하세요", + "community_name": "커뮤니티", + "enterprise_name": "엔터프라이즈", + "forever": "영구 무료", + "contact_us": "문의하기", + "feature_llms": "100개 이상의 LLM 지원", + "feature_ollama": "로컬 Ollama 또는 vLLM 설정 지원", + "feature_embeddings": "6000개 이상의 임베딩 모델", + "feature_files": "50개 이상의 파일 확장자 지원.", + "feature_podcasts": "로컬 TTS 제공업체를 이용한 팟캐스트 지원.", + "feature_sources": "15개 이상의 외부 소스와 연결.", + "feature_extension": "인증이 필요한 콘텐츠를 포함한 동적 웹페이지를 위한 크로스 브라우저 확장 프로그램", + "upcoming_mindmaps": "출시 예정: 병합 가능한 마인드맵", + "upcoming_notes": "출시 예정: 노트 관리", + "community_desc": "강력한 기능을 갖춘 오픈소스 버전", + "get_started": "시작하기", + "everything_community": "커뮤니티의 모든 기능", + "priority_support": "우선 지원", + "access_controls": "접근 제어", + "collaboration": "협업 및 다중 사용자 기능", + "video_gen": "영상 생성", + "advanced_security": "고급 보안 기능", + "enterprise_desc": "특정 요구사항이 있는 대규모 조직을 위한 플랜", + "contact_sales": "영업팀 문의" + }, + "contact": { + "title": "문의", + "subtitle": "여러분의 이야기를 듣고 싶습니다.", + "we_are_here": "저희가 도와드리겠습니다", + "full_name": "이름", + "email_address": "이메일 주소", + "company": "회사", + "message": "메시지", + "optional": "선택 사항", + "name_placeholder": "홍길동", + "email_placeholder": "example@email.com", + "company_placeholder": "예시 주식회사", + "message_placeholder": "메시지를 입력하세요", + "submit": "제출", + "submitting": "제출 중...", + "name_required": "이름을 입력해주세요", + "name_too_long": "이름이 너무 깁니다", + "invalid_email": "유효하지 않은 이메일 주소입니다", + "email_too_long": "이메일이 너무 깁니다", + "company_required": "회사를 입력해주세요", + "company_too_long": "회사 이름이 너무 깁니다", + "message_sent": "메시지가 성공적으로 전송되었습니다!", + "we_will_contact": "최대한 빠르게 답변드리겠습니다.", + "send_failed": "메시지 전송에 실패했습니다", + "try_again_later": "잠시 후 다시 시도해주세요.", + "something_wrong": "문제가 발생했습니다" + }, + "connectors": { + "title": "커넥터", + "subtitle": "연결된 서비스와 데이터 소스를 관리하세요.", + "add_connector": "커넥터 추가", + "your_connectors": "내 커넥터", + "view_manage": "연결된 모든 서비스를 확인하고 관리하세요.", + "no_connectors": "커넥터를 찾을 수 없습니다", + "no_connectors_desc": "아직 추가된 커넥터가 없습니다. 검색 기능을 향상시키려면 커넥터를 추가하세요.", + "add_first": "첫 커넥터 추가하기", + "name": "이름", + "type": "유형", + "last_indexed": "마지막 색인 시점", + "periodic": "주기적", + "actions": "작업", + "never": "없음", + "not_indexable": "색인 불가", + "index_date_range": "날짜 범위로 색인", + "quick_index": "빠른 색인", + "quick_index_auto": "빠른 색인 (자동 날짜 범위)", + "delete_connector": "커넥터 삭제", + "delete_confirm": "이 커넥터를 삭제하시겠습니까? 이 작업은 되돌릴 수 없습니다.", + "select_date_range": "색인할 날짜 범위 선택", + "select_date_range_desc": "콘텐츠 색인을 위한 시작일과 종료일을 선택하세요. 비워두면 기본 범위가 사용됩니다.", + "start_date": "시작일", + "end_date": "종료일", + "pick_date": "날짜 선택", + "clear_dates": "날짜 초기화", + "last_30_days": "최근 30일", + "last_year": "최근 1년", + "start_indexing": "색인 시작", + "failed_load": "커넥터를 불러오지 못했습니다", + "delete_success": "커넥터가 성공적으로 삭제되었습니다", + "delete_failed": "커넥터 삭제에 실패했습니다", + "indexing_started": "커넥터 콘텐츠 색인이 시작되었습니다", + "indexing_failed": "커넥터 콘텐츠 색인에 실패했습니다" + }, + "documents": { + "title": "문서", + "subtitle": "문서와 파일을 관리하세요.", + "no_rows_selected": "선택된 행이 없습니다", + "delete_success_count": "{count}개의 문서를 성공적으로 삭제했습니다", + "delete_partial_failed": "일부 문서를 삭제하지 못했습니다", + "delete_success": "문서가 성공적으로 삭제되었습니다", + "delete_error": "문서 삭제 중 오류가 발생했습니다", + "filter_by_title": "제목으로 필터링...", + "bulk_delete": "선택 항목 삭제", + "filter_types": "유형 필터링", + "columns": "열", + "confirm_delete": "삭제 확인", + "confirm_delete_desc": "{count}개의 문서를 삭제하시겠습니까? 이 작업은 되돌릴 수 없습니다.", + "uploading": "업로드 중", + "upload_success": "문서가 성공적으로 업로드되었습니다", + "upload_failed": "문서 업로드에 실패했습니다", + "loading": "문서 로딩 중", + "error_loading": "문서를 불러오는 중 오류가 발생했습니다", + "retry": "다시 시도", + "no_documents": "문서를 찾을 수 없습니다", + "type": "유형", + "content_summary": "콘텐츠 요약", + "view_full": "요약 보기", + "filter_placeholder": "제목으로 필터링...", + "rows_per_page": "페이지당 행 수", + "refresh": "새로고침", + "upload_documents": "문서 업로드", + "create_shared_note": "공유 노트 생성", + "processing_documents": "문서 처리 중...", + "delete_in_progress_warning": "{count}개의 문서가 대기 중이거나 처리 중이어서 삭제할 수 없습니다.", + "delete_conflict_error": "{count}개의 문서가 처리를 시작했습니다. 잠시 후 다시 시도해주세요." + }, + "add_connector": { + "title": "도구 연결하기", + "subtitle": "즐겨 사용하는 서비스와 통합하여 리서치 역량을 강화하세요.", + "messaging": "메시징", + "project_management": "프로젝트 관리", + "documentation": "문서화", + "development": "개발", + "databases": "데이터베이스", + "productivity": "생산성", + "web_crawling": "웹 크롤링", + "connect": "연결", + "coming_soon": "출시 예정", + "connected": "연결됨", + "manage": "관리", + "tavily_desc": "Tavily API를 사용하여 웹을 검색합니다", + "searxng_desc": "자체 SearxNG 메타 검색 인스턴스를 사용하여 웹 결과를 가져옵니다.", + "linkup_desc": "Linkup API를 사용하여 웹을 검색합니다", + "elasticsearch_desc": "Elasticsearch에 연결하여 문서, 로그, 메트릭을 색인하고 검색합니다.", + "baidu_desc": "Baidu AI Search API를 사용하여 중국어 웹을 검색합니다", + "slack_desc": "Slack 워크스페이스에 연결하여 메시지와 채널에 접근합니다.", + "teams_desc": "Microsoft Teams에 연결하여 팀의 대화 내용에 접근합니다.", + "discord_desc": "Discord 서버에 연결하여 메시지와 채널에 접근합니다.", + "linear_desc": "Linear에 연결하여 이슈, 댓글, 프로젝트 데이터를 검색합니다.", + "jira_desc": "Jira에 연결하여 이슈, 티켓, 프로젝트 데이터를 검색합니다.", + "clickup_desc": "ClickUp에 연결하여 작업, 댓글, 프로젝트 데이터를 검색합니다.", + "notion_desc": "Notion 워크스페이스에 연결하여 페이지와 데이터베이스에 접근합니다.", + "github_desc": "GitHub PAT를 연결하여 접근 가능한 저장소의 코드와 문서를 색인합니다.", + "confluence_desc": "Confluence에 연결하여 페이지, 댓글, 문서를 검색합니다.", + "bookstack_desc": "BookStack에 연결하여 위키 페이지와 문서를 검색합니다.", + "airtable_desc": "Airtable에 연결하여 레코드, 테이블, 데이터베이스 콘텐츠를 검색합니다.", + "luma_desc": "Luma에 연결하여 이벤트, 모임을 검색합니다.", + "circleback_desc": "Circleback에서 웹훅을 통해 회의 노트, 녹취록, 액션 아이템을 받습니다.", + "calendar_desc": "Google Calendar에 연결하여 이벤트, 회의, 일정을 검색합니다.", + "gmail_desc": "Gmail 계정에 연결하여 이메일을 검색합니다.", + "google_drive_desc": "Google Drive에 연결하여 파일과 문서를 색인하고 검색합니다.", + "zoom_desc": "Zoom에 연결하여 회의 녹화본과 녹취록에 접근합니다.", + "webcrawler_desc": "공개된 모든 웹페이지의 콘텐츠를 크롤링하고 색인합니다." + }, + "upload_documents": { + "title": "문서 업로드", + "subtitle": "파일을 업로드하여 AI 기반 대화를 통해 검색하고 활용할 수 있도록 만드세요.", + "file_size_limit": "파일당 최대 크기: 500MB", + "drop_files": "파일이나 폴더를 여기에 놓으세요", + "drag_drop": "파일이나 폴더를 여기로 드래그 앤 드롭하세요", + "drag_drop_more": "파일을 더 추가하려면 놓거나 찾아보세요", + "or_browse": "또는 클릭하여 찾아보기", + "browse_files": "파일 찾아보기", + "browse_folder": "폴더 찾아보기", + "selected_files": "{count}개 파일 선택됨", + "total_size": "전체 크기", + "clear_all": "모두 지우기", + "uploading_files": "파일 업로드 중", + "uploading": "업로드 중", + "upload_button": "{count}개 파일 업로드", + "upload_initiated": "업로드 작업 시작됨", + "upload_initiated_desc": "파일 업로드가 시작되었습니다", + "upload_error": "업로드 오류", + "upload_error_desc": "파일 업로드 중 오류가 발생했습니다", + "supported_file_types": "지원되는 파일 형식", + "file_too_large": "파일이 너무 큽니다", + "file_too_large_desc": "\"{name}\"이(가) 파일당 제한 용량인 {maxMB}MB를 초과합니다.", + "no_supported_files_in_folder": "선택한 폴더에서 지원되는 파일 형식을 찾을 수 없습니다.", + "uploading_folder": "폴더 업로드 중…", + "upload_folder_button": "폴더 업로드 ({count}개 파일)", + "select_files_or_folder": "파일 또는 폴더 선택", + "tap_select_files_or_folder": "탭하여 파일 또는 폴더 선택", + "processing_mode": "처리 모드", + "basic_mode": "기본", + "basic_mode_desc": "표준 처리 (페이지당 1 크레딧)", + "premium_mode": "프리미엄", + "premium_mode_desc": "금융, 의료, 법률 문서에 최적화 (페이지당 10 크레딧)" + }, + "add_webpage": { + "title": "크롤링할 웹페이지 추가", + "subtitle": "일괄 색인 및 주기적 동기화를 위해 지식 베이스에 웹페이지를 추가하세요", + "label": "크롤링할 URL 입력", + "placeholder": "URL을 입력하고 Enter를 누르세요", + "hint": "각 URL 입력 후 Enter를 눌러 여러 개를 추가할 수 있습니다", + "tips_title": "URL 크롤링 팁:", + "tip_1": "http:// 또는 https://를 포함한 전체 URL을 입력하세요", + "tip_2": "웹사이트가 크롤링을 허용하는지 확인하세요", + "tip_3": "공개 웹페이지가 가장 잘 작동합니다", + "tip_4": "크롤링은 웹사이트 크기에 따라 시간이 걸릴 수 있습니다", + "chat_tip": "색인 없이 웹페이지에서 빠른 답변을 얻고 싶으신가요? URL을 채팅에 바로 붙여넣으세요.", + "cancel": "취소", + "submit": "크롤링할 URL 제출", + "submitting": "제출 중...", + "error_no_url": "URL을 하나 이상 추가해주세요", + "error_invalid_urls": "유효하지 않은 URL이 발견되었습니다: {urls}", + "crawling_toast": "URL 크롤링", + "crawling_toast_desc": "URL 크롤링 프로세스를 시작합니다...", + "success_toast": "크롤링 성공", + "success_toast_desc": "URL이 크롤링을 위해 제출되었습니다", + "error_toast": "크롤링 오류", + "error_toast_desc": "URL 크롤링 중 오류가 발생했습니다", + "error_generic": "URL 크롤링 중 오류가 발생했습니다", + "invalid_url_toast": "유효하지 않은 URL", + "invalid_url_toast_desc": "유효한 URL을 입력해주세요", + "duplicate_url_toast": "중복된 URL", + "duplicate_url_toast_desc": "이미 추가된 URL입니다" + }, + "add_youtube": { + "title": "유튜브 동영상 추가", + "subtitle": "지식 베이스에 유튜브 동영상이나 재생목록을 추가하세요", + "label": "유튜브 동영상 또는 재생목록 URL", + "placeholder": "유튜브 동영상 또는 재생목록 URL을 붙여넣으세요", + "hint": "유튜브 URL을 붙여넣으면 자동으로 추가됩니다. URL을 입력하고 Enter를 눌러도 됩니다.", + "tips_title": "유튜브 동영상 추가 팁:", + "tip_1": "표준 유튜브 URL을 사용하세요 (youtube.com/watch?v= 또는 youtu.be/)", + "tip_2": "동영상이 공개적으로 접근 가능한지 확인하세요", + "tip_3": "지원 형식: youtube.com/watch?v=VIDEO_ID 또는 youtu.be/VIDEO_ID", + "tip_4": "처리는 동영상 길이에 따라 시간이 걸릴 수 있습니다", + "tip_5": "재생목록 URL(youtube.com/playlist?list=...)을 붙여넣으면 모든 동영상이 한 번에 추가됩니다", + "chat_tip": "지식 베이스에 저장하지 않고 빠르게 요약만 원하시나요? 유튜브 링크를 채팅에 바로 붙여넣으세요.", + "preview": "미리보기", + "cancel": "취소", + "submit": "추가", + "processing": "처리 중...", + "error_no_video": "유튜브 동영상 URL을 하나 이상 추가해주세요", + "error_invalid_urls": "유효하지 않은 유튜브 URL이 발견되었습니다: {urls}", + "processing_toast": "유튜브 동영상 처리 중", + "processing_toast_desc": "유튜브 동영상 처리를 시작합니다...", + "success_toast": "처리 성공", + "success_toast_desc": "유튜브 동영상이 처리를 위해 제출되었습니다", + "error_toast": "처리 오류", + "error_toast_desc": "유튜브 동영상 처리 중 오류가 발생했습니다", + "error_generic": "유튜브 동영상 처리 중 오류가 발생했습니다", + "invalid_url_toast": "유효하지 않은 유튜브 URL", + "invalid_url_toast_desc": "유효한 유튜브 동영상 또는 재생목록 URL을 입력해주세요", + "duplicate_url_toast": "중복된 URL", + "duplicate_url_toast_desc": "이미 추가된 유튜브 동영상입니다", + "resolving_playlist": "재생목록 동영상을 확인하는 중...", + "resolving_playlist_toast": "재생목록 확인 중", + "resolving_playlist_toast_desc": "재생목록에서 동영상 목록을 가져오는 중입니다...", + "playlist_resolved_toast": "재생목록 확인 완료", + "playlist_resolved_toast_desc": "재생목록에서 {count}개의 동영상이 추가되었습니다", + "playlist_error_toast": "재생목록 오류" + }, + "settings": { + "title": "설정", + "subtitle": "이 워크스페이스의 LLM 설정과 역할 할당을 관리하세요.", + "back_to_dashboard": "대시보드로 돌아가기", + "models": "모델", + "roles": "역할", + "llm_role_management": "LLM 역할 관리", + "llm_role_desc": "LLM 설정을 다양한 목적에 맞는 특정 역할에 할당하세요.", + "no_llm_configs_found": "LLM 설정을 찾을 수 없습니다. 역할을 할당하기 전에 모델 설정 탭에서 LLM 제공업체를 하나 이상 추가해주세요.", + "select_llm_config": "LLM 설정 선택", + "long_context_llm": "긴 컨텍스트 LLM", + "fast_llm": "빠른 LLM", + "strategic_llm": "전략적 LLM", + "long_context_desc": "긴 문서 요약과 복잡한 질의응답을 처리합니다", + "long_context_examples": "문서 분석, 리서치 종합, 복잡한 질의응답", + "large_context_window": "넓은 컨텍스트 윈도우", + "deep_reasoning": "심층 추론", + "complex_analysis": "복잡한 분석", + "fast_llm_desc": "빠른 응답과 실시간 상호작용에 최적화됨", + "fast_llm_examples": "빠른 검색, 간단한 질문, 즉각적인 응답", + "low_latency": "낮은 지연 시간", + "quick_responses": "빠른 응답", + "real_time_chat": "실시간 채팅", + "strategic_llm_desc": "계획 수립과 전략적 의사결정을 위한 고급 추론", + "strategic_llm_examples": "워크플로우 계획, 전략적 분석, 복잡한 문제 해결", + "strategic_thinking": "전략적 사고", + "long_term_planning": "장기 계획", + "complex_reasoning": "복잡한 추론", + "use_cases": "사용 사례", + "assign_llm_config": "LLM 설정 할당", + "unassigned": "미할당", + "assigned": "할당됨", + "model": "모델", + "base": "기본", + "all_roles_assigned": "모든 역할이 할당되어 사용 준비가 완료되었습니다! LLM 설정이 완료되었습니다.", + "save_changes": "변경 사항 저장", + "saving": "저장 중", + "reset": "재설정", + "status": "상태", + "status_ready": "준비 완료", + "status_setup": "설정 필요", + "complete_role_assignments": "모든 기능을 사용하려면 모든 역할 할당을 완료하세요. 각 역할은 워크플로우에서 서로 다른 목적을 수행합니다.", + "all_roles_saved": "모든 역할이 할당되고 저장되었습니다!", + "progress": "진행 상황", + "roles_assigned_count": "{total}개 중 {assigned}개 역할 할당됨" + }, + "logs": { + "title": "작업 로그", + "subtitle": "모든 작업 실행 로그를 모니터링하고 분석하세요", + "refresh": "새로고침", + "delete_selected": "선택 항목 삭제", + "confirm_title": "정말로 진행하시겠습니까?", + "confirm_delete_desc": "이 작업은 되돌릴 수 없습니다. 선택한 로그 {count}개가 영구적으로 삭제됩니다.", + "cancel": "취소", + "delete": "삭제", + "level": "수준", + "status": "상태", + "source": "출처", + "message": "메시지", + "created_at": "생성일", + "actions": "작업", + "system": "시스템", + "filter_by_message": "메시지로 필터링...", + "filter_by": "필터링 기준", + "total_logs": "전체 로그", + "active_tasks": "활성 작업", + "success_rate": "성공률", + "recent_failures": "최근 실패", + "last_hours": "최근 {hours}시간", + "currently_running": "실행 중", + "successful": "성공", + "need_attention": "확인 필요", + "no_logs": "로그를 찾을 수 없습니다", + "loading": "로그 로딩 중...", + "error_loading": "로그를 불러오는 중 오류가 발생했습니다", + "columns": "열", + "failed_load_summary": "요약을 불러오지 못했습니다", + "retry": "다시 시도", + "view": "보기", + "toggle_columns": "열 표시 전환", + "rows_per_page": "페이지당 행 수", + "view_metadata": "메타데이터 보기", + "log_deleted_success": "로그가 성공적으로 삭제되었습니다", + "log_deleted_error": "로그 삭제에 실패했습니다", + "confirm_delete_log_title": "확인하시겠습니까?", + "confirm_delete_log_desc": "이 작업은 되돌릴 수 없습니다. 로그 항목이 영구적으로 삭제됩니다.", + "deleting": "삭제 중" + }, + "onboard": { + "welcome_title": "SurfSense에 오신 것을 환영합니다", + "welcome_subtitle": "시작하려면 LLM 설정을 구성해보세요", + "step_of": "{total}단계 중 {current}단계", + "percent_complete": "{percent}% 완료", + "add_llm_provider": "LLM 제공업체 추가", + "assign_llm_roles": "LLM 역할 할당", + "setup_llm_configuration": "LLM 설정 구성", + "configure_providers_and_assign_roles": "LLM 제공업체를 추가하고 특정 역할에 할당하세요", + "assign_llm_roles_title": "LLM 역할 할당", + "complete_role_assignment": "계속하려면 LLM 설정을 특정 역할에 할당하세요", + "setup_complete": "설정 완료", + "configure_first_provider": "첫 모델 제공업체를 구성하세요", + "assign_specific_roles": "LLM 설정에 특정 역할을 할당하세요", + "all_set": "SurfSense를 사용할 준비가 모두 끝났습니다!", + "loading_config": "설정을 불러오는 중...", + "previous": "이전", + "next": "다음", + "complete_setup": "설정 완료", + "add_provider_instruction": "계속하려면 LLM 제공업체를 하나 이상 추가하세요. 다음 단계에서 여러 제공업체를 구성하고 각각에 특정 역할을 선택할 수 있습니다.", + "your_llm_configs": "내 LLM 설정", + "model": "모델", + "language": "언어", + "base": "기본", + "add_provider_title": "LLM 제공업체 추가", + "add_provider_subtitle": "시작하려면 첫 모델 제공업체를 구성하세요", + "add_provider_button": "제공업체 추가", + "add_new_llm_provider": "새 LLM 제공업체 추가", + "configure_new_provider": "AI 어시스턴트를 위한 새 언어 모델 제공업체를 구성하세요", + "config_name": "설정 이름", + "config_name_required": "설정 이름 *", + "config_name_placeholder": "예: My OpenAI GPT-4", + "provider": "제공업체", + "provider_required": "제공업체 *", + "provider_placeholder": "제공업체 선택", + "language_optional": "언어 (선택 사항)", + "language_placeholder": "언어 선택", + "custom_provider_name": "사용자 지정 제공업체 이름 *", + "custom_provider_placeholder": "예: my-custom-provider", + "model_name_required": "모델 이름 *", + "model_name_placeholder": "예: gpt-4", + "examples": "예시", + "api_key_required": "API 키 *", + "api_key_placeholder": "API 키", + "api_base_optional": "API 베이스 URL (선택 사항)", + "api_base_placeholder": "예: https://api.openai.com/v1", + "adding": "추가 중...", + "add_provider": "제공업체 추가", + "cancel": "취소", + "assign_roles_instruction": "LLM 설정을 특정 역할에 할당하세요. 각 역할은 워크플로우에서 서로 다른 목적을 수행합니다.", + "no_llm_configs_found": "LLM 설정을 찾을 수 없습니다", + "add_provider_before_roles": "역할을 할당하기 전에 이전 단계에서 LLM 제공업체를 하나 이상 추가해주세요.", + "long_context_llm_title": "긴 컨텍스트 LLM", + "long_context_llm_desc": "긴 문서 요약과 복잡한 질의응답을 처리합니다", + "long_context_llm_examples": "문서 분석, 리서치 종합, 복잡한 질의응답", + "fast_llm_title": "빠른 LLM", + "fast_llm_desc": "빠른 응답과 실시간 상호작용에 최적화됨", + "fast_llm_examples": "빠른 검색, 간단한 질문, 즉각적인 응답", + "strategic_llm_title": "전략적 LLM", + "strategic_llm_desc": "계획 수립과 전략적 의사결정을 위한 고급 추론", + "strategic_llm_examples": "워크플로우 계획, 전략적 분석, 복잡한 문제 해결", + "use_cases": "사용 사례", + "assign_llm_config": "LLM 설정 할당", + "select_llm_config": "LLM 설정 선택", + "assigned": "할당됨", + "all_roles_assigned_saved": "모든 역할이 할당되고 저장되었습니다!", + "progress": "진행 상황", + "roles_assigned": "{total}개 중 {assigned}개 역할 할당됨", + "global_configs": "전역 설정", + "your_configs": "내 설정" + }, + "model_config": { + "title": "모델 설정", + "subtitle": "LLM 제공업체 설정과 API 설정을 관리하세요.", + "refresh": "새로고침", + "loading": "설정 로딩 중...", + "total_configs": "전체 설정", + "unique_providers": "고유 제공업체", + "system_status": "시스템 상태", + "active": "활성", + "your_configs": "내 설정", + "manage_configs": "LLM 제공업체를 관리하고 구성하세요", + "no_configs": "아직 설정이 없습니다", + "no_configs_desc": "자체 LLM 제공업체 설정을 추가하세요.", + "add_first_config": "첫 설정 추가", + "created": "생성됨" + }, + "sidebar": { + "recents": "최근 항목", + "chats": "채팅", + "shared_chats": "공유된 채팅", + "search_chats": "채팅 검색", + "no_chats_found": "채팅을 찾을 수 없습니다", + "no_shared_chats": "공유된 채팅이 없습니다", + "shared_chat": "공유된 채팅", + "view_all_shared_chats": "공유된 모든 채팅 보기", + "view_all_chats": "모든 채팅 보기", + "show_all": "모두 보기", + "hide": "숨기기", + "no_chats": "채팅이 없습니다", + "start_new_chat_hint": "새 채팅 시작하기", + "error_loading_chats": "채팅을 불러오는 중 오류가 발생했습니다", + "chat_deleted": "채팅이 성공적으로 삭제되었습니다", + "error_deleting_chat": "채팅 삭제에 실패했습니다", + "delete": "삭제", + "try_different_search": "다른 검색어를 시도해보세요", + "updated": "업데이트됨", + "more_options": "추가 옵션", + "clear_search": "검색 지우기", + "archive": "보관", + "unarchive": "복원", + "chat_archived": "채팅이 보관되었습니다", + "chat_unarchived": "채팅이 복원되었습니다", + "chat_renamed": "채팅 이름이 변경되었습니다", + "error_renaming_chat": "채팅 이름 변경에 실패했습니다", + "rename": "이름 변경", + "rename_chat": "채팅 이름 변경", + "rename_chat_description": "이 대화의 새 이름을 입력하세요.", + "chat_title_placeholder": "채팅 제목", + "renaming": "이름 변경 중", + "no_archived_chats": "보관된 채팅이 없습니다", + "error_archiving_chat": "채팅 보관에 실패했습니다", + "new_chat": "새 채팅", + "select_workspace": "워크스페이스 선택", + "manage_members": "멤버 관리", + "workspace_settings": "워크스페이스 설정", + "logs": "로그", + "see_all_workspaces": "모든 워크스페이스 보기", + "expand_sidebar": "사이드바 펼치기", + "collapse_sidebar": "사이드바 접기", + "user_settings": "사용자 설정", + "language": "언어", + "theme": "테마", + "light": "라이트", + "dark": "다크", + "system": "시스템", + "logout": "로그아웃", + "loggingOut": "로그아웃 중...", + "learn_more": "더 알아보기", + "documentation": "문서", + "github": "GitHub", + "download_for_os": "{os}용 다운로드", + "inbox": "받은 편지함", + "search_inbox": "받은 편지함 검색", + "mark_all_read": "모두 읽음으로 표시", + "mark_as_read": "읽음으로 표시", + "mentions": "멘션", + "comments": "댓글", + "status": "상태", + "no_results_found": "결과를 찾을 수 없습니다", + "no_mentions": "멘션이 없습니다", + "no_mentions_hint": "다른 사람이 남긴 멘션이 여기에 표시됩니다", + "no_comments": "댓글이 없습니다", + "no_comments_hint": "멘션과 답글이 여기에 표시됩니다", + "no_status_updates": "상태 업데이트가 없습니다", + "no_status_updates_hint": "문서 및 커넥터 업데이트가 여기에 표시됩니다", + "filter": "필터", + "all": "전체", + "unread": "읽지 않음", + "errors_only": "오류만", + "connectors": "커넥터", + "all_connectors": "모든 커넥터", + "sources": "소스", + "all_sources": "모든 소스", + "close": "닫기", + "cancel": "취소" + }, + "errors": { + "something_went_wrong": "문제가 발생했습니다", + "try_again": "다시 시도해주세요", + "not_found": "찾을 수 없음", + "unauthorized": "인증되지 않음", + "forbidden": "접근 금지", + "server_error": "서버 오류", + "network_error": "네트워크 오류" + }, + "workspaceSettings": { + "title": "워크스페이스 설정", + "back_to_app": "앱으로 돌아가기", + "nav_general": "일반", + "nav_general_desc": "이름, 설명 및 기본 정보", + "nav_models": "모델", + "nav_agent_models": "채팅 모델", + "nav_agent_models_desc": "프롬프트와 인용을 포함한 모델", + "nav_system_instructions": "시스템 지침", + "nav_system_instructions_desc": "워크스페이스 전체 AI 지침", + "nav_public_links": "공개 채팅", + "nav_public_links_desc": "공개적으로 공유된 채팅 관리", + "nav_team_roles": "팀 역할", + "nav_team_roles_desc": "팀 역할 및 권한 관리", + "general_name_label": "이름", + "general_name_placeholder": "워크스페이스 이름을 입력하세요", + "general_name_description": "워크스페이스의 고유한 이름입니다.", + "general_description_label": "설명", + "general_description_placeholder": "워크스페이스 설명을 입력하세요", + "general_description_description": "이 워크스페이스가 어떤 용도로 사용되는지에 대한 간단한 설명입니다.", + "general_reset": "변경 사항 재설정", + "general_save": "변경 사항 저장", + "general_saving": "저장 중", + "general_unsaved_changes": "저장하지 않은 변경 사항이 있습니다. 적용하려면 \"변경 사항 저장\"을 클릭하세요." + }, + "homepage": { + "hero_title_part1": "팀을 위한", + "hero_title_part2": "AI 워크스페이스", + "hero_description": "데이터 제한 없이 팀을 위한 개인정보 보호 중심의 오픈소스 NotebookLM 대안입니다.", + "cta_start_trial": "무료 체험 시작", + "cta_explore": "둘러보기", + "integrations_title": "통합", + "integrations_subtitle": "팀의 가장 중요한 도구들과 통합하세요", + "features_title": "팀을 위한 AI 기반 지식 허브", + "features_subtitle": "협업을 강화하고, 생산성을 높이고, 워크플로우를 간소화하도록 설계된 강력한 기능들.", + "feature_workflow_title": "간소화된 워크플로우", + "feature_workflow_desc": "모든 지식과 리소스를 하나의 지능형 워크스페이스에 집중시키세요. 필요한 것을 즉시 찾아 의사결정 속도를 높이세요.", + "feature_collaboration_title": "매끄러운 협업", + "feature_collaboration_desc": "팀 전체를 하나로 정렬시키는 실시간 협업 도구로 손쉽게 함께 작업하세요.", + "feature_customizable_title": "완벽한 커스터마이징", + "feature_customizable_desc": "100개 이상의 선도적인 LLM 중에서 선택하고 필요에 따라 모든 모델을 원활하게 호출하세요.", + "cta_transform": "팀의", + "cta_transform_bold": "탐색과 협업 방식을 혁신하세요", + "cta_unite_start": "지능형 검색으로", + "cta_unite_knowledge": "팀의 지식을", + "cta_unite_middle": "하나의 협업 공간에서", + "cta_unite_search": "통합하세요", + "cta_talk_to_us": "문의하기", + "features": { + "find_ask_act": { + "title": "찾고, 묻고, 실행하기", + "description": "회사와 개인 지식 전반에서 즉각적인 정보, 상세한 업데이트, 출처가 명시된 답변을 얻으세요." + }, + "real_time_collab": { + "title": "실시간으로 함께 작업하기", + "description": "회사 문서를 실시간 편집, 동기화된 콘텐츠, 현재 접속 상태가 반영되는 다중 사용자 공간으로 전환하세요." + }, + "beyond_text": { + "title": "텍스트를 넘어선 협업", + "description": "팀이 함께 댓글을 남기고, 공유하고, 다듬을 수 있는 팟캐스트와 멀티미디어를 제작하세요." + }, + "context_counts": { + "title": "맥락이 중요한 곳에", + "description": "채팅과 문서에 직접 댓글을 추가하여 명확하고 즉각적인 피드백을 남기세요." + }, + "citation_illustration_title": "클릭 가능한 출처 참조를 보여주는 인용 기능 일러스트", + "referenced_chunk": "참조된 청크", + "collab_illustration_label": "텍스트 편집기에서의 실시간 협업 일러스트.", + "real_time": "실시간", + "collab_part1": "협", + "collab_part2": "", + "collab_part3": "업", + "annotation_illustration_label": "주석 댓글이 있는 텍스트 편집기 일러스트.", + "add_context_with": "다음으로 맥락 추가", + "comments": "댓글", + "example_comment": "내일 이 이야기를 나눠봐요!" + } + }, + "public_chat": { + "not_found_title": "이 채팅은 삭제되었습니다.", + "click_here": "여기를 클릭", + "sign_in_prompt": "하여 SurfSense에 로그인하고 나만의 채팅을 시작해보세요." + } +} From c679f2a3ef286162d3223fdce0cdb7c4b2ab3044 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 16:45:45 +0200 Subject: [PATCH 082/160] fix(tiktok): fetch profile video lists headful + dismiss login modal The profile feed (/api/post/item_list) returns an empty 200 to headless sessions but serves data headful on the same proxy IP. Run fetch_item_list headful and dismiss the login modal that blocks mid-scroll. --- .../platforms/tiktok/session/listing.py | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index e94984440..c1f3dfc15 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -11,8 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile -feed (``/api/post/item_list``) is additionally soft-blocked: TikTok returns an -empty 200 even to its own signed request, so profile targets yield nothing. +feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so +:func:`fetch_item_list` runs headful. ponytail: headful needs a display — run it +under Xvfb on a headless server. """ from __future__ import annotations @@ -81,12 +82,33 @@ def _has_mstoken(page: Any) -> bool: return False +def _dismiss_login_modal(page: Any) -> None: + """Close the login modal that blocks scrolling; Escape as fallback. + + Only the modal's own close button, never a generic dialog button (avoids + clicking "Log in"). + """ + try: + closed = page.evaluate( + """() => { + const btn = document.querySelector('[data-e2e="modal-close-inner-button"]'); + if (btn) { btn.click(); return true; } + return false; + }""" + ) + if not closed: + page.keyboard.press("Escape") + except Exception: + pass + + def _scroll_page(page: Any, collected: list[dict[str, Any]], target_count: int) -> None: """Page down a listing feed until enough items are captured or it stops growing.""" last_height = 0 for _ in range(_SCROLL_MAX_ROUNDS): if len(collected) >= target_count: break + _dismiss_login_modal(page) page.evaluate("window.scrollTo(0, document.body.scrollHeight)") page.wait_for_timeout(_SCROLL_SETTLE_MS) height = page.evaluate("document.body.scrollHeight") @@ -207,12 +229,14 @@ def _fetch_sync( markers: tuple[str, ...], extract: ExtractFn, interact: InteractFn, + *, + headless: bool = True, ) -> list[dict[str, Any]]: collected: list[dict[str, Any]] = [] kwargs = build_stealthy_kwargs(get_stealth_config()) StealthyFetcher.fetch( url, - headless=True, + headless=headless, network_idle=False, proxy=get_proxy_url(), page_action=_build_page_action( @@ -224,7 +248,10 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: - """Return up to ``target_count`` itemStructs from a listing page's XHRs.""" + """Return up to ``target_count`` itemStructs from a listing page's XHRs. + + Runs headful: the profile feed returns an empty 200 to headless sessions. + """ return await asyncio.to_thread( _fetch_sync, page_url, @@ -232,6 +259,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An _ITEM_LIST_MARKERS, items_from_response, _scroll_page, + headless=False, ) From 44966315f1aade60eb2e843f58dcb0d4cdd93579 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 16:45:53 +0200 Subject: [PATCH 083/160] docs(tiktok): update e2e stage 2 notes for headful profile feed Profile videos now return via the headful item_list capture; reword the stage 2 comments to expect real videos, ErrorItem only as fallback. --- surfsense_backend/scripts/e2e_tiktok_scrape.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index cb5a996eb..ad3337ef2 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,9 +7,8 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile via the full pipeline: TikTok soft-blocks the anonymous - profile feed, so this asserts graceful degradation — real videos OR - a single honest ErrorItem, never a silent empty. + Stage 2 — profile via the full pipeline: asserts real videos come back + (fetch_item_list runs headful), degrading to one ErrorItem if not. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. @@ -107,10 +106,8 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - # TikTok soft-blocks the profile feed for anonymous headless sessions - # (empty 200). The shipped contract is a single honest ErrorItem, not a - # silent empty. This stage verifies that graceful degradation, and passes if - # EITHER real videos come back (session got trusted) OR an ErrorItem does. + # fetch_item_list runs headful, so we expect real videos; still accept an + # ErrorItem (never a silent empty) to keep the graceful-degradation contract. items = await scrape_tiktok( TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) From 678450341454801f4f53550f83d44201d82f089a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:27 +0200 Subject: [PATCH 084/160] feat(crawl): add CRAWL_HEADED_XVFB_ENABLED flag Single switch promising an Xvfb display so the stealth browser can run headful (TikTok's profile feed is empty to headless Chromium). Defaults FALSE. --- surfsense_backend/app/config/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 85a913989..1a4d5efe7 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1147,6 +1147,12 @@ class Config: # round-trip => default FALSE to honor the "no speed regression" bar; flip on # when leak-safety outweighs the marginal latency. CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" + # Promises an Xvfb display so the browser can run headful (TikTok's profile + # feed is empty to headless Chromium). Entrypoint starts Xvfb when TRUE; + # FALSE keeps every browser headless. + CRAWL_HEADED_XVFB_ENABLED = ( + os.getenv("CRAWL_HEADED_XVFB_ENABLED", "FALSE").upper() == "TRUE" + ) # Litellm TTS Configuration TTS_SERVICE = os.getenv("TTS_SERVICE") From a551dd7553591ccd65487892b3290be0848d86ed Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:36 +0200 Subject: [PATCH 085/160] feat(docker): start Xvfb when CRAWL_HEADED_XVFB_ENABLED is set For every browser-running role (api/worker/all, not migrate) the entrypoint brings up a virtual display and exports DISPLAY, registering Xvfb for graceful shutdown. Off keeps browsers headless. --- .../scripts/docker/entrypoint.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/surfsense_backend/scripts/docker/entrypoint.sh b/surfsense_backend/scripts/docker/entrypoint.sh index 2efd78f94..4e46d184a 100644 --- a/surfsense_backend/scripts/docker/entrypoint.sh +++ b/surfsense_backend/scripts/docker/entrypoint.sh @@ -121,7 +121,28 @@ start_beat() { echo " Celery Beat PID=${PIDS[-1]}" } +# ── Headful browser display ────────────────────────────────── +# Give the stealth browser a virtual display so it can run headful (TikTok's +# profile feed is empty to headless Chromium). Off => every browser stays headless. +start_xvfb() { + if [ "$(echo "${CRAWL_HEADED_XVFB_ENABLED:-false}" | tr '[:upper:]' '[:lower:]')" != "true" ]; then + return + fi + local display="${XVFB_DISPLAY:-:99}" + echo "Starting Xvfb on ${display} (CRAWL_HEADED_XVFB_ENABLED=true)..." + Xvfb "${display}" -screen 0 1920x1080x24 -nolisten tcp >/dev/null 2>&1 & + PIDS+=($!) + export DISPLAY="${display}" + sleep 1 # let the X server accept connections before a browser starts + echo " Xvfb PID=${PIDS[-1]} DISPLAY=${DISPLAY}" +} + # ── Main: run based on role ────────────────────────────────── +# migrate never launches a browser; every other role might, so start the display. +if [ "${SERVICE_ROLE}" != "migrate" ]; then + start_xvfb +fi + case "${SERVICE_ROLE}" in migrate) run_migrations From 6400dc5f0479d14c3681c7b90849a53ba0f8e5d9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:45 +0200 Subject: [PATCH 086/160] fix(tiktok): gate headful profile feed on CRAWL_HEADED_XVFB_ENABLED fetch_item_list goes headful only when the flag promises a display; otherwise it stays headless so the browser launch never fails and the empty feed degrades to an ErrorItem. Verified under xvfb-run: real videos returned on a clean proxy IP, graceful degrade on a flagged one. --- .../proprietary/platforms/tiktok/session/listing.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index c1f3dfc15..dbdf83768 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -11,9 +11,9 @@ The pure response-shape parsing lives in :func:`items_from_response`; this modul is the untested browser-I/O glue (covered by the e2e smoke, not unit tests). Needs a residential proxy; datacenter IPs get empty bodies and 429s. The profile -feed (``/api/post/item_list``) returns an empty 200 to headless sessions, so -:func:`fetch_item_list` runs headful. ponytail: headful needs a display — run it -under Xvfb on a headless server. +feed returns an empty 200 to headless sessions, so :func:`fetch_item_list` goes +headful only when ``CRAWL_HEADED_XVFB_ENABLED`` promises an Xvfb display — else it +stays headless and degrades to an ``ErrorItem`` instead of crashing on launch. """ from __future__ import annotations @@ -25,6 +25,7 @@ from typing import Any from scrapling.fetchers import StealthyFetcher +from app.config import config from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, @@ -250,7 +251,8 @@ def _fetch_sync( async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, Any]]: """Return up to ``target_count`` itemStructs from a listing page's XHRs. - Runs headful: the profile feed returns an empty 200 to headless sessions. + Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed + is empty to headless sessions); headless otherwise so launch never fails. """ return await asyncio.to_thread( _fetch_sync, @@ -259,7 +261,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An _ITEM_LIST_MARKERS, items_from_response, _scroll_page, - headless=False, + headless=not config.CRAWL_HEADED_XVFB_ENABLED, ) From d3a6e2cd45f864766b5bd9c997477283e5a335bd Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:53 +0200 Subject: [PATCH 087/160] docs(docker): note entrypoint wires xvfb at runtime The xvfb runtime path is now implemented, so drop the stale 'deferred to Slice B' note. --- surfsense_backend/Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index fffe2e45d..7140f8809 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -31,11 +31,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ dos2unix \ git \ # ── Phase 3e stealth hardening ────────────────────────────────────────── - # Xvfb: virtual framebuffer so the stealth browser can run headful - # (headless=False) without a real display — many WAFs flag headless Chromium. - # Gated at runtime by CRAWL_HEADED_XVFB_ENABLED (Slice B); installed now so - # the image is ready. Real font packages make canvas/emoji/font-enumeration - # fingerprints resemble a real desktop (set proven against Kasada/Akamai). + # Xvfb: virtual display so the browser can run headful without real hardware + # (entrypoint starts it when CRAWL_HEADED_XVFB_ENABLED; used by TikTok's profile + # feed). Real fonts make canvas/emoji/font fingerprints look like a real desktop. xvfb \ fonts-noto-color-emoji \ fonts-unifont \ From c29b7f4b6bfdd20442d1f3b9292557ea8cbc752a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:53 +0200 Subject: [PATCH 088/160] docs(env): document CRAWL_HEADED_XVFB_ENABLED --- surfsense_backend/.env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index cd250025c..c0e9ff5bd 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -401,6 +401,9 @@ TURNSTILE_SECRET_KEY= # Route DNS via Cloudflare DoH (anti DNS-leak). Adds a DNS round-trip => off by # default to avoid any speed regression; enable for leak-safety-first setups. # CRAWL_DNS_OVER_HTTPS=FALSE +# Run the browser headful on Xvfb — required for TikTok's profile video feed +# (empty to headless Chromium). Entrypoint starts Xvfb when TRUE. +# CRAWL_HEADED_XVFB_ENABLED=FALSE # File Parser Service ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING From 493579913ecab766dc626be785a542fd53bf3319 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:17:53 +0200 Subject: [PATCH 089/160] docs(tiktok): note stage 2 profile feed is headful only when flag is set --- surfsense_backend/scripts/e2e_tiktok_scrape.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index ad3337ef2..33a5d8e57 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -7,8 +7,8 @@ Run from the backend directory: What it exercises (everything REAL — live network, live proxy, live browser): Stage 1 — proxy egress proof (informational). - Stage 2 — profile via the full pipeline: asserts real videos come back - (fetch_item_list runs headful), degrading to one ErrorItem if not. + Stage 2 — profile via the full pipeline: real videos when headful + (CRAWL_HEADED_XVFB_ENABLED), else one ErrorItem — never silent empty. Stage 3 — blob video path over HTTP (URL taken from a captured hashtag struct). Stage 4 — hashtag listing via the stealth browser (captures item_list XHRs). Stage 5 — full scrape_tiktok() pipeline on a hashtag. @@ -106,8 +106,8 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - # fetch_item_list runs headful, so we expect real videos; still accept an - # ErrorItem (never a silent empty) to keep the graceful-degradation contract. + # Headful (flag on) yields real videos; still accept an ErrorItem (never a + # silent empty) so the graceful-degradation contract holds either way. items = await scrape_tiktok( TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) From 15fb50c83e68751daee1aede8d4130e01b9a25c5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 090/160] feat(tiktok): add TIKTOK_LISTING_MAX_ATTEMPTS knob Bounds the browser-listing retry-on-empty; default 3, set 1 for a static IP. --- surfsense_backend/app/config/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 1a4d5efe7..528f6cc9d 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -728,6 +728,12 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Browser-listing retries when a feed comes back empty. The profile feed is + # withheld from flagged exit IPs; since the proxy rotates per request, each + # retry lands a fresh IP, turning a bad first draw into a hit. Only spends on + # empty results. ponytail: assumes a rotating proxy — set to 1 for a static IP, + # where retrying just re-hits the same (flagged) exit. + TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. From c8fcf87685c0e7522dd6b38ea4081bcb61b91f01 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 091/160] fix(tiktok): retry empty item_list captures on a fresh exit IP The profile feed is withheld from flagged IPs and the proxy rotates per request, so re-fetching an empty capture (up to TIKTOK_LISTING_MAX_ATTEMPTS) turns a bad first draw into a hit instead of an ErrorItem. --- .../platforms/tiktok/session/listing.py | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index dbdf83768..5668c497a 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -253,16 +253,32 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed is empty to headless sessions); headless otherwise so launch never fails. + + Retries on an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``: the feed is + withheld from flagged exit IPs, and the proxy rotates per request, so each retry + is a fresh IP — turning a bad first draw into a hit instead of an ``ErrorItem``. """ - return await asyncio.to_thread( - _fetch_sync, - page_url, - target_count, - _ITEM_LIST_MARKERS, - items_from_response, - _scroll_page, - headless=not config.CRAWL_HEADED_XVFB_ENABLED, - ) + headless = not config.CRAWL_HEADED_XVFB_ENABLED + attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS) + for attempt in range(1, attempts + 1): + items = await asyncio.to_thread( + _fetch_sync, + page_url, + target_count, + _ITEM_LIST_MARKERS, + items_from_response, + _scroll_page, + headless=headless, + ) + if items or attempt == attempts: + return items + logger.info( + "[tiktok] empty item_list for %s (attempt %d/%d); retrying on a fresh exit IP", + page_url, + attempt, + attempts, + ) + return [] async def fetch_user_search(page_url: str, target_count: int) -> list[dict[str, Any]]: From 8f825b7b3c262aa302fb3155d125a761f618069f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 092/160] test(tiktok): cover item_list retry-on-empty loop --- .../platforms/tiktok/test_listing_retry.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py b/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py new file mode 100644 index 000000000..2676a2267 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_listing_retry.py @@ -0,0 +1,60 @@ +"""Retry-on-empty for the browser ``item_list`` seam (no browser, fake fetch). + +``fetch_item_list`` re-fetches an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` +so a flagged rotating exit IP on the first draw doesn't collapse straight to an +``ErrorItem``. These drive that loop deterministically by faking ``_fetch_sync``. +""" + +from __future__ import annotations + +from app.proprietary.platforms.tiktok.session import listing + + +class _Fake: + """Returns each queued result once, repeating the last; counts calls.""" + + def __init__(self, results: list[list[dict]]): + self.results = results + self.calls = 0 + + def __call__(self, *args, **kwargs): + out = self.results[min(self.calls, len(self.results) - 1)] + self.calls += 1 + return out + + +def _patch(monkeypatch, fake: _Fake, attempts: int) -> None: + monkeypatch.setattr(listing, "_fetch_sync", fake) + monkeypatch.setattr(listing.config, "TIKTOK_LISTING_MAX_ATTEMPTS", attempts) + + +async def test_returns_first_nonempty_without_retrying(monkeypatch): + fake = _Fake([[{"id": "1"}]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [{"id": "1"}] + assert fake.calls == 1 # a draw with items never retries + + +async def test_retries_past_empty_draws_then_hits(monkeypatch): + fake = _Fake([[], [], [{"id": "9"}]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [{"id": "9"}] + assert fake.calls == 3 # two empty (flagged-IP) draws retried, third lands + + +async def test_stops_at_attempt_ceiling_when_always_empty(monkeypatch): + fake = _Fake([[]]) + _patch(monkeypatch, fake, 3) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [] + assert fake.calls == 3 # capped; caller then emits the ErrorItem + + +async def test_single_attempt_config_disables_retry(monkeypatch): + fake = _Fake([[]]) + _patch(monkeypatch, fake, 1) + items = await listing.fetch_item_list("https://tt/@x", 5) + assert items == [] + assert fake.calls == 1 # static-IP setups opt out via attempts=1 From 97e250be4b1cdeed31d1ae4b5fc3708e64e0eaf7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:25:48 +0200 Subject: [PATCH 093/160] docs(env): document TIKTOK_LISTING_MAX_ATTEMPTS --- surfsense_backend/.env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index c0e9ff5bd..d94acd14e 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -293,6 +293,9 @@ MICROS_PER_PAGE=1000 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# Browser-listing retries when a feed is empty (profile feed is withheld from +# flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. +# TIKTOK_LISTING_MAX_ATTEMPTS=3 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 From 01bc3f8de3009f492271f657453fbe5155c0ac84 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:28:47 +0200 Subject: [PATCH 094/160] docs(config): trim narration from xvfb + listing-retry comments --- surfsense_backend/app/config/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 528f6cc9d..00835dc65 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -728,11 +728,8 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) - # Browser-listing retries when a feed comes back empty. The profile feed is - # withheld from flagged exit IPs; since the proxy rotates per request, each - # retry lands a fresh IP, turning a bad first draw into a hit. Only spends on - # empty results. ponytail: assumes a rotating proxy — set to 1 for a static IP, - # where retrying just re-hits the same (flagged) exit. + # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static + # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service @@ -1154,8 +1151,7 @@ class Config: # when leak-safety outweighs the marginal latency. CRAWL_DNS_OVER_HTTPS = os.getenv("CRAWL_DNS_OVER_HTTPS", "FALSE").upper() == "TRUE" # Promises an Xvfb display so the browser can run headful (TikTok's profile - # feed is empty to headless Chromium). Entrypoint starts Xvfb when TRUE; - # FALSE keeps every browser headless. + # feed is empty to headless Chromium). Off keeps every browser headless. CRAWL_HEADED_XVFB_ENABLED = ( os.getenv("CRAWL_HEADED_XVFB_ENABLED", "FALSE").upper() == "TRUE" ) From f74a73efdbd03ad8512112cfbda66f259fdf80f8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:28:47 +0200 Subject: [PATCH 095/160] docs(tiktok): trim fetch_item_list retry docstring to intent --- .../app/proprietary/platforms/tiktok/session/listing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py index 5668c497a..bc8af56ff 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/session/listing.py @@ -253,10 +253,7 @@ async def fetch_item_list(page_url: str, target_count: int) -> list[dict[str, An Headful when ``CRAWL_HEADED_XVFB_ENABLED`` promises a display (the profile feed is empty to headless sessions); headless otherwise so launch never fails. - - Retries on an empty capture up to ``TIKTOK_LISTING_MAX_ATTEMPTS``: the feed is - withheld from flagged exit IPs, and the proxy rotates per request, so each retry - is a fresh IP — turning a bad first draw into a hit instead of an ``ErrorItem``. + Retries an empty draw up to ``TIKTOK_LISTING_MAX_ATTEMPTS`` for a fresh exit IP. """ headless = not config.CRAWL_HEADED_XVFB_ENABLED attempts = max(1, config.TIKTOK_LISTING_MAX_ATTEMPTS) From 9793751b44fa5fa2ff7dd6365c2e83a1e4623ec2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:28:47 +0200 Subject: [PATCH 096/160] docs(tiktok): trim stage 2 comment to the why --- surfsense_backend/scripts/e2e_tiktok_scrape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/scripts/e2e_tiktok_scrape.py b/surfsense_backend/scripts/e2e_tiktok_scrape.py index 33a5d8e57..b399ca3fb 100644 --- a/surfsense_backend/scripts/e2e_tiktok_scrape.py +++ b/surfsense_backend/scripts/e2e_tiktok_scrape.py @@ -106,8 +106,8 @@ async def stage_profile_listing() -> tuple[bool, list[dict[str, Any]]]: _hr(f"STAGE 2 — profile listing graceful-degrade: @{_PROFILE}") from app.proprietary.platforms.tiktok import TikTokScrapeInput, scrape_tiktok - # Headful (flag on) yields real videos; still accept an ErrorItem (never a - # silent empty) so the graceful-degradation contract holds either way. + # Accept videos OR an ErrorItem (never a silent empty): the feed degrades + # gracefully when the exit IP is flagged. items = await scrape_tiktok( TikTokScrapeInput(profiles=[_PROFILE], resultsPerPage=_COUNT), limit=_COUNT ) From cd82f861dec7c6252bd01ebecf9e8830488f29eb Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:40:04 +0200 Subject: [PATCH 097/160] fix(tiktok-agent): stop steering the subagent to keyword search for videos The playbook listed search_queries as a video source in three places, then contradicted itself. Replace with action-oriented guidance: videos via hashtags/URLs, accounts via user_search. Drop 'login-walled' (the agent has no login path). --- .../subagents/builtins/tiktok/system_prompt.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md index 5b6de993c..03114cd12 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -6,7 +6,7 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- `tiktok_scrape` — videos from urls / profiles / hashtags / search_queries +- `tiktok_scrape` — videos from a hashtag, a profile, or a TikTok URL - `tiktok_comments` — a video's comment thread, from `video_urls` - `tiktok_user_search` — find accounts by keyword, from `queries` - `tiktok_trending` — the current Explore trending-video feed @@ -14,15 +14,15 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#') and/or `search_queries`. +- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. -- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags`, `search_queries`, or a direct video URL for videos. +- Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos. - Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. -- Finding accounts (not videos): call `tiktok_user_search` with `queries` — this is the reliable path for account discovery (keyword *video* search is login-walled). +- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL. - "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. - Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). - Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. -- Batch multiple hashtags, search terms, queries, or video URLs into one call rather than many single-item calls. +- Batch multiple hashtags, queries, or video URLs into one call rather than many single-item calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, count changes). From 1911b783792e8c71c1df3e01f7d0354ab34220b7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:40:04 +0200 Subject: [PATCH 098/160] fix(tiktok-agent): drop keyword search from the routing description's video sources --- .../multi_agent_chat/subagents/builtins/tiktok/description.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md index d57c551d4..750979348 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/description.md @@ -1,2 +1,2 @@ -TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a search query, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. +TikTok specialist: pulls structured public TikTok data — videos (caption/text, author, play/like/comment/share counts, music, hashtags, timestamps, web URL) from a hashtag feed, a creator profile, or a known video URL; a video's public comment thread; accounts found by keyword; and the current Explore trending-video feed. Also compares fresh TikTok results against earlier findings in this chat. Use whenever the task is to find what is trending or being said on TikTok about a topic, gather a creator's or hashtag's videos, read a video's comments, discover accounts by keyword, or scrape a specific video URL. Triggers include "search TikTok for X", "what's trending on TikTok", "videos with #X", "comments on this TikTok", "find TikTok accounts about X", and "scrape this TikTok video". Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), Reddit (use the Reddit specialist), or YouTube (use the YouTube specialist). From 6a0ff1495fbcad084e161e6371b9f3ae7eb9a644 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:40:04 +0200 Subject: [PATCH 099/160] fix(tiktok-mcp): flag that search_queries returns no videos in the scrape tool --- .../features/scrapers/platforms/tiktok.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py index aac8efc0e..c295ec3b6 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -21,7 +21,7 @@ def register( @mcp.tool( name="surfsense_tiktok_scrape", - title="Search or scrape TikTok", + title="Scrape TikTok videos", annotations=SCRAPE, structured_output=False, ) @@ -51,7 +51,11 @@ def register( ] = None, search_queries: Annotated[ list[str] | None, - Field(description="Terms to search TikTok for, e.g. ['cooking']."), + Field( + description="Keyword search terms. Returns no videos — use " + "hashtags/profiles/urls for videos, or the user-search tool for " + "accounts." + ), ] = None, results_per_page: Annotated[ int, @@ -63,12 +67,13 @@ def register( workspace: WorkspaceParam = None, response_format: ResponseFormatParam = "markdown", ) -> str: - """Search or scrape public TikTok videos. + """Scrape public TikTok videos by hashtag, profile, or URL. - Use this for ANY TikTok research — a creator's videos, a hashtag feed, - a search, or a specific video URL — instead of a generic web search. - Returns videos with text, author, stats, music, and the web URL. - Example: hashtags=['food'], max_items=20. + Use for TikTok video research — a creator's videos, a hashtag feed, or a + specific video/profile/hashtag URL — instead of a generic web search. + Returns videos with text, author, stats, music, and the web URL. For + accounts by keyword use the user-search tool; keyword search returns no + videos. Example: hashtags=['food'], max_items=20. """ return await run_scraper( client, From 903502296ef0f084aebe6c7639d4ea1d75c8359b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 10 Jul 2026 17:46:08 +0200 Subject: [PATCH 100/160] fix(tiktok-marketing): stop advertising keyword search as a video source The hero, intro, use case, API-reference fields, and two FAQs all listed search as a video path, contradicting the FAQ that admits keyword video search is login-walled. Reconcile every claim to one truth: videos via hashtag/profile/URL; keyword search returns no videos; keyword -> user_search for accounts. --- .../lib/connectors-marketing/tiktok.tsx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index f23b73865..d77865633 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -8,7 +8,7 @@ export const tiktok: ConnectorPageContent = { metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", metaDescription: - "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, search, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, profile, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", keywords: [ "tiktok scraper", "tiktok scraper api", @@ -29,7 +29,7 @@ export const tiktok: ConnectorPageContent = { h1: "TikTok Scraper API for Trend and Creator Research", heroLede: - "The SurfSense TikTok API extracts public videos by hashtag, search, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", + "The SurfSense TikTok API extracts public videos by hashtag, creator profile, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", transcript: { prompt: "Find trending TikToks about meal prep this week", @@ -55,7 +55,7 @@ export const tiktok: ConnectorPageContent = { }, extractIntro: - "Every call returns structured items. Scrape videos from a hashtag, search query, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", + "Every call returns structured items. Scrape videos from a hashtag, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", extractFields: [ { label: "Videos", @@ -89,7 +89,7 @@ export const tiktok: ConnectorPageContent = { { title: "Trend and hashtag monitoring", description: - "Track a hashtag or search term and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", + "Track a hashtag and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", }, { title: "Creator and influencer discovery", @@ -161,7 +161,7 @@ export const tiktok: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "TikTok URLs to scrape: a video, a profile (/@user), a hashtag (/tag/name), or a search URL. Max 20.", + "TikTok URLs to scrape: a video, a profile (/@user), or a hashtag (/tag/name). Max 20.", }, { name: "profiles", @@ -180,13 +180,13 @@ export const tiktok: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "Search terms to run on TikTok. Each returns up to results_per_page videos. Max 20.", + "Keyword search terms. Keyword video search is login-walled and returns no videos — use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", }, { name: "results_per_page", type: "integer", defaultValue: "10", - description: "Max videos to pull per profile, hashtag, or search target. 1 to 100.", + description: "Max videos to pull per profile or hashtag target. 1 to 100.", }, { name: "max_items", @@ -260,17 +260,17 @@ export const tiktok: ConnectorPageContent = { { question: "What are the rate limits?", answer: - "Each call caps at 100 returned videos across all sources, with up to 20 hashtags, searches, profiles, or URLs per request. SurfSense manages the underlying request budget and request signing for you.", + "Each call caps at 100 returned videos across all sources, with up to 20 hashtags, profiles, or URLs per request. SurfSense manages the underlying request budget and request signing for you.", }, { question: "Can I scrape a specific creator's videos?", answer: - "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag, by search query, or by a direct video URL.", + "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag or by a direct video URL.", }, { question: "What TikTok data can I scrape?", answer: - "Four verbs: scrape (videos by hashtag, search, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", }, ], From 8afa4c6fc6a6caec0f32f3bca6add6c696aa1b1b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:39 +0530 Subject: [PATCH 101/160] refactor(instagram): remove unused comments capability --- .../app/capabilities/instagram/__init__.py | 1 - .../instagram/comments/__init__.py | 3 - .../instagram/comments/definition.py | 22 ------- .../instagram/comments/executor.py | 53 ---------------- .../instagram/comments/schemas.py | 61 ------------------- 5 files changed, 140 deletions(-) delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/__init__.py delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/definition.py delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/executor.py delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/__init__.py b/surfsense_backend/app/capabilities/instagram/__init__.py index 4004f86c7..17866e4b5 100644 --- a/surfsense_backend/app/capabilities/instagram/__init__.py +++ b/surfsense_backend/app/capabilities/instagram/__init__.py @@ -2,6 +2,5 @@ from __future__ import annotations -from app.capabilities.instagram.comments import definition as _comments # noqa: F401 from app.capabilities.instagram.details import definition as _details # noqa: F401 from app.capabilities.instagram.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/instagram/comments/__init__.py b/surfsense_backend/app/capabilities/instagram/comments/__init__.py deleted file mode 100644 index 84899d758..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""``instagram.comments`` verb: post/reel URLs → comments (and replies).""" - -from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/comments/definition.py b/surfsense_backend/app/capabilities/instagram/comments/definition.py deleted file mode 100644 index 7794228ac..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/definition.py +++ /dev/null @@ -1,22 +0,0 @@ -"""``instagram.comments`` capability registration (billed per comment; see config -``INSTAGRAM_SCRAPE_MICROS_PER_COMMENT``).""" - -from __future__ import annotations - -from app.capabilities.core import BillingUnit, Capability, register_capability -from app.capabilities.instagram.comments.executor import build_comments_executor -from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput - -INSTAGRAM_COMMENTS = Capability( - name="instagram.comments", - description=( - "Fetch comments (and optionally replies) for Instagram post or reel URLs." - ), - input_schema=CommentsInput, - output_schema=CommentsOutput, - executor=build_comments_executor(), - billing_unit=BillingUnit.INSTAGRAM_COMMENT, - docs_url="/docs/connectors/native/instagram", -) - -register_capability(INSTAGRAM_COMMENTS) diff --git a/surfsense_backend/app/capabilities/instagram/comments/executor.py b/surfsense_backend/app/capabilities/instagram/comments/executor.py deleted file mode 100644 index 260ee06c9..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/executor.py +++ /dev/null @@ -1,53 +0,0 @@ -"""``instagram.comments`` executor: verb input → scraper → comment items.""" - -from __future__ import annotations - -from collections.abc import Awaitable, Callable - -from app.capabilities.core import Executor -from app.capabilities.core.progress import emit_progress -from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput -from app.exceptions import ForbiddenError -from app.proprietary.platforms.instagram import ( - InstagramAccessBlockedError, - InstagramScrapeInput, - scrape_instagram, -) - -ScrapeFn = Callable[..., Awaitable[list[dict]]] - - -def build_comments_executor(scrape_fn: ScrapeFn | None = None) -> Executor: - """Bind the executor to a scraper fn (defaults to the proprietary actor).""" - scrape_fn = scrape_fn or scrape_instagram - - async def execute(payload: CommentsInput) -> CommentsOutput: - actor_input = InstagramScrapeInput( - resultsType="comments", - directUrls=payload.urls, - resultsLimit=payload.max_comments_per_post, - isNewestComments=payload.newest_first, - includeNestedComments=payload.include_replies, - ) - emit_progress( - "starting", - "Fetching Instagram comments", - total=payload.max_items, - unit="comment", - ) - try: - items = await scrape_fn(actor_input, limit=payload.max_items) - except InstagramAccessBlockedError as exc: - raise ForbiddenError( - f"Instagram refused anonymous access: {exc}", - code="INSTAGRAM_ACCESS_BLOCKED", - ) from exc - emit_progress( - "done", - f"Scraped {len(items)} comment(s)", - current=len(items), - unit="comment", - ) - return CommentsOutput(items=items) - - return execute diff --git a/surfsense_backend/app/capabilities/instagram/comments/schemas.py b/surfsense_backend/app/capabilities/instagram/comments/schemas.py deleted file mode 100644 index fadb83215..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/schemas.py +++ /dev/null @@ -1,61 +0,0 @@ -"""``instagram.comments`` I/O contracts. - -A lean surface over ``InstagramScrapeInput`` (``resultsType="comments"``). The -scraper's ``InstagramComment`` is reused verbatim as the output element. -""" - -from __future__ import annotations - -from pydantic import BaseModel, Field - -from app.capabilities.instagram.scrape.schemas import ( - MAX_INSTAGRAM_ITEMS, - MAX_INSTAGRAM_SOURCES, -) -from app.proprietary.platforms.instagram import InstagramComment - -MAX_COMMENTS_PER_POST = 50 -"""Anonymous web media pages surface at most ~50 comments per post.""" - - -class CommentsInput(BaseModel): - urls: list[str] = Field( - min_length=1, - max_length=MAX_INSTAGRAM_SOURCES, - description="Post or reel URLs to fetch comments for (shortCode or numeric-ID forms).", - ) - newest_first: bool = Field( - default=False, - description="Return newest comments first.", - ) - include_replies: bool = Field( - default=False, - description="Include nested replies; each reply is a separate billable item.", - ) - max_comments_per_post: int = Field( - default=10, - ge=1, - le=MAX_COMMENTS_PER_POST, - description="Max comments per post (Instagram caps at 50).", - ) - max_items: int = Field( - default=10, - ge=1, - le=MAX_INSTAGRAM_ITEMS, - description="Max total comments to return across all posts.", - ) - - @property - def estimated_units(self) -> int: - return self.max_items - - -class CommentsOutput(BaseModel): - items: list[InstagramComment] = Field( - default_factory=list, - description="One item per comment (or reply), in emission order.", - ) - - @property - def billable_units(self) -> int: - return len(self.items) From de1990f9f6dc0cf2aaae3bfb8baa05fd46af4585 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:46 +0530 Subject: [PATCH 102/160] refactor(instagram): simplify scrape and details capability schemas --- .../capabilities/instagram/details/schemas.py | 31 +++++++------------ .../instagram/scrape/definition.py | 4 +-- .../capabilities/instagram/scrape/schemas.py | 14 ++++----- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/surfsense_backend/app/capabilities/instagram/details/schemas.py b/surfsense_backend/app/capabilities/instagram/details/schemas.py index 31972b095..1372f881f 100644 --- a/surfsense_backend/app/capabilities/instagram/details/schemas.py +++ b/surfsense_backend/app/capabilities/instagram/details/schemas.py @@ -1,13 +1,14 @@ """``instagram.details`` I/O contracts. A lean surface over ``InstagramScrapeInput`` (``resultsType="details"``). Each -output item is a profile / hashtag / place, discriminated by the synthesized -``detailKind`` field (a SurfSense addition; every other field mirrors the actor). +output item is a profile (``detailKind="profile"``, a SurfSense addition; every +other field mirrors the actor). Hashtag/place details are login-walled and +therefore unsupported. """ from __future__ import annotations -from typing import Annotated, Literal +from typing import Literal from pydantic import BaseModel, Field, model_validator @@ -15,16 +16,9 @@ from app.capabilities.instagram.scrape.schemas import ( MAX_INSTAGRAM_ITEMS, MAX_INSTAGRAM_SOURCES, ) -from app.proprietary.platforms.instagram import ( - InstagramHashtag, - InstagramPlace, - InstagramProfile, -) +from app.proprietary.platforms.instagram import InstagramProfile -InstagramDetailItem = Annotated[ - InstagramProfile | InstagramHashtag | InstagramPlace, - Field(discriminator="detailKind"), -] +InstagramDetailItem = InstagramProfile class DetailsInput(BaseModel): @@ -32,18 +26,17 @@ class DetailsInput(BaseModel): default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, description=( - "Profile / hashtag / place URLs (or bare profile IDs). The URL type " - "determines the detail kind. Provide these OR search_queries." + "Profile URLs or bare profile IDs. Provide these OR search_queries." ), ) search_queries: list[str] = Field( default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, - description="Discovery keywords. Provide these OR urls (never both).", + description="Discovery keywords resolved to profiles. Provide these OR urls.", ) - search_type: Literal["hashtag", "profile", "place"] = Field( - default="hashtag", - description="What to discover from search_queries (no 'user' — use instagram.scrape).", + search_type: Literal["profile", "user"] = Field( + default="profile", + description="Discovery kind (profile-only; hashtag/place are login-walled).", ) search_limit: int = Field( default=10, @@ -78,7 +71,7 @@ class DetailsInput(BaseModel): class DetailsOutput(BaseModel): items: list[InstagramDetailItem] = Field( default_factory=list, - description="One item per profile/hashtag/place, keyed by detailKind.", + description="One profile detail item per resolved profile.", ) @property diff --git a/surfsense_backend/app/capabilities/instagram/scrape/definition.py b/surfsense_backend/app/capabilities/instagram/scrape/definition.py index e84ca4938..7b5d9769f 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/definition.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/definition.py @@ -10,8 +10,8 @@ from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput INSTAGRAM_SCRAPE = Capability( name="instagram.scrape", description=( - "Scrape public Instagram posts, reels, or mentions from " - "profile/post/hashtag/place URLs, or discover content via search queries." + "Scrape public Instagram posts, reels, or mentions from profile/post/" + "reel URLs, or discover public profiles via search queries." ), input_schema=ScrapeInput, output_schema=ScrapeOutput, diff --git a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py index 0e5646dc7..03e03e55d 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py @@ -26,8 +26,8 @@ class ScrapeInput(BaseModel): default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, description=( - "Instagram URLs or bare profile IDs: profile, post (/p/), reel " - "(/reel/), hashtag (/explore/tags/), or place (/explore/locations/). " + "Instagram URLs or bare profile IDs: profile, post (/p/), or reel " + "(/reel/). Hashtag/place URLs are unsupported (login-walled). " "Provide these OR search_queries (never both)." ), ) @@ -35,13 +35,13 @@ class ScrapeInput(BaseModel): default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, description=( - "Discovery keywords (hashtags as plaintext, no '#'). Provide these " - "OR urls (never both)." + "Discovery keywords resolved to profiles via Google (IG's keyword " + "search is login-walled). Provide these OR urls (never both)." ), ) - search_type: Literal["hashtag", "profile", "place", "user"] = Field( - default="hashtag", - description="What to discover from search_queries. Only used with search_queries.", + search_type: Literal["profile", "user"] = Field( + default="profile", + description="Discovery kind (profile-only; hashtag/place are login-walled).", ) result_type: Literal["posts", "reels", "mentions"] = Field( default="posts", From 4813dc96e3433cb0e44b71af789e8db205dce5b6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:51 +0530 Subject: [PATCH 103/160] refactor(instagram): streamline scraper, fetch, and parsing pipeline --- .../platforms/instagram/__init__.py | 6 - .../proprietary/platforms/instagram/fetch.py | 69 ++++-- .../platforms/instagram/parsers.py | 210 ++++++++++++----- .../platforms/instagram/schemas.py | 63 +---- .../platforms/instagram/scraper.py | 221 +++++++----------- .../platforms/instagram/url_resolver.py | 25 +- 6 files changed, 304 insertions(+), 290 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/__init__.py b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py index e3a2a122a..2302ca9fc 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py @@ -2,10 +2,7 @@ from .fetch import InstagramAccessBlockedError from .schemas import ( - InstagramComment, - InstagramHashtag, InstagramMediaItem, - InstagramPlace, InstagramProfile, InstagramScrapeInput, ) @@ -13,10 +10,7 @@ from .scraper import iter_instagram, scrape_instagram __all__ = [ "InstagramAccessBlockedError", - "InstagramComment", - "InstagramHashtag", "InstagramMediaItem", - "InstagramPlace", "InstagramProfile", "InstagramScrapeInput", "iter_instagram", diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index 73320eddb..e2b7c5818 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -32,6 +32,7 @@ import json import logging import random import time +from collections.abc import Callable from contextlib import asynccontextmanager, suppress from contextvars import ContextVar from datetime import UTC, datetime @@ -75,12 +76,6 @@ _MAX_ROTATIONS = 3 _MAX_BACKOFFS = 4 _BACKOFF_BASE_S = 5.0 -# Endpoints Instagram serves only to logged-in clients (confirmed live). A bare -# 401/403 here is an endpoint auth wall, not a per-IP block, so every rotated IP -# hits the same wall — fail fast instead of burning the pool, exactly like the -# /accounts/login/ redirect branch. Content endpoints (profiles) still rotate. -_AUTH_WALLED_PATHS = ("web/search/topsearch/", "api/v1/tags/web_info/") - # Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't # burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits; # a pool with a stricter per-IP cap may need it raised — watch for 429 log spam. @@ -149,6 +144,20 @@ def _parse_json(page: Any) -> Any | None: return None +def _page_text(page: Any) -> str | None: + """Best-effort HTML/text body of a scrapling response, or ``None``.""" + for attr in ("text", "body", "content"): + val = getattr(page, attr, None) + if callable(val): + with suppress(Exception): + val = val() + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + return val + return None + + def _is_login_redirect(page: Any) -> bool: """True if IG redirected this request to the anonymous login wall. @@ -308,20 +317,44 @@ async def resolve_redirect(url: str) -> str | None: async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None: - """GET an Instagram web endpoint through a warmed HTTP session. + """GET an Instagram web endpoint through a warmed session; parse JSON. Returns parsed JSON (dict or list), or ``None`` on 404 / non-block failure. - Warms cookies once per session; rotates the residential IP and re-warms on - 401/403; backs off on 429. Raises :class:`InstagramAccessBlockedError` only - when every rotated IP refuses anonymous access (the login-wall branch, which - we cannot satisfy). + See :func:`_fetch` for the warm/rotate/backoff resilience contract. + """ + return await _fetch(path, params, _parse_json) + + +async def fetch_html(path: str, params: dict[str, Any] | None = None) -> str | None: + """GET an Instagram web page through a warmed session; return its HTML text. + + Same warm/rotate/backoff resilience as :func:`fetch_json` (a login-wall + redirect still raises :class:`InstagramAccessBlockedError`), but hands back + the raw HTML body for the pages that embed their data in the document + (``/p//`` og-meta / ld+json) instead of a JSON XHR endpoint. + """ + return await _fetch(path, params, _page_text) + + +async def _fetch( + path: str, + params: dict[str, Any] | None, + extract: Callable[[Any], Any | None], +) -> Any | None: + """GET an Instagram web endpoint through a warmed HTTP session. + + Applies ``extract`` to the 200 response (JSON parse or HTML text); returns + ``None`` on 404 / non-block failure. Warms cookies once per session; rotates + the residential IP and re-warms on 401/403; backs off on 429. Raises + :class:`InstagramAccessBlockedError` only when every rotated IP refuses + anonymous access (the login-wall branch, which we cannot satisfy). """ holder = _current_session.get() if holder is None: # No bound session (e.g. a direct call outside fan_out): open a # short-lived warmed session for this one fetch, then tear it down. async with proxy_session(): - return await fetch_json(path, params) + return await _fetch(path, params, extract) url = _build_url(path, params) attempt = 0 @@ -357,7 +390,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N f"Instagram login wall on {path} (endpoint requires auth)" ) if status == 200: - return _parse_json(page) + return extract(page) if status == 404: return None if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: @@ -369,13 +402,9 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N await asyncio.sleep(delay + random.uniform(0, 1)) continue if status in _ROTATE_STATUSES: - # Bare 401/403 on a login-gated endpoint: rotating never clears an - # endpoint auth wall, so fail fast (mirrors the login-redirect - # branch above). Other endpoints rotate — a per-IP 401 recovers. - if any(p in path for p in _AUTH_WALLED_PATHS): - raise InstagramAccessBlockedError( - f"Instagram login wall on {path} (endpoint requires auth)" - ) + # Bare 401/403: a per-IP block that a fresh exit IP recovers, so + # rotate and re-warm. (The endpoint-level auth wall is caught by + # the login-redirect branch above and fails fast without rotating.) if attempt < _MAX_ROTATIONS: attempt += 1 await holder.rotate() diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index 96d66a2c5..55884537d 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -15,6 +15,7 @@ parity is additive. from __future__ import annotations +import json import re from datetime import UTC, datetime from typing import Any @@ -136,28 +137,6 @@ def parse_media(node: dict[str, Any]) -> dict[str, Any]: } -def parse_comment(node: dict[str, Any], *, post_url: str | None) -> dict[str, Any]: - """Map a raw comment node to a flat comment item dict.""" - owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} - code = _shortcode(node) - return { - "id": node.get("id"), - "postUrl": post_url, - "commentUrl": f"{_BASE}/p/{code}/c/{node.get('id')}/" if code else None, - "text": node.get("text"), - "ownerUsername": owner.get("username"), - "ownerProfilePicUrl": owner.get("profile_pic_url"), - "timestamp": _utc_from_sec(node.get("created_at")), - "repliesCount": _edge_count(node, "edge_threaded_comments") - or _int(node.get("child_comment_count")), - "likesCount": _edge_count(node, "edge_liked_by") - or _int(node.get("comment_like_count")), - "owner": {"id": owner.get("id"), "username": owner.get("username")} - if owner - else None, - } - - def parse_profile(user: dict[str, Any]) -> dict[str, Any]: """Map a raw ``web_profile_info`` ``data.user`` to a flat profile item dict.""" username = user.get("username") @@ -185,39 +164,162 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]: } -def parse_hashtag(data: dict[str, Any]) -> dict[str, Any]: - """Map a raw ``tags/web_info`` payload to a flat hashtag item dict.""" - node = data.get("data") if isinstance(data.get("data"), dict) else data - name = node.get("name") - top = _edges(node.get("edge_hashtag_to_top_posts")) - recent = _edges(node.get("edge_hashtag_to_media")) - return { - "detailKind": "hashtag", - "id": node.get("id"), - "name": name, - "url": f"{_BASE}/explore/tags/{name}/" if name else None, - "postsCount": _edge_count(node, "edge_hashtag_to_media"), - "topPosts": [parse_media(n) for n in top], - "posts": [parse_media(n) for n in recent], - } +# Anonymous single-post extraction (/p//, /reel//) -------- +# +# Instagram serves logged-out visitors the post's public metadata inside the +# document itself, not via a JSON XHR (the ``?__a=1`` API 404s / login-walls for +# anonymous callers). Two durable, anonymous surfaces carry it: +# 1. ``', + re.IGNORECASE | re.DOTALL, +) +_OG_RE = re.compile( + r' dict[str, Any]: - """Map a raw ``locations/web_info`` payload to a flat place item dict.""" - loc = data.get("location") if isinstance(data.get("location"), dict) else data - recent = _edges(loc.get("edge_location_to_media")) +def _html_int(value: Any) -> int | None: + """Coerce a string/number (``"1,234"``) to int, or ``None``.""" + if isinstance(value, bool): + return None + if isinstance(value, int | float): + return int(value) + if isinstance(value, str): + digits = value.replace(",", "").strip() + if digits.isdigit(): + return int(digits) + return None + + +def _ldjson_blocks(html: str) -> list[dict[str, Any]]: + """Parse every ``application/ld+json`` script block into dicts.""" + out: list[dict[str, Any]] = [] + for raw in _LDJSON_RE.findall(html): + try: + data = json.loads(raw.strip()) + except (ValueError, TypeError): + continue + for node in data if isinstance(data, list) else [data]: + if isinstance(node, dict): + out.append(node) + return out + + +def _og_tags(html: str) -> dict[str, str]: + """Map ``og:`` -> content for the post document.""" + return {k.lower(): v for k, v in _OG_RE.findall(html)} + + +def _ld_interaction(node: dict[str, Any]) -> dict[str, int]: + """Pull like/comment counts out of schema.org ``interactionStatistic``.""" + stats = node.get("interactionStatistic") + items = stats if isinstance(stats, list) else [stats] if stats else [] + out: dict[str, int] = {} + for stat in items: + if not isinstance(stat, dict): + continue + itype = str(stat.get("interactionType") or "") + count = _html_int(stat.get("userInteractionCount")) + if count is None: + continue + if "Like" in itype: + out["likes"] = count + elif "Comment" in itype: + out["comments"] = count + return out + + +def _ld_author_username(node: dict[str, Any]) -> str | None: + """Owner handle from a schema.org ``author`` (alternateName / identifier).""" + author = node.get("author") + author = author[0] if isinstance(author, list) and author else author + if not isinstance(author, dict): + return None + for key in ("alternateName", "identifier", "name"): + val = author.get(key) + if isinstance(val, dict): + val = val.get("value") + if isinstance(val, str) and val.strip(): + return val.strip().lstrip("@") or None + return None + + +def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> dict[str, Any] | None: + """Map an anonymous ``/p//`` (or ``/reel/``) HTML page to a media dict. + + Prefers the embedded schema.org ``ld+json`` block, falling back to Open Graph + meta tags for whatever it omits. Returns a dict shaped like + :func:`parse_media` (so it flows through ``InstagramMediaItem`` unchanged), or + ``None`` when the document carries neither surface (e.g. a login interstitial + slipped past the fetch-layer redirect check — the caller treats ``None`` as + "nothing to emit", never a silent success). + """ + if not isinstance(html, str) or not html.strip(): + return None + blocks = _ldjson_blocks(html) + og = _og_tags(html) + if not blocks and not og: + return None + + node = next( + (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), + blocks[0] if blocks else {}, + ) + counts = _ld_interaction(node) + if "likes" not in counts or "comments" not in counts: + m = _OG_COUNTS_RE.search(og.get("description", "")) + if m: + counts.setdefault("likes", _html_int(m.group(1)) or 0) + counts.setdefault("comments", _html_int(m.group(2)) or 0) + + caption = ( + node.get("articleBody") + or node.get("caption") + or node.get("description") + or og.get("description") + ) + caption = caption if isinstance(caption, str) else None + + video = node.get("video") + video = video[0] if isinstance(video, list) and video else video + video_url = ( + video.get("contentUrl") if isinstance(video, dict) else None + ) or og.get("video") + is_video = bool(video_url) or og.get("type") == "video.other" + + image = node.get("image") + image = image[0] if isinstance(image, list) and image else image + display_url = ( + image.get("url") if isinstance(image, dict) else image + if isinstance(image, str) + else None + ) or og.get("image") + return { - "detailKind": "place", - "name": loc.get("name"), - "location_id": str(loc.get("id")) if loc.get("id") is not None else None, - "slug": loc.get("slug"), - "lat": loc.get("lat"), - "lng": loc.get("lng"), - "location_address": loc.get("address_json") or loc.get("address"), - "location_city": loc.get("city"), - "phone": loc.get("phone"), - "website": loc.get("website"), - "category": loc.get("category"), - "media_count": _edge_count(loc, "edge_location_to_media"), - "posts": [parse_media(n) for n in recent], + "id": node.get("identifier") if isinstance(node.get("identifier"), str) else None, + "type": "Video" if is_video else "Image", + "shortCode": shortcode, + "caption": caption, + "hashtags": _HASHTAG_RE.findall(caption) if caption else [], + "mentions": _MENTION_RE.findall(caption) if caption else [], + "url": url, + "commentsCount": counts.get("comments"), + "displayUrl": display_url, + "videoUrl": video_url if is_video else None, + "likesCount": counts.get("likes"), + "timestamp": node.get("uploadDate") or node.get("datePublished"), + "ownerUsername": _ld_author_username(node), } diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py index 2175744a2..76ad81d12 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -19,11 +19,8 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field -InstagramResultsType = Literal[ - "posts", "details", "comments", "reels", "mentions", "stories" -] -InstagramSearchType = Literal["hashtag", "profile", "place", "user"] -InstagramDetailKind = Literal["profile", "hashtag", "place"] +InstagramResultsType = Literal["posts", "details", "reels", "mentions"] +InstagramSearchType = Literal["profile", "user"] class InstagramScrapeInput(BaseModel): @@ -41,12 +38,10 @@ class InstagramScrapeInput(BaseModel): resultsLimit: int | None = Field(default=None, ge=1) onlyPostsNewerThan: str | None = None search: str | None = None - searchType: InstagramSearchType = "hashtag" + searchType: InstagramSearchType = "profile" searchLimit: int | None = Field(default=None, ge=1, le=250) addParentData: bool = False skipPinnedPosts: bool = False - isNewestComments: bool = False - includeNestedComments: bool = False addProfileStatistics: bool = False @@ -109,22 +104,6 @@ class InstagramMediaItem(_ItemBase): dataSource: dict[str, Any] | None = None -class InstagramComment(_ItemBase): - """A comment on a post / reel.""" - - id: str | None = None - postUrl: str | None = None - commentUrl: str | None = None - text: str | None = None - ownerUsername: str | None = None - ownerProfilePicUrl: str | None = None - timestamp: str | None = None - repliesCount: int | None = None - replies: list[dict[str, Any]] = Field(default_factory=list) - likesCount: int | None = None - owner: dict[str, Any] | None = None - - class InstagramProfile(_ItemBase): """A profile detail item (``detailKind = "profile"``).""" @@ -149,39 +128,3 @@ class InstagramProfile(_ItemBase): relatedProfiles: list[dict[str, Any]] = Field(default_factory=list) latestPosts: list[dict[str, Any]] = Field(default_factory=list) statistics: dict[str, Any] | None = None - - -class InstagramHashtag(_ItemBase): - """A hashtag detail item (``detailKind = "hashtag"``).""" - - detailKind: Literal["hashtag"] = "hashtag" - id: str | None = None - name: str | None = None - url: str | None = None - postsCount: int | None = None - topPosts: list[dict[str, Any]] = Field(default_factory=list) - posts: list[dict[str, Any]] = Field(default_factory=list) - related: list[dict[str, Any]] = Field(default_factory=list) - searchTerm: str | None = None - searchSource: str | None = None - - -class InstagramPlace(_ItemBase): - """A place detail item (``detailKind = "place"``).""" - - detailKind: Literal["place"] = "place" - name: str | None = None - location_id: str | None = None - slug: str | None = None - lat: float | None = None - lng: float | None = None - location_address: str | None = None - location_city: str | None = None - location_zip: str | None = None - phone: str | None = None - website: str | None = None - category: str | None = None - media_count: int | None = None - posts: list[dict[str, Any]] = Field(default_factory=list) - searchTerm: str | None = None - searchSource: str | None = None diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index f748d5026..381b40d5b 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -10,11 +10,15 @@ stays sequential. ``fan_out`` is ported from ``../reddit/scraper.py`` but bound to *this* module's proxy holders so every worker warms its own session once and reuses it. +Anonymous-only. Every surface here is reachable without a login: profile web +info, the media embedded in a profile page, single-post/reel pages, and +Google-backed handle discovery. Login-walled surfaces (hashtag/place feeds, +comment threads, IG's native keyword search) are deliberately absent. + Flows are selected by ``resultsType``: -- ``posts`` / ``reels`` / ``mentions`` -> media items (profile / hashtag feeds, - or discovery search) -- ``comments`` -> comment items for post/reel URLs -- ``details`` -> profile / hashtag / place metadata (by URL or discovery search) +- ``posts`` / ``reels`` / ``mentions`` -> media items (profile feed, or a single + ``/p/``/``/reel/`` page, or discovery search) +- ``details`` -> profile metadata (by URL or discovery search) ponytail: deep feed pagination (past the first web page of media) needs the GraphQL cursor endpoint whose doc-id drifts; v1 emits the first page and stops. @@ -32,20 +36,18 @@ from contextlib import aclosing from datetime import UTC, datetime, timedelta from typing import Any +from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput +from app.proprietary.platforms.google_search.scraper import scrape_serps + from .fetch import ( InstagramAccessBlockedError, bind_proxy_holder, + fetch_html, fetch_json, now_iso, open_proxy_holder, ) -from .parsers import ( - parse_comment, - parse_hashtag, - parse_media, - parse_place, - parse_profile, -) +from .parsers import parse_media, parse_post, parse_profile from .schemas import InstagramScrapeInput from .url_resolver import ResolvedUrl, resolve_url @@ -62,14 +64,7 @@ __all__ = [ # residential pool with parallel login walls. _FANOUT_CONCURRENCY = 8 -# Per-post comment fetches fan across their own warm sessions; kept below the -# top-level width so N concurrent targets x this can't explode the IP count. -_COMMENT_CONCURRENCY = 4 - _PROFILE_PATH = "api/v1/users/web_profile_info/" -_HASHTAG_PATH = "api/v1/tags/web_info/" -_LOCATION_PATH = "api/v1/locations/web_info/" -_SEARCH_PATH = "web/search/topsearch/" # Instagram usernames: 1-30 chars of letters/digits/period/underscore. Used to # treat a profile/user discovery query as a direct (anonymous) handle lookup. @@ -235,7 +230,7 @@ async def _media_flow( cutoff: datetime | None, per_target: int, ) -> AsyncIterator[dict[str, Any]]: - """Emit media items for a profile / hashtag / place URL.""" + """Emit media items for a profile feed, or a single ``/p/``/``/reel/`` page.""" from .parsers import _edges result_type = input_model.resultsType @@ -258,128 +253,95 @@ async def _media_flow( if emitted >= per_target: return return - if resolved.kind == "hashtag": - data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) - if isinstance(data, dict): - parsed = parse_hashtag(data) - emitted = 0 - for node in [*parsed.get("topPosts", []), *parsed.get("posts", [])]: - if not _media_matches(node, result_type): - continue - if not _is_after(node.get("timestamp"), cutoff): - continue - yield _emit(node, input_url=resolved.url) - emitted += 1 - if emitted >= per_target: - return - return - if resolved.kind == "place": - data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) - if isinstance(data, dict): - parsed = parse_place(data) - emitted = 0 - for node in parsed.get("posts", []): - if not _is_after(node.get("timestamp"), cutoff): - continue - yield _emit(node, input_url=resolved.url) - emitted += 1 - if emitted >= per_target: - return - return - - -async def _comments_flow( - resolved: ResolvedUrl, - *, - input_model: InstagramScrapeInput, - per_target: int, -) -> AsyncIterator[dict[str, Any]]: - """Emit comment items for a post / reel URL. - - ponytail: the anonymous comment page uses a GraphQL cursor whose doc-id - drifts; v1 sources the comments embedded in the media info payload and caps - at the actor's 50/post ceiling. Deeper paging is the upgrade path in - ``fetch.py``. - """ - from .parsers import _edges - - path = f"p/{resolved.value}/" - data = await fetch_json(path, {"__a": 1, "__d": "dis"}) - node = None - if isinstance(data, dict): - items = data.get("items") - if isinstance(items, list) and items: - node = items[0] - else: - gql = data.get("graphql") - node = gql.get("shortcode_media") if isinstance(gql, dict) else None - if not isinstance(node, dict): - return - comment_nodes = _edges(node.get("edge_media_to_parent_comment")) or _edges( - node.get("edge_media_to_comment") - ) - cap = min(per_target, 50) - emitted = 0 - for cnode in comment_nodes: - item = parse_comment(cnode, post_url=resolved.url) - yield _emit(item, input_url=resolved.url) - emitted += 1 - if input_model.includeNestedComments: - for reply in _edges(cnode.get("edge_threaded_comments")): - if emitted >= cap: - return - yield _emit( - parse_comment(reply, post_url=resolved.url), - input_url=resolved.url, - ) - emitted += 1 - if emitted >= cap: + if resolved.kind in ("post", "reel"): + # Single-post extraction: the anonymous ``?__a=1`` JSON API 404s/login- + # walls, but the public /p// document embeds the post's og-meta + + # ld+json, which parse_post reads. Numeric-ID URLs can't be keyed this + # way (the page needs the shortCode), so they're skipped upstream. + if resolved.numeric_post_id: return + html = await fetch_html(f"p/{resolved.value}/") + item = parse_post(html, url=resolved.url, shortcode=resolved.value) + if item is None: + return + if not _media_matches(item, result_type): + return + if not _is_after(item.get("timestamp"), cutoff): + return + yield _emit(item, input_url=resolved.url) + return async def _details_flow( resolved: ResolvedUrl, *, input_model: InstagramScrapeInput ) -> AsyncIterator[dict[str, Any]]: - """Emit one profile / hashtag / place detail item for a URL.""" + """Emit one profile detail item for a URL (anonymous web_profile_info).""" if resolved.kind == "profile": user = await _profile_user(resolved.value) if user is not None: yield _emit(parse_profile(user), input_url=resolved.url) - return - if resolved.kind == "hashtag": - data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) - if isinstance(data, dict): - yield _emit(parse_hashtag(data), input_url=resolved.url) - return - if resolved.kind == "place": - data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) - if isinstance(data, dict): - yield _emit(parse_place(data), input_url=resolved.url) - return + + +def _kind_matches(resolved: ResolvedUrl, search_type: str) -> bool: + """True if a resolved IG URL is the kind the discovery query asked for. + + Discovery is profile-only now (hashtag/place feeds are login-walled), so + every supported ``search_type`` maps to a profile target. + """ + return resolved.kind == "profile" + + +async def _discover_via_google( + query: str, *, search_type: str, limit: int +) -> list[ResolvedUrl]: + """Discover IG profile targets via Google ``site:instagram.com`` (anonymous). + + Instagram's own keyword search is login-walled, so we reuse the existing + ``google_search`` platform, classify each organic URL with ``resolve_url``, + keep the profile hits, de-dup, and cap at ``limit``. + + Quality caveat: results reflect Google's index/ranking of instagram.com, not + IG's own relevance. This is discovery, not search parity (see README). + """ + serps = await scrape_serps( + GoogleSearchScrapeInput( + queries=query, site="instagram.com", maxPagesPerQuery=2 + ), + limit=2, + ) + resolved: list[ResolvedUrl] = [] + seen: set[tuple[str, str]] = set() + for serp in serps: + for org in serp.get("organicResults") or []: + url = org.get("url", "") if isinstance(org, dict) else "" + r = resolve_url(url) + if r is None or not _kind_matches(r, search_type): + continue + key = (r.kind, r.value) + if key in seen: + continue + seen.add(key) + resolved.append(r) + if len(resolved) >= limit: + return resolved + return resolved async def _discover( query: str, *, search_type: str, limit: int ) -> list[ResolvedUrl]: - """Resolve a discovery query into targets - anonymously. + """Resolve a discovery query into profile targets - anonymously. - Instagram's keyword search (``topsearch``) is login-walled, so we never call - it. A profile/user query is resolved as a direct handle lookup against the - anonymous profile endpoint ("messi" -> instagram.com/messi/). Hashtag/place - keyword discovery has NO anonymous endpoint (topsearch and the tag/location - feeds all require a session), so we surface a clear block instead of a - misleading empty success. + A query that is a valid handle resolves directly against the anonymous + profile endpoint ("messi" -> instagram.com/messi/). A non-handle query (e.g. + "national geographic") goes through Google ``site:instagram.com`` since IG's + native keyword search is login-walled. """ - if search_type in ("profile", "user"): - handle = query.strip().lstrip("@") - if _HANDLE_RE.match(handle): - url = f"https://www.instagram.com/{handle}/" - return [ResolvedUrl("profile", handle, url)][:limit] - return [] # not a handle, and no anonymous fuzzy search -> nothing to do - raise InstagramAccessBlockedError( - f"Instagram {search_type} search requires login and is unavailable in " - "anonymous mode - pass a profile URL or handle via directUrls instead" - ) + handle = query.strip().lstrip("@") + if _HANDLE_RE.match(handle): + url = f"https://www.instagram.com/{handle}/" + return [ResolvedUrl("profile", handle, url)][:limit] + return await _discover_via_google(query, search_type=search_type, limit=limit) def _resolve_inputs(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: @@ -425,17 +387,6 @@ async def iter_instagram( cutoff = _parse_newer_than(input_model.onlyPostsNewerThan) per_target = input_model.resultsLimit or 10 - if result_type == "comments": - jobs = [ - _comments_flow(r, input_model=input_model, per_target=per_target) - for r in targets - if r.kind in ("post", "reel") - ] - async with aclosing(fan_out(jobs, concurrency=_COMMENT_CONCURRENCY)) as stream: - async for item in stream: - yield item - return - if result_type == "details": jobs = [_details_flow(r, input_model=input_model) for r in targets] async with aclosing(fan_out(jobs)) as stream: diff --git a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py index 596b85c2c..aec46a678 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py @@ -1,18 +1,18 @@ """Classify and normalize an Instagram URL into a scrape job. -Covers the supported ``directUrls`` shapes: a profile, a post (``/p/``), a reel -(``/reel/``), a hashtag (``/explore/tags/``), and a place -(``/explore/locations/``), plus bare profile IDs. Non-Instagram hosts resolve to -``None`` so the orchestrator can skip them. Mirrors the frozen ``ResolvedUrl`` -dataclass shape of ``../reddit/url_resolver.py``. +Covers the anonymously-scrapable ``directUrls`` shapes: a profile, a post +(``/p/``), and a reel (``/reel/``), plus bare profile IDs. Hashtag and place +URLs are deliberately unsupported — their feeds are login-walled for anonymous +callers (use Google-backed discovery + single-post extraction instead). +Non-Instagram hosts resolve to ``None`` so the orchestrator can skip them. +Mirrors the frozen ``ResolvedUrl`` dataclass shape of ``../reddit/url_resolver.py``. Normalization rules (from the reference spec): - ``_u/`` and ``/profilecard/`` segments are stripped. - Story URLs (``/stories//...``) reduce to the profile. -- Location URLs are valid with the numeric ID alone (no trailing slug). -- Numeric post-ID URLs are only valid for the ``comments`` flow; elsewhere the - shortCode form is required, so a numeric-ID URL resolves with - ``numeric_post_id`` set and callers reject it outside comments. +- Numeric post-ID URLs cannot be single-post-extracted anonymously (the HTML + page keys on the shortCode), so they resolve with ``numeric_post_id`` set and + the media flow skips them. - ``share/`` redirect resolution is handled at fetch time (network), not here. """ @@ -22,7 +22,7 @@ from dataclasses import dataclass from typing import Literal from urllib.parse import urlparse -ResolvedKind = Literal["profile", "post", "reel", "hashtag", "place"] +ResolvedKind = Literal["profile", "post", "reel"] _INSTAGRAM_HOSTS = frozenset( {"m.instagram.com", "www.instagram.com", "instagram.com"} @@ -75,11 +75,6 @@ def resolve_url(url: str) -> ResolvedUrl | None: if not segments: return None head = segments[0] - if head == "explore" and len(segments) >= 3 and segments[1] == "tags": - return ResolvedUrl("hashtag", segments[2], url) - if head == "explore" and len(segments) >= 3 and segments[1] == "locations": - slug = segments[3] if len(segments) >= 4 else None - return ResolvedUrl("place", segments[2], url, slug=slug) if head == "p" and len(segments) >= 2: code = segments[1] return ResolvedUrl("post", code, url, numeric_post_id=code.isdigit()) From 36bb4a1bea72722a6cd4af5bdcbfa172b28f23ed Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:54 +0530 Subject: [PATCH 104/160] docs(instagram): update platform scraper README --- .../proprietary/platforms/instagram/README.md | 148 +++++++++++------- 1 file changed, 92 insertions(+), 56 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md index fa42e30a6..e4d7ca0f7 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/README.md +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -1,37 +1,44 @@ -# Instagram scraper (anonymous, no browser) +# Instagram scraper (anonymous) -Platform-native Instagram scraper (anonymous, no browser). Standalone module: it -depends only on `app.utils.proxy` + `scrapling` and exposes a stable public API. -Its input/output surface is a **reference-compatible** mirror of the public -Instagram scraper actor spec (same `resultsType` / `directUrls` / camelCase field -names, additive `extra="allow"` parity), so callers written against that surface -work unchanged. It is **not** wired into ingestion or Celery — the capability -layer under `app/capabilities/instagram/` is what turns these primitives into -REST + agent + MCP surfaces. +Platform-native Instagram scraper. **Anonymous-only** and browser-free: every +flow stays on the cheap HTTP tier (`app.utils.proxy` + `scrapling`), and profile +discovery reuses the `google_search` platform (see below). It exposes a stable +public API whose input/output surface mirrors the public Instagram scraper actor +spec (same `resultsType` / `directUrls` / camelCase field names, additive +`extra="allow"` parity), so callers written against that surface work unchanged. +It is **not** wired into ingestion or Celery — the capability layer under +`app/capabilities/instagram/` turns these primitives into REST + agent + MCP +surfaces. ## Approach -Instagram's public web app exposes anonymous, logged-out JSON behind a handful of -`www.instagram.com` endpoints once a session carries an anonymous `csrftoken` + -`mid` cookie pair and the `x-ig-app-id` web header: +Instagram's public web app exposes anonymous, logged-out data once a session +carries an anonymous `csrftoken` + `mid` cookie pair and the `x-ig-app-id` web +header: > Warm an anonymous session with one plain GET to `www.instagram.com/` (mints -> `csrftoken` + `mid`), then GET the web JSON endpoints through that same +> `csrftoken` + `mid`), then GET the web endpoints through that same > Chrome-impersonated, sticky-IP session. Rotate the residential IP + re-warm on > a login wall (401/403), back off on 429. -Endpoints used (anonymous web tier only): +Surfaces used: -| Flow | Endpoint | -|---|---| -| profile / posts / reels | `api/v1/users/web_profile_info/?username=…` | -| comments | `p//?__a=1&__d=dis` | -| hashtag | `api/v1/tags/web_info/?tag_name=…` | -| place | `api/v1/locations/web_info/?location_id=…` | -| discovery search | `web/search/topsearch/?query=…` | +| Flow | Surface | Extractor | +|---|---|---| +| profile / details | `api/v1/users/web_profile_info/?username=…` (JSON) | `parse_profile` | +| profile feed (posts/reels/mentions) | the media embedded in the same profile JSON | `parse_media` | +| single post / reel | `/p//` (HTML: ld+json + og-meta) | `parse_post` | +| profile discovery | Google `site:instagram.com ` | `resolve_url` | -**No browser, no Chromium, no `solve_cloudflare`** — this stays on the cheap HTTP -tier the sibling scrapers already use. +**Why anonymous-only is a hard constraint.** Live logged-out probes show that +Instagram walls the interesting endpoints for anyone without a `sessionid` +account cookie: `api/v1/tags/web_info/`, `api/v1/locations/web_info/`, the +comment thread API (`?__a=1`), and `web/search/topsearch/` all **302 to +`/accounts/login/`**. We cannot log in (see below), so hashtag feeds, place +feeds, comment scraping, and IG's native keyword search were **removed** — they +can only ever return a login wall. What survives is what a logged-out browser can +actually read: a profile's web info + its embedded recent media, and a public +post/reel page's embedded metadata. ## Anonymous only — no authentication, ever @@ -47,59 +54,88 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. | File | Responsibility | |---|---| | `__init__.py` | Public exports: `InstagramScrapeInput`, item models, `iter_instagram`, `scrape_instagram`, `InstagramAccessBlockedError`. | -| `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramComment`, `InstagramProfile`, `InstagramHashtag`, `InstagramPlace`) each with `to_output()`. | -| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json`. No browser imports. | -| `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`/`hashtag`/`place`; non-Instagram → `None`. Strips `_u/`, `profilecard/`; story → profile. | -| `parsers.py` | Pure JSON→dict mapping (`parse_media`, `parse_comment`, `parse_profile`, `parse_hashtag`, `parse_place`, `_edges`). I/O-free. | -| `scraper.py` | Orchestrator: `_media_flow`/`_comments_flow`/`_details_flow`/`_discover`, `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | +| `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramProfile`) each with `to_output()`. | +| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json` (JSON) / `fetch_html` (HTML) sharing one resilient `_fetch(path, params, extract)` loop. | +| `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`; non-Instagram (and hashtag/place) → `None`. Strips `_u/`, `profilecard/`; story → profile. | +| `parsers.py` | Pure mapping (`parse_media`, `parse_profile`, `parse_post` [ld+json/og], `_edges`). I/O-free. | +| `scraper.py` | Orchestrator: `_media_flow`/`_details_flow`/`_discover` (+ `_discover_via_google`), `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | ## How it works 1. `iter_instagram` resolves `directUrls` (or runs a discovery `search` per the comma-split queries) into targets and fans them out on a pool of warm proxy - sessions (`fan_out`, 8-way; 4-way for comments). Each worker opens one - sticky-IP session and warms `csrftoken`/`mid` once, reusing it across the - sequential targets it pulls. -2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media feeds, - `comments` → per-post comment items, `details` → profile/hashtag/place - metadata. Media items de-dupe by `id` across targets. -3. `fetch_json` warms the session on first use, rotates the IP + re-warms on - 401/403, backs off on 429, returns `None` on 404. -4. Parsers map raw web JSON to flat dicts; the orchestrator stamps `scrapedAt` - and applies `resultsLimit` / `onlyPostsNewerThan` as request-time policy. + sessions (`fan_out`, 8-way). Each worker opens one sticky-IP session and warms + `csrftoken`/`mid` once, reusing it across the sequential targets it pulls. +2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media items, + `details` → profile metadata. Media items de-dupe by `id` across targets. + - A **profile** target → `web_profile_info` JSON → `parse_media` over the + embedded recent-media edges (feed) or `parse_profile` (details). + - A **post/reel** target → `fetch_html("p//")` → `parse_post`, which + reads the page's `application/ld+json` (preferred) and Open Graph meta + (fallback). Numeric-ID post URLs are skipped (the page keys on the shortCode). +3. `fetch_json` / `fetch_html` warm the session on first use, rotate the IP + + re-warm on 401/403, back off on 429, return `None` on 404, and raise + `InstagramAccessBlockedError` on a `/accounts/login/` redirect. +4. Parsers map raw web JSON/HTML to flat dicts; the orchestrator stamps + `scrapedAt` and applies `resultsLimit` / `onlyPostsNewerThan` as request-time + policy. + +## Profile discovery (Google-backed) + +Instagram's native keyword search is login-walled, so `_discover` resolves a +query that is a valid handle directly (`"messi"` → `instagram.com/messi/`) and +routes any other query (e.g. `"national geographic"`) through +`_discover_via_google`, which calls the `google_search` platform with +`site:instagram.com`, classifies each organic URL with `resolve_url`, keeps the +**profile** hits (discovery is profile-only), de-dupes, and caps at `searchLimit`. + +Caveats: + +- **Coupling**: Instagram depends on the `google_search` platform. The dependency + is one-directional and lives behind `_discover_via_google` so it stays testable. +- **Quality**: results reflect Google's index/ranking of `instagram.com`, not + IG's own relevance. This is discovery, not search parity. ## Observed limits & calibration caveats -- Anonymous web JSON is rate-limited per IP; the sticky-session pool keeps each - IP's request rate modest but a hot pool will still hit login walls — that's the - `InstagramAccessBlockedError` path, not a bug. +- Anonymous web JSON/HTML is rate-limited per IP; the sticky-session pool keeps + each IP's request rate modest but a hot pool will still hit login walls — that's + the `InstagramAccessBlockedError` path, not a bug. - `likesCount` is frequently withheld on anonymous responses (surfaces as `-1` or absent upstream); treat it as best-effort. -- Comments on the anonymous media page cap at ~50/post; deeper paging needs the - GraphQL cursor endpoint whose doc-id drifts (see the `ponytail:` note in - `scraper.py`/`fetch.py`). +- **Single-post extraction** reads whatever the public `/p/` document embeds + (ld+json + og-meta). If Instagram strips both for a given post (private, taken + down, or a login interstitial), `parse_post` returns `None` — an honest empty, + never a fabricated item. ponytail: the embedded-blob shapes can drift; a live + probe that dumps the raw HTML pins them (see Testing) and any change is contained + to `parse_post`. - The `$3.50 / 1k items` default meter assumes the proxy-bytes-per-item measured - on the reference targets; re-measure with the `references/` scale harness before - high-volume production use. + on the reference targets; re-measure with the scale harness before high-volume use. ## Testing - Offline unit tests: `tests/unit/platforms/instagram/` — `test_skeleton.py` - (schema + URL resolver), `test_parsers.py` (fixture-pinned mapping), - `test_fetch_resilience.py` (warm / rotate / backoff loop with fake sessions, no - network), `test_budget.py` (fair-share caps + de-dup). -- Live e2e (needs network + residential proxy): `scripts/e2e_instagram_scraper.py` - — step 0 is the go/no-go cookie probe; later steps exercise the flows and dump - trimmed, PII-anonymized fixtures. + (schema + URL resolver), `test_parsers.py` (mapping incl. `parse_post` + ld+json/og shapes; fixture-pinned tests skip when the fixture is absent), + `test_discovery.py` (Google-backed profile discovery with a fake `scrape_serps`), + `test_fetch_resilience.py` (warm / rotate / backoff loop + fan-out with fake + sessions, no network), `test_budget.py` (fair-share caps + de-dup). +- Stress / accuracy harness (live, needs network + residential proxy): + `scripts/stress/stress_instagram_scraper.py` — `--mode live-discovery` (profile + discovery accuracy), `--mode probe-post` (dumps a real anonymous `/p/` payload + to `fixtures/post.json` and shows what `parse_post` extracted), and + `--mode accuracy` (field coverage across the profile + single-post flows). ```bash cd surfsense_backend -.venv/bin/python -m pytest tests/unit/platforms/instagram/ -.venv/bin/python scripts/e2e_instagram_scraper.py # live; regenerates fixtures +uv run pytest tests/unit/platforms/instagram/ +# Live single-post probe: confirms /p/ is anonymously extractable + pins the shape +uv run python scripts/stress/stress_instagram_scraper.py --mode probe-post \ + --post https://www.instagram.com/p// ``` ## TODO / out of scope (v1) -- Deep feed pagination past the first web page of media (GraphQL cursor doc-id). -- Deep comment pagination past the ~50/post embedded ceiling. +- Deep feed pagination past the first web page of profile media (GraphQL cursor + doc-id). - Sticky-IP provider parity (same `__sid` caveat as the Reddit sibling). From 8fc86b7fa57bdc36366ab9979754a14275a6d4b6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:58 +0530 Subject: [PATCH 105/160] refactor(mcp): align Instagram scraper tool with new pipeline --- .../features/scrapers/platforms/instagram.py | 114 +++++------------- surfsense_mcp/mcp_server/selfcheck.py | 1 - surfsense_mcp/mcp_server/server.py | 2 +- 3 files changed, 30 insertions(+), 87 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py index 39dfea71d..9683e6581 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -1,4 +1,4 @@ -"""Instagram scraper tools: posts/reels, comments, and details.""" +"""Instagram scraper tools: posts/reels and profile details (anonymous-only).""" from __future__ import annotations @@ -14,14 +14,13 @@ from ..annotations import SCRAPE from ..capability import run_scraper ResultType = Literal["posts", "reels", "mentions"] -SearchType = Literal["hashtag", "profile", "place", "user"] -DetailSearchType = Literal["hashtag", "profile", "place"] +SearchType = Literal["profile", "user"] def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the Instagram scrape, comments, and details tools.""" + """Register the Instagram scrape and details tools (anonymous-only).""" @mcp.tool( name="surfsense_instagram_scrape", @@ -33,21 +32,22 @@ def register( urls: Annotated[ list[str] | None, Field( - description="Instagram URLs: a profile, post (/p/), reel " - "(/reel/), hashtag (/explore/tags/), or place " - "(/explore/locations/). Provide urls OR search_queries." + description="Instagram URLs: a profile, post (/p/), or reel " + "(/reel/). Hashtag/place URLs are unsupported (login-walled). " + "Provide urls OR search_queries." ), ] = None, search_queries: Annotated[ list[str] | None, Field( - description="Terms to discover content for (hashtags as plain " - "text, no '#'). Provide search_queries OR urls." + description="Terms to discover public profiles for (resolved via " + "Google). Provide search_queries OR urls." ), ] = None, search_type: Annotated[ - SearchType, Field(description="What to discover from search_queries.") - ] = "hashtag", + SearchType, + Field(description="Discovery kind (profile-only)."), + ] = "profile", result_type: Annotated[ ResultType, Field(description="Which feed to return. 'mentions' needs profile URLs."), @@ -61,10 +61,10 @@ def register( """Scrape public Instagram posts or reels from URLs or search queries. Use this for Instagram content research: a creator's recent posts, a - hashtag or location feed, or discovering posts by keyword. For a post's - comment section use surfsense_instagram_comments; for profile/hashtag/ - place metadata use surfsense_instagram_details. Returns per-item caption, - likes, comments count, media URLs, and owner. + single post/reel, or discovering public profiles by keyword. For a + profile's metadata use surfsense_instagram_details. Returns per-item + caption, likes, comments count, media URLs, and owner. Anonymous-only: + hashtag/place feeds and comment threads are login-walled and unavailable. Example: urls=['https://www.instagram.com/natgeo/'], result_type='reels'. """ return await run_scraper( @@ -83,64 +83,9 @@ def register( response_format=response_format, ) - @mcp.tool( - name="surfsense_instagram_comments", - title="Fetch Instagram comments", - annotations=SCRAPE, - structured_output=False, - ) - async def instagram_comments( - urls: Annotated[ - list[str], - Field( - min_length=1, - description="Instagram post or reel URLs, e.g. " - "['https://www.instagram.com/p/Cabc123/'].", - ), - ], - max_comments_per_post: Annotated[ - int, - Field(ge=1, le=50, description="Max comments per post (Instagram caps at 50)."), - ] = 10, - include_replies: Annotated[ - bool, Field(description="Include nested replies.") - ] = False, - newest_first: Annotated[ - bool, Field(description="Return newest comments first.") - ] = False, - max_items: Annotated[ - int, Field(ge=1, description="Max total comments across all posts.") - ] = 20, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch the comments (and optionally replies) on Instagram posts or reels. - - Use this when the user wants a post's discussion or audience reaction - rather than the post itself; get post URLs from surfsense_instagram_scrape - if you only have a topic or profile. Returns comment text, author, likes, - and replies. - Example: urls=['https://www.instagram.com/p/Cabc123/'], include_replies=True. - """ - return await run_scraper( - client, - context, - platform="instagram", - verb="comments", - payload={ - "urls": urls, - "max_comments_per_post": max_comments_per_post, - "include_replies": include_replies, - "newest_first": newest_first, - "max_items": max_items, - }, - workspace=workspace, - response_format=response_format, - ) - @mcp.tool( name="surfsense_instagram_details", - title="Fetch Instagram profile/hashtag/place details", + title="Fetch Instagram profile details", annotations=SCRAPE, structured_output=False, ) @@ -148,35 +93,34 @@ def register( urls: Annotated[ list[str] | None, Field( - description="Profile, hashtag, or place URLs (or bare profile " - "IDs). Provide urls OR search_queries." + description="Profile URLs (or bare profile IDs). Provide urls OR " + "search_queries." ), ] = None, search_queries: Annotated[ list[str] | None, Field( - description="Terms to discover profiles/hashtags/places for. " - "Provide search_queries OR urls." + description="Terms to discover public profiles for. Provide " + "search_queries OR urls." ), ] = None, search_type: Annotated[ - DetailSearchType, - Field(description="What to discover from search_queries."), - ] = "hashtag", + SearchType, + Field(description="Discovery kind (profile-only)."), + ] = "profile", max_items: Annotated[ int, Field(ge=1, description="Max detail items to return.") ] = 10, workspace: WorkspaceParam = None, response_format: ResponseFormatParam = "markdown", ) -> str: - """Fetch Instagram profile, hashtag, or place metadata. + """Fetch Instagram profile metadata. - Use this for entity lookups: a profile's follower/post counts and bio, a - hashtag's post volume and top posts, or a place's coordinates and post - count. For a feed of posts use surfsense_instagram_scrape instead. Each - item carries a detailKind field marking whether it is a profile, hashtag, - or place. - Example: urls=['https://www.instagram.com/explore/tags/crossfit/']. + Use this for entity lookups: a profile's follower/post counts and bio. + For a feed of posts use surfsense_instagram_scrape instead. Each item + carries a detailKind field (always "profile"). Anonymous-only: hashtag + and place details are login-walled and unavailable. + Example: urls=['https://www.instagram.com/natgeo/']. """ return await run_scraper( client, diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index f7b7a76c5..ba2e95a78 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -26,7 +26,6 @@ EXPECTED_TOOLS = { "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_instagram_scrape", - "surfsense_instagram_comments", "surfsense_instagram_details", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 19cf966f7..088952fd1 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -37,7 +37,7 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " "communities), YouTube (videos, transcripts, comments), Instagram " - "(posts, reels, comments, profile/hashtag/place details), Google " + "(posts, reels, profile details), Google " "Maps (places, reviews), Google Search results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " From 9e9dd8a1243ae82d40fae2aec39bc4463e09a123 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:02 +0530 Subject: [PATCH 106/160] feat(subagent): update Instagram subagent prompts and tools --- .../subagents/builtins/instagram/description.md | 4 ++-- .../subagents/builtins/instagram/system_prompt.md | 9 +++------ .../subagents/builtins/instagram/tools/index.py | 5 ++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md index 9efedc52a..9d65defe4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md @@ -1,2 +1,2 @@ -Instagram specialist: pulls structured data from Instagram — posts and reels (caption, likes, comments count, media URLs, owner, timestamp), a post's comments and replies, and profile/hashtag/place details (follower and post counts, bio, hashtag volume and top posts, place coordinates). Finds content by hashtag, profile, or place search, and compares fresh Instagram data against earlier findings in this chat. -Use whenever the task involves Instagram content or an instagram.com link. Triggers include "get this Instagram profile/post/reel", "find posts about X on Instagram", "how many followers/likes", "get the comments on this post", "what are people saying about this reel", "look up this hashtag/location", and comparisons against earlier Instagram results in this chat. Not for general web pages (use the web crawling specialist for non-Instagram URLs). +Instagram specialist: pulls structured data from public Instagram — posts and reels (caption, likes, comments count, media URLs, owner, timestamp) and profile details (follower and post counts, bio). Finds public profiles by search and compares fresh Instagram data against earlier findings in this chat. Anonymous-only: hashtag/place feeds and comment threads are login-walled and unavailable. +Use whenever the task involves public Instagram content or an instagram.com profile/post/reel link. Triggers include "get this Instagram profile/post/reel", "find the Instagram profile for X", "how many followers/likes", and comparisons against earlier Instagram results in this chat. Not for general web pages (use the web crawling specialist for non-Instagram URLs). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md index 7d5278afa..617a376b7 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -7,19 +7,16 @@ Answer the delegated question from live Instagram data gathered with your verbs, - `instagram_scrape` -- `instagram_comments` - `instagram_details` - `read_run` / `search_run` (free readers for stored scrape output) -- Known profile/post/reel/hashtag/place links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). -- Finding content on a topic: call `instagram_scrape` with `search_queries` and the matching `search_type` (hashtag, profile, or place). -- Comments / sentiment on specific posts or reels: call `instagram_comments` with the post `urls`. -- Profile, hashtag, or place metadata (follower counts, bio, hashtag volume, coordinates): call `instagram_details`. +- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). Hashtag/place URLs are unsupported (login-walled). +- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). +- Profile metadata (follower counts, bio, post count): call `instagram_details`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. -- Multi-post comment analysis: a batched comments result lists posts in order, so a truncated preview usually shows only the first post(s). Before summarizing, page the stored run (or `search_run` by post id) until you have read real comments for EVERY post in the batch — never infer one post's sentiment from another's, and never report a post as "limited data" while its comments sit unread in the run. - Comparison requests: pull the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py index 1bfbabad9..033ec6f0c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py @@ -1,4 +1,4 @@ -"""``instagram`` sub-agent tools: the three Instagram capability verbs.""" +"""``instagram`` sub-agent tools: the Instagram capability verbs.""" from __future__ import annotations @@ -8,7 +8,6 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools -from app.capabilities.instagram.comments.definition import INSTAGRAM_COMMENTS from app.capabilities.instagram.details.definition import INSTAGRAM_DETAILS from app.capabilities.instagram.scrape.definition import INSTAGRAM_SCRAPE @@ -16,7 +15,7 @@ NAME = "instagram" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [INSTAGRAM_SCRAPE, INSTAGRAM_COMMENTS, INSTAGRAM_DETAILS] +_CI_VERBS = [INSTAGRAM_SCRAPE, INSTAGRAM_DETAILS] def load_tools( From 69a8433910377c0ea6d9fbb813cc8e7b748bfa93 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:05 +0530 Subject: [PATCH 107/160] docs(web): refresh Instagram connector and MCP docs --- .../docs/connectors/native/instagram.mdx | 40 ++++++------------- .../content/docs/how-to/mcp-server.mdx | 2 +- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/surfsense_web/content/docs/connectors/native/instagram.mdx b/surfsense_web/content/docs/connectors/native/instagram.mdx index f9758662b..5881ac832 100644 --- a/surfsense_web/content/docs/connectors/native/instagram.mdx +++ b/surfsense_web/content/docs/connectors/native/instagram.mdx @@ -1,9 +1,11 @@ --- title: Instagram -description: Scrape public Instagram posts, reels, comments, and profile/hashtag/place details +description: Scrape public Instagram posts, reels, and profile details (anonymous-only) --- -The Instagram scraper has three verbs: **scrape** for posts and reels, **comments** for a post's comment threads, and **details** for profile, hashtag, and place metadata. It reads only public, logged-out data — no account, login, or Graph API app required. +The Instagram scraper has two verbs: **scrape** for posts and reels, and **details** for profile metadata. It reads only public, logged-out data — no account, login, or Graph API app required. + +Instagram login-walls hashtag feeds, place feeds, and comment threads for anonymous visitors, so those are not supported. The scraper focuses on what is reliably public: profiles, posts, and reels. ## Scrape posts and reels @@ -11,15 +13,15 @@ The Instagram scraper has three verbs: **scrape** for posts and reels, **comment POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/scrape ``` -Give it Instagram URLs (profile, post `/p/`, reel `/reel/`, hashtag `/explore/tags/`, or place `/explore/locations/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, and owner. +Give it Instagram URLs (profile, post `/p/`, or reel `/reel/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, and owner. Provide exactly one source: `urls` **or** `search_queries` (never both). | Field | Default | Description | |-------|---------|-------------| -| `urls` | — | Profile, post, reel, hashtag, or place URLs or bare profile IDs (max 20 sources per call) | -| `search_queries` | — | Discovery keywords (hashtags as plaintext, no `#`); each returns up to `max_per_target` items | -| `search_type` | `hashtag` | What to discover from `search_queries`: `hashtag`, `profile`, `place`, or `user` | +| `urls` | — | Profile, post, or reel URLs or bare profile IDs (max 20 sources per call) | +| `search_queries` | — | Discovery keywords resolved to public profiles via Google; each returns up to `max_per_target` items | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | | `result_type` | `posts` | Which feed to return: `posts`, `reels`, or `mentions` (`mentions` needs profile URLs) | | `newer_than` | — | Only return posts newer than this: `YYYY-MM-DD`, ISO timestamp, or relative (`2 months`), UTC | | `skip_pinned_posts` | `false` | Exclude pinned posts in posts mode | @@ -39,37 +41,19 @@ curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/instagram/scrape" \ Billing is per returned item. -## Fetch comments - -```bash -POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/comments -``` - -Give it post or reel URLs; returns comment items with author, text, like and reply counts, and optionally nested replies — useful for gauging sentiment on a specific post. - -| Field | Default | Description | -|-------|---------|-------------| -| `urls` | required | 1–20 post or reel URLs | -| `max_comments_per_post` | `10` | Max comments per post (Instagram caps at 50) | -| `include_replies` | `false` | Include nested replies | -| `newest_first` | `false` | Return newest comments first | -| `max_items` | `20` | Max total comments across all posts (max 100) | - -Billing is per returned comment or reply. - ## Fetch details ```bash POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/details ``` -Give it profile, hashtag, or place URLs (or search queries) and get entity metadata back: a profile's follower/post counts and bio, a hashtag's post volume and top posts, or a place's coordinates and post count. Each item carries a `detailKind` field marking whether it is a `profile`, `hashtag`, or `place`. +Give it profile URLs (or search queries) and get profile metadata back: follower/post counts and bio. Each item carries a `detailKind` field, always `profile`. | Field | Default | Description | |-------|---------|-------------| -| `urls` | — | Profile, hashtag, or place URLs or bare profile IDs (max 20) | -| `search_queries` | — | Terms to discover profiles/hashtags/places for (provide these OR urls) | -| `search_type` | `hashtag` | What to discover from `search_queries`: `hashtag`, `profile`, or `place` | +| `urls` | — | Profile URLs or bare profile IDs (max 20) | +| `search_queries` | — | Terms to discover public profiles for (provide these OR urls) | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | | `max_items` | `10` | Max detail items to return | Billing is per returned item. diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index 632dc5a4d..1318197f8 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_comments`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From 42d03fdbc7ab7e46974ce8b21c1d2a1dff76ea41 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:08 +0530 Subject: [PATCH 108/160] feat(web): update Instagram marketing and playground catalog --- surfsense_web/app/(home)/mcp-server/page.tsx | 1 - .../lib/connectors-marketing/instagram.tsx | 72 +++++++------------ surfsense_web/lib/playground/catalog.ts | 1 - 3 files changed, 25 insertions(+), 49 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index a5557e11e..3bdfcf100 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -94,7 +94,6 @@ const TOOL_GROUPS = [ "surfsense_youtube_scrape", "surfsense_youtube_comments", "surfsense_instagram_scrape", - "surfsense_instagram_comments", "surfsense_instagram_details", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 33b963eb8..39a4a04cb 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -6,9 +6,9 @@ export const instagram: ConnectorPageContent = { name: "Instagram", icon: IconBrandInstagram, - metaTitle: "Instagram Scraper API for Social Listening | SurfSense", + metaTitle: "Instagram Scraper API for Creator Research | SurfSense", metaDescription: - "Scrape public Instagram posts, reels, comments, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", + "Scrape public Instagram posts, reels, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", keywords: [ "instagram scraper", "instagram scraper api", @@ -16,21 +16,21 @@ export const instagram: ConnectorPageContent = { "instagram api alternative", "scrape instagram", "instagram graph api alternative", - "instagram comment scraper", "instagram profile scraper", - "instagram hashtag scraper", + "instagram post scraper", + "instagram reel scraper", "instagram data api", "instagram mcp server", - "instagram sentiment analysis", + "creator research", "social listening", ], - h1: "Instagram Scraper API for Social Listening and Creator Research", + h1: "Instagram Scraper API for Creator Research and Social Listening", heroLede: - "The SurfSense Instagram API extracts public posts, reels, comments, and profile, hashtag, and place details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post and what their audiences say, so you spot trends and sentiment first.", + "The SurfSense Instagram API extracts public posts, reels, and profile details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.", transcript: { - prompt: "Pull recent reels from @competitor and tell me what the comments think", + prompt: "Pull recent reels from @competitor and summarize what they're posting", toolCall: 'instagram.scrape({ urls: ["instagram.com/competitor/"],\n result_type: "reels", max_items: 20 })', rows: [ @@ -40,9 +40,9 @@ export const instagram: ConnectorPageContent = { tag: "top reel", }, { - primary: "Comments skew positive on price, negative on shipping", - secondary: "1,203 comments · 0.71 positive", - tag: "sentiment", + primary: "Cadence up 40% this month, all short-form reels", + secondary: "20 reels · 12 days", + tag: "trend", }, { primary: "3 creators tagged asking for a collab", @@ -50,34 +50,22 @@ export const instagram: ConnectorPageContent = { tag: "lead signal", }, ], - resultSummary: "20 reels · 4,910 comments · surfaced in 2.4s", + resultSummary: "20 reels · surfaced in 2.4s", }, extractIntro: - "Every call returns structured items keyed by type. Point the API at a profile, post, reel, hashtag, or place URL, or discover content with a search query.", + "Every call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.", extractFields: [ { label: "Posts & Reels", description: "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.", }, - { - label: "Comments", - description: "Comment text, author, like and reply counts, and nested replies for any post.", - }, { label: "Profiles", description: "Follower, following, and post counts, bio, external URL, verified and business flags.", }, - { - label: "Hashtags", - description: "Post volume, top posts, and recent posts for any /explore/tags/ hashtag.", - }, - { - label: "Places", - description: "Name, coordinates, address, and recent posts for any /explore/locations/ place.", - }, { label: "Owner & Media", description: @@ -90,17 +78,12 @@ export const instagram: ConnectorPageContent = { { title: "Creator and competitor monitoring", description: - "Track what your competitors and target creators post, and how their audiences react. Feed the stream to an agent that flags viral formats, launches, and shifts in engagement the moment they land.", + "Track what your competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.", }, { - title: "Audience sentiment and comment mining", + title: "Content and format research", description: - "Pull full comment threads on a post or reel and score them for sentiment, so you can measure how a launch, a collab, or a campaign actually resonated with real followers.", - }, - { - title: "Hashtag and trend research", - description: - "Map the volume and top content behind any hashtag before you spend on a campaign. Turn trend research into a content calendar your team can act on.", + "Study a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.", }, { title: "Influencer vetting and outreach", @@ -123,12 +106,7 @@ export const instagram: ConnectorPageContent = { { feature: "Coverage", official: "Mostly your own or connected accounts", - surfsense: "Any public profile, post, hashtag, or place", - }, - { - feature: "Comments", - official: "Limited to accounts you manage", - surfsense: "Public comments and replies on any post, up to 50 per post", + surfsense: "Any public profile, post, or reel", }, { feature: "Setup", @@ -163,21 +141,21 @@ export const instagram: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "Instagram URLs or bare profile IDs: profile, post (/p/), reel (/reel/), hashtag (/explore/tags/), or place (/explore/locations/). Max 20.", + "Instagram URLs or bare profile IDs: profile, post (/p/), or reel (/reel/). Hashtag and place URLs are login-walled and unsupported. Max 20.", }, { name: "search_queries", type: "string[]", defaultValue: "[]", description: - "Discovery keywords (hashtags as plaintext, no '#'). Provide these OR urls, not both. Max 20.", + "Discovery keywords resolved to public profiles via Google. Provide these OR urls, not both. Max 20.", }, { name: "search_type", type: "string", - defaultValue: '"hashtag"', + defaultValue: '"profile"', description: - "What to discover from search_queries: hashtag, profile, place, or user. Only used with search_queries.", + "What to discover from search_queries: profile or user. Only used with search_queries.", }, { name: "result_type", @@ -261,7 +239,7 @@ export const instagram: ConnectorPageContent = { { question: "Is scraping Instagram legal?", answer: - "SurfSense reads only public Instagram data, the same posts, reels, and comments any logged-out visitor can see. It never logs in and cannot access private accounts or stories. As always, review Instagram's terms and your own compliance needs before you run at scale.", + "SurfSense reads only public Instagram data, the same posts, reels, and profiles any logged-out visitor can see. It never logs in and cannot access private accounts, stories, or login-walled feeds. As always, review Instagram's terms and your own compliance needs before you run at scale.", }, { question: "Do I need an Instagram account or the Graph API?", @@ -271,12 +249,12 @@ export const instagram: ConnectorPageContent = { { question: "What are the rate limits?", answer: - "Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request, and up to 50 comments per post. SurfSense manages the underlying request budget and proxy rotation for you, so you scale reads without managing tokens.", + "Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request. SurfSense manages the underlying request budget and proxy rotation for you, so you scale reads without managing tokens.", }, { - question: "Can I get comments and replies?", + question: "Can I scrape hashtags, places, or comments?", answer: - "Yes. The instagram.comments verb returns public comments on any post or reel, with author, like and reply counts, and optionally the nested replies. Get post URLs first from instagram.scrape if you only have a topic or a profile.", + "No. Instagram login-walls hashtag feeds, place feeds, and comment threads for logged-out visitors, so SurfSense does not offer them. The API focuses on what is reliably public and anonymous: profiles, posts, and reels.", }, ], diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index b1287e447..5f72c07ec 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -55,7 +55,6 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ icon: InstagramIcon, verbs: [ { name: "instagram.scrape", verb: "scrape", label: "Scrape" }, - { name: "instagram.comments", verb: "comments", label: "Comments" }, { name: "instagram.details", verb: "details", label: "Details" }, ], }, From 82a63043f7f4833afa5f360988de4ba5fcc8f17b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:11 +0530 Subject: [PATCH 109/160] test(instagram): update capability tests for schema changes --- .../capabilities/instagram/test_executor.py | 28 ++----------------- .../capabilities/instagram/test_registry.py | 8 +----- .../capabilities/instagram/test_schemas.py | 23 +++++---------- 3 files changed, 11 insertions(+), 48 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py index 9087b0bcf..a19e597c5 100644 --- a/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py @@ -9,8 +9,6 @@ from __future__ import annotations import pytest -from app.capabilities.instagram.comments.executor import build_comments_executor -from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.instagram.details.executor import build_details_executor from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput from app.capabilities.instagram.scrape.executor import build_scrape_executor @@ -50,10 +48,10 @@ async def test_scrape_maps_urls_and_wraps_items(): async def test_scrape_joins_search_queries(): fake = _FakeScraper([]) execute = build_scrape_executor(fake) - await execute(ScrapeInput(search_queries=["fit", "gym"], search_type="hashtag")) + await execute(ScrapeInput(search_queries=["natgeo", "nasa"], search_type="profile")) actor_input, _ = fake.calls[0] - assert actor_input.search == "fit,gym" - assert actor_input.searchType == "hashtag" + assert actor_input.search == "natgeo,nasa" + assert actor_input.searchType == "profile" assert actor_input.directUrls == [] @@ -66,26 +64,6 @@ async def test_scrape_access_blocked_maps_to_forbidden(): await execute(ScrapeInput(urls=["x"])) -async def test_comments_maps_flags(): - fake = _FakeScraper([{"id": "c1", "text": "nice"}]) - execute = build_comments_executor(fake) - out = await execute( - CommentsInput( - urls=["https://www.instagram.com/p/Cabc/"], - newest_first=True, - include_replies=True, - max_comments_per_post=25, - ) - ) - assert isinstance(out, CommentsOutput) - assert out.items[0].text == "nice" - actor_input, _ = fake.calls[0] - assert actor_input.resultsType == "comments" - assert actor_input.isNewestComments is True - assert actor_input.includeNestedComments is True - assert actor_input.resultsLimit == 25 - - async def test_details_maps_and_wraps_discriminated_items(): fake = _FakeScraper( [ diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py index 89257253a..503feedff 100644 --- a/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py @@ -1,4 +1,4 @@ -"""The instagram namespace registers its three verbs for the doors/agent to read. +"""The instagram namespace registers its verbs for the doors/agent to read. Unlike the stale reddit assertion (``billing_unit is None``), these assert the real meters — the Capability definitions are the source of truth. @@ -23,12 +23,6 @@ def test_instagram_scrape_registered_with_item_meter(): assert cap.billing_unit is BillingUnit.INSTAGRAM_ITEM -def test_instagram_comments_registered_with_comment_meter(): - cap = get_capability("instagram.comments") - assert cap.name == "instagram.comments" - assert cap.billing_unit is BillingUnit.INSTAGRAM_COMMENT - - def test_instagram_details_registered_with_item_meter(): cap = get_capability("instagram.details") assert cap.name == "instagram.details" diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py index 13efb8a48..afabbf833 100644 --- a/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py @@ -5,7 +5,6 @@ from __future__ import annotations import pytest from pydantic import ValidationError -from app.capabilities.instagram.comments.schemas import CommentsInput from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput from app.capabilities.instagram.scrape.schemas import ( MAX_INSTAGRAM_ITEMS, @@ -47,30 +46,22 @@ def test_scrape_bounds(): ) -def test_comments_requires_urls_and_caps_at_50(): +def test_scrape_rejects_walled_search_type(): + # Discovery is profile-only; hashtag/place are login-walled and rejected. with pytest.raises(ValidationError): - CommentsInput(urls=[]) - with pytest.raises(ValidationError): - CommentsInput( - urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=51 - ) - ok = CommentsInput( - urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=50 - ) - assert ok.max_comments_per_post == 50 + ScrapeInput(search_queries=["travel"], search_type="hashtag") -def test_details_discriminated_union_selects_by_detail_kind(): +def test_details_wraps_profile_items(): out = DetailsOutput( items=[ {"detailKind": "profile", "username": "natgeo"}, - {"detailKind": "hashtag", "name": "fit"}, - {"detailKind": "place", "name": "Copenhagen"}, + {"detailKind": "profile", "username": "nasa"}, ] ) kinds = [type(i).__name__ for i in out.items] - assert kinds == ["InstagramProfile", "InstagramHashtag", "InstagramPlace"] - assert out.billable_units == 3 + assert kinds == ["InstagramProfile", "InstagramProfile"] + assert out.billable_units == 2 def test_details_rejects_both_sources(): From ba70ae1f452ffb72c7c5a5ac565dd327c7afc56b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:14 +0530 Subject: [PATCH 110/160] test(instagram): update platform tests and add discovery fixtures --- .../platforms/instagram/fixtures/post.json | 1 + .../platforms/instagram/test_discovery.py | 78 +++++++++++ .../instagram/test_fetch_resilience.py | 32 +++-- .../unit/platforms/instagram/test_parsers.py | 124 ++++++++++-------- .../unit/platforms/instagram/test_skeleton.py | 26 ++-- 5 files changed, 178 insertions(+), 83 deletions(-) create mode 100644 surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_discovery.py diff --git a/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json b/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json new file mode 100644 index 000000000..5eba4b5ad --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json @@ -0,0 +1 @@ +{"url": "https://www.instagram.com/p/Dan5IDtDE2I/", "shortcode": "Dan5IDtDE2I", "html": "\n\n\n\n\n\n\n\nInstagram\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_before_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"onViewportGuessValidation\",null,[[{\"dimension\":\"width\",\"numPixels\":767,\"operation\":\"max\",\"result\":false},{\"dimension\":\"width\",\"numPixels\":1264,\"operation\":\"min\",\"result\":true}],{\"width_px\":1366,\"height_px\":722,\"guess_source\":\"fallback\"}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"replayStyleInjectsForSSR\",null,[[]]]]}\n
Close
\"natgeodocs's
Never miss a post from natgeodocs
Sign up for Instagram to stay in the loop.
\"Photo
\"natgeo's
\"natgeodocs's
More options

\"natgeodocs's
natgeodocs
Verified
\u00a0Edited\u2022
The true story of four children who survived 40 days and 40 nights stranded in the Colombian rainforest. The siblings recount what happened in #LostInTheJungle, now streaming on @disneyplus and @hulu.
\"jeffred441's
Everyone will definitely going to enjoy watching Lost in the Jungle on @hulu and @disneyplus
Reply
\"saneehaazeemahmed's
Fav survival story
Reply
\"daikon.do's
Eldest sister final boss
10 likes
Reply
\"fourmiboys's
This was great, sorry they had to go through it.
1 like
Reply
\"babs_thecreator's
Incredible \u2764\ufe0f
Reply
\"urss_chhavi01's
\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c
1 like
Reply
\"urss_chhavi01's
\ud83d\ude0d\u2764\ufe0f\u2764\ufe0f\ud83d\ude0d
1 like
Reply
\"joyetha_assam_vibes's
\ud83d\ude2e
Reply
\"alexottscience's
Ayahuasca saved them \ud83c\udf3f
1 like
Reply
\"laurelineorsettiworldwide's
Spectacular! Inspirational!
1 like
Reply
Like
2.3K
Comment
10
Share
Save
Log in to like or comment.

English
Down chevron icon
\u00a9 2026 Instagram from Meta
\n {\"require\":[[\"CometSSRMergedContentInjector\",\"onPayloadReceived\",null,[{\"fizzRootId\":\"ssrb_root_content\",\"id\":\"render_pass_0\",\"payloadType\":\"LAST\",\"readyPreloaders\":[\"adp_PolarisLoggedOutDesktopWWWPostRootContentQueryRelayPreloader_6a5159be0652c5722640683\",\"adp_PolarisWWWLoggedOutDynamicLandingDialogQueryRelayPreloader_6a5159be065439d65551266\"],\"renderPassCount\":0,\"status\":\"success\"},{\"clientRenderErrors\":[],\"productRecoverableErrors\":[]}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_after_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_last_payload_0\"]]]}\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"} \ No newline at end of file diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_discovery.py b/surfsense_backend/tests/unit/platforms/instagram/test_discovery.py new file mode 100644 index 000000000..c95a80315 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_discovery.py @@ -0,0 +1,78 @@ +"""Offline tests for Google-backed Instagram discovery. + +Discovery is profile-only (hashtag/place feeds are login-walled). A valid handle +resolves directly; any other query falls back to the ``google_search`` platform +(``site:instagram.com``), classifying organic results with ``resolve_url`` and +keeping only profile hits. These tests inject a fake ``scrape_serps`` so there is +no network: they pin the classification, de-dup, and ``limit`` cap. +""" + +from __future__ import annotations + +from app.proprietary.platforms.instagram import scraper + + +def _fake_serps(*organic_urls: str): + async def _scrape_serps(input_model, *, limit=None): + assert input_model.site == "instagram.com" + return [{"organicResults": [{"url": u} for u in organic_urls]}] + + return _scrape_serps + + +async def test_google_discovery_keeps_only_profiles(monkeypatch): + # A non-handle query goes to Google; only profile URLs survive (hashtag / + # post / non-instagram results are dropped since discovery is profile-only). + monkeypatch.setattr( + scraper, + "scrape_serps", + _fake_serps( + "https://www.instagram.com/natgeo/", + "https://www.instagram.com/explore/tags/travel/", + "https://www.instagram.com/p/ABC123/", + "https://example.com/not-instagram", + ), + ) + targets = await scraper._discover( + "nat geo photos", search_type="profile", limit=10 + ) + assert [(t.kind, t.value) for t in targets] == [("profile", "natgeo")] + + +async def test_google_discovery_dedupes(monkeypatch): + monkeypatch.setattr( + scraper, + "scrape_serps", + _fake_serps( + "https://www.instagram.com/natgeo/", + "https://www.instagram.com/natgeo/", + ), + ) + targets = await scraper._discover( + "nat geo photos", search_type="profile", limit=10 + ) + assert len(targets) == 1 + + +async def test_google_discovery_respects_limit(monkeypatch): + monkeypatch.setattr( + scraper, + "scrape_serps", + _fake_serps( + "https://www.instagram.com/a_a/", + "https://www.instagram.com/b_b/", + "https://www.instagram.com/c_c/", + ), + ) + targets = await scraper._discover("some brand name", search_type="profile", limit=2) + assert [t.value for t in targets] == ["a_a", "b_b"] + + +async def test_discover_profile_handle_fast_path_skips_google(monkeypatch): + # A valid handle resolves directly without touching Google. + async def _boom(input_model, *, limit=None): + raise AssertionError("Google should not be called for a valid handle") + + monkeypatch.setattr(scraper, "scrape_serps", _boom) + targets = await scraper._discover("messi", search_type="user", limit=10) + assert [(t.kind, t.value) for t in targets] == [("profile", "messi")] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index e142a33f7..859c70443 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -172,7 +172,7 @@ async def test_login_redirect_fails_fast_without_rotating(): try: raised = False try: - await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) + await fetch_json("api/v1/users/web_profile_info/", {"username": "natgeo"}) except InstagramAccessBlockedError: raised = True finally: @@ -185,7 +185,7 @@ async def test_404_returns_none_without_rotating(): holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) token = _current_session.set(holder) try: - result = await fetch_json("api/v1/tags/web_info/") + result = await fetch_json("api/v1/users/web_profile_info/") finally: _current_session.reset(token) assert result is None @@ -399,12 +399,22 @@ async def test_discover_profile_is_anonymous_handle_lookup(): assert [(t.kind, t.value) for t in targets] == [("profile", "messi")] -async def test_discover_hashtag_search_blocks_anonymously(): - # hashtag/place keyword discovery has no anonymous endpoint at all, so it must - # fail loud (clear message) rather than return a misleading empty success. - raised = False - try: - await scraper._discover("travel", search_type="hashtag", limit=10) - except InstagramAccessBlockedError: - raised = True - assert raised +async def test_discover_nonhandle_routes_through_google(monkeypatch): + # A non-handle profile query goes through Google (site:instagram.com) and + # classifies the organic results into profile targets (the only kind now). + async def _fake_scrape_serps(input_model, *, limit=None): + assert input_model.site == "instagram.com" + return [ + { + "organicResults": [ + {"url": "https://www.instagram.com/natgeo/"}, + {"url": "https://www.instagram.com/p/Cabc/"}, # wrong kind + ] + } + ] + + monkeypatch.setattr(scraper, "scrape_serps", _fake_scrape_serps) + targets = await scraper._discover( + "national geographic", search_type="profile", limit=10 + ) + assert [(t.kind, t.value) for t in targets] == [("profile", "natgeo")] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index 6f6c1411e..a4c781776 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -14,10 +14,8 @@ from pathlib import Path import pytest from app.proprietary.platforms.instagram.parsers import ( - parse_comment, - parse_hashtag, parse_media, - parse_place, + parse_post, parse_profile, ) @@ -62,23 +60,6 @@ def test_parse_media_marks_video_type(): assert item["videoViewCount"] == 99 -def test_parse_comment(): - node = { - "id": "c1", - "text": "nice", - "created_at": 1_600_000_000, - "shortcode": "Cabc", - "owner": {"username": "bob", "id": "5"}, - "edge_liked_by": {"count": 3}, - } - item = parse_comment(node, post_url="https://www.instagram.com/p/Cabc/") - assert item["id"] == "c1" - assert item["text"] == "nice" - assert item["ownerUsername"] == "bob" - assert item["likesCount"] == 3 - assert item["postUrl"] == "https://www.instagram.com/p/Cabc/" - - def test_parse_profile_flattens_counts_and_latest_posts(): user = { "id": "9", @@ -100,45 +81,63 @@ def test_parse_profile_flattens_counts_and_latest_posts(): assert len(item["latestPosts"]) == 1 -def test_parse_hashtag(): - data = { - "data": { - "id": "h1", - "name": "crossfit", - "edge_hashtag_to_media": { - "count": 5, - "edges": [{"node": {"id": "m1", "shortcode": "A"}}], - }, - "edge_hashtag_to_top_posts": { - "edges": [{"node": {"id": "t1", "shortcode": "B"}}] - }, - } - } - item = parse_hashtag(data) - assert item["detailKind"] == "hashtag" - assert item["name"] == "crossfit" - assert item["postsCount"] == 5 - assert len(item["topPosts"]) == 1 - assert len(item["posts"]) == 1 +_POST_URL = "https://www.instagram.com/p/Cabc/" -def test_parse_place(): - data = { - "location": { - "id": "7538318", - "name": "Copenhagen", - "slug": "copenhagen", - "edge_location_to_media": { - "count": 3, - "edges": [{"node": {"id": "m1", "shortcode": "A"}}], - }, - } - } - item = parse_place(data) - assert item["detailKind"] == "place" - assert item["name"] == "Copenhagen" - assert item["location_id"] == "7538318" - assert len(item["posts"]) == 1 +def test_parse_post_prefers_ldjson(): + html = """ + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None + assert item["type"] == "Video" + assert item["shortCode"] == "Cabc" + assert item["url"] == _POST_URL + assert item["ownerUsername"] == "natgeo" + assert item["caption"] == "sunset over #bali with @friend" + assert item["hashtags"] == ["bali"] + assert item["mentions"] == ["friend"] + assert item["likesCount"] == 4200 + assert item["commentsCount"] == 37 + assert item["videoUrl"] == "https://cdn/v.mp4" + assert item["timestamp"] == "2024-01-02T03:04:05Z" + + +def test_parse_post_falls_back_to_og_meta(): + html = """ + + + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None + assert item["likesCount"] == 1234 + assert item["commentsCount"] == 56 + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["type"] == "Video" + + +def test_parse_post_returns_none_without_surfaces(): + # A login interstitial / empty doc carries neither ld+json nor og -> None, + # never a silent empty-success item. + assert parse_post("login", url=_POST_URL) is None + assert parse_post(None, url=_POST_URL) is None + assert parse_post("", url=_POST_URL) is None @pytest.mark.skipif( @@ -151,3 +150,14 @@ def test_fixture_profile_maps(): item = parse_profile(user) assert item["detailKind"] == "profile" assert item["username"] + + +@pytest.mark.skipif( + not (_FIXTURES / "post.json").exists(), + reason="captured fixture absent (run the single-post probe to dump /p/ HTML)", +) +def test_fixture_post_maps(): + raw = json.loads((_FIXTURES / "post.json").read_text()) + item = parse_post(raw["html"], url=raw["url"], shortcode=raw.get("shortcode")) + assert item is not None, "captured /p/ HTML produced no media item" + assert item["url"] == raw["url"] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py index 59017c81c..f609883eb 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py @@ -1,9 +1,10 @@ """Offline skeleton tests: input surface parity + URL classification. No network. Locks the two invariants the reference-compatible surface promises — -no auth fields ever, and additive ``extra="allow"`` parity — plus the full +no auth fields ever, and additive ``extra="allow"`` parity — plus the ``url_resolver`` classification/normalization table (``_u/`` and profilecard -stripping, story→profile, ID-only locations, numeric post-ID flagging). +stripping, story→profile, numeric post-ID flagging). Hashtag/place URLs are +login-walled and deliberately resolve to ``None``. """ from __future__ import annotations @@ -32,7 +33,7 @@ def test_input_has_no_auth_fields(): def test_input_defaults(): model = InstagramScrapeInput() assert model.resultsType == "posts" - assert model.searchType == "hashtag" + assert model.searchType == "profile" assert model.directUrls == [] assert model.addParentData is False @@ -70,19 +71,14 @@ def test_resolve_post_and_reel(): assert r.kind == "reel" and r.value == "Cxyz" -def test_resolve_hashtag(): - r = resolve_url("https://www.instagram.com/explore/tags/crossfit/") - assert r.kind == "hashtag" and r.value == "crossfit" - - -def test_resolve_place_with_slug_and_id_only(): - with_slug = resolve_url( - "https://www.instagram.com/explore/locations/7538318/copenhagen/" +def test_resolve_hashtag_and_place_unsupported(): + # Login-walled surfaces: they must resolve to None so the orchestrator skips + # them rather than building a target that can only return a login wall. + assert resolve_url("https://www.instagram.com/explore/tags/crossfit/") is None + assert ( + resolve_url("https://www.instagram.com/explore/locations/7538318/copenhagen/") + is None ) - assert with_slug.kind == "place" and with_slug.value == "7538318" - assert with_slug.slug == "copenhagen" - id_only = resolve_url("https://www.instagram.com/explore/locations/7538318/") - assert id_only.kind == "place" and id_only.value == "7538318" def test_resolve_strips_u_and_profilecard(): From 949cc1b29f7bb795b330893e2cee783c77c1b185 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:26 +0530 Subject: [PATCH 111/160] chore(scripts): update Instagram e2e scraper script --- .../scripts/e2e_instagram_scraper.py | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/surfsense_backend/scripts/e2e_instagram_scraper.py b/surfsense_backend/scripts/e2e_instagram_scraper.py index 95625fd06..c81075932 100644 --- a/surfsense_backend/scripts/e2e_instagram_scraper.py +++ b/surfsense_backend/scripts/e2e_instagram_scraper.py @@ -14,11 +14,14 @@ It: approach is invalid — later steps are skipped. Step 1 — scrape a profile's posts. Step 2 — scrape a profile's reels. - Step 3 — fetch comments for a discovered post URL. - Step 4 — fetch profile / hashtag / place details. - Step 5 — run a discovery search. + Step 3 — anonymous single-post extraction for a discovered ``/p/`` URL. + Step 4 — fetch profile details. + Step 5 — run a profile discovery search (Google-backed). Step 6 — dump trimmed, PII-anonymized raw fixtures into tests/unit/platforms/instagram/fixtures/ for the offline parser tests. + +Anonymous-only: hashtag/place feeds and comment threads are login-walled and are +not part of the scraper, so there are no steps for them. """ import asyncio @@ -41,15 +44,15 @@ from app.proprietary.platforms.instagram import ( # noqa: E402 scrape_instagram, ) from app.proprietary.platforms.instagram.fetch import ( # noqa: E402 + fetch_html, fetch_json, proxy_session, warm_session, ) +from app.proprietary.platforms.instagram.url_resolver import resolve_url # noqa: E402 _PROFILE = "natgeo" -_HASHTAG = "https://www.instagram.com/explore/tags/travel/" -_PLACE = "https://www.instagram.com/explore/locations/213385402/new-york-new-york/" -_SEARCH_TERM = "coffee" +_SEARCH_TERM = "national geographic" _FIXTURE_DIR = ( _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures" @@ -129,30 +132,27 @@ async def step2_reels() -> bool: return _check("reels returned items", len(items) >= 0, f"{len(items)} reels") -async def step3_comments(post_url: str | None) -> bool: - _hr("STEP 3 — comments for a post") +async def step3_single_post(post_url: str | None) -> bool: + _hr("STEP 3 — single-post extraction for a /p/ URL") if not post_url: return _check("had a post URL", False, "step 1 found no post") items = await scrape_instagram( InstagramScrapeInput( - resultsType="comments", directUrls=[post_url], resultsLimit=10 + resultsType="posts", directUrls=[post_url], resultsLimit=1 ), - limit=10, + limit=1, ) - print(f" {len(items)} comments for {post_url}") - return _check("comments returned", len(items) >= 0, f"{len(items)} comments") + got = items[0] if items else {} + print(f" {len(items)} item for {post_url} | owner={got.get('ownerUsername')}") + return _check("single post returned an item", len(items) > 0, post_url) async def step4_details() -> bool: - _hr("STEP 4 — profile / hashtag / place details") + _hr("STEP 4 — profile details") items = await scrape_instagram( InstagramScrapeInput( resultsType="details", - directUrls=[ - f"https://www.instagram.com/{_PROFILE}/", - _HASHTAG, - _PLACE, - ], + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], ), limit=10, ) @@ -162,12 +162,12 @@ async def step4_details() -> bool: async def step5_search() -> bool: - _hr("STEP 5 — discovery search") + _hr("STEP 5 — profile discovery search (Google-backed)") items = await scrape_instagram( InstagramScrapeInput( resultsType="posts", search=_SEARCH_TERM, - searchType="hashtag", + searchType="profile", searchLimit=3, resultsLimit=3, ), @@ -177,7 +177,7 @@ async def step5_search() -> bool: return _check("search returned results", len(items) >= 0, f"{len(items)} items") -async def step6_dump_fixtures() -> bool: +async def step6_dump_fixtures(post_url: str | None) -> bool: _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests") profile = await fetch_json( "api/v1/users/web_profile_info/", {"username": _PROFILE} @@ -189,6 +189,17 @@ async def step6_dump_fixtures() -> bool: json.dumps(_anonymize(profile)), encoding="utf-8" ) wrote.append("profile.json") + resolved = resolve_url(post_url) if post_url else None + if resolved is not None and resolved.kind in ("post", "reel"): + html = await fetch_html(f"p/{resolved.value}/") + if html: + (_FIXTURE_DIR / "post.json").write_text( + json.dumps( + {"url": post_url, "shortcode": resolved.value, "html": html} + ), + encoding="utf-8", + ) + wrote.append("post.json") return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}") @@ -214,10 +225,10 @@ async def main() -> int: results.append(await step1_posts()) results.append(await step2_reels()) post_url = await _first_post_url() - results.append(await step3_comments(post_url)) + results.append(await step3_single_post(post_url)) results.append(await step4_details()) results.append(await step5_search()) - results.append(await step6_dump_fixtures()) + results.append(await step6_dump_fixtures(post_url)) _hr("SUMMARY") print(f" {sum(results)}/{len(results)} steps passed") return 0 if all(results) else 1 From 4bd3ec275c2b4b16d417a5b88724a0e202bfb748 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:50:26 +0530 Subject: [PATCH 112/160] delete(instagram): remove obsolete post fixture for Instagram tests --- .../tests/unit/platforms/instagram/fixtures/post.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json diff --git a/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json b/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json deleted file mode 100644 index 5eba4b5ad..000000000 --- a/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json +++ /dev/null @@ -1 +0,0 @@ -{"url": "https://www.instagram.com/p/Dan5IDtDE2I/", "shortcode": "Dan5IDtDE2I", "html": "\n\n\n\n\n\n\n\nInstagram\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_before_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"onViewportGuessValidation\",null,[[{\"dimension\":\"width\",\"numPixels\":767,\"operation\":\"max\",\"result\":false},{\"dimension\":\"width\",\"numPixels\":1264,\"operation\":\"min\",\"result\":true}],{\"width_px\":1366,\"height_px\":722,\"guess_source\":\"fallback\"}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"replayStyleInjectsForSSR\",null,[[]]]]}\n
Close
\"natgeodocs's
Never miss a post from natgeodocs
Sign up for Instagram to stay in the loop.
\"Photo
\"natgeo's
\"natgeodocs's
More options

\"natgeodocs's
natgeodocs
Verified
\u00a0Edited\u2022
The true story of four children who survived 40 days and 40 nights stranded in the Colombian rainforest. The siblings recount what happened in #LostInTheJungle, now streaming on @disneyplus and @hulu.
\"jeffred441's
Everyone will definitely going to enjoy watching Lost in the Jungle on @hulu and @disneyplus
Reply
\"saneehaazeemahmed's
Fav survival story
Reply
\"daikon.do's
Eldest sister final boss
10 likes
Reply
\"fourmiboys's
This was great, sorry they had to go through it.
1 like
Reply
\"babs_thecreator's
Incredible \u2764\ufe0f
Reply
\"urss_chhavi01's
\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c
1 like
Reply
\"urss_chhavi01's
\ud83d\ude0d\u2764\ufe0f\u2764\ufe0f\ud83d\ude0d
1 like
Reply
\"joyetha_assam_vibes's
\ud83d\ude2e
Reply
\"alexottscience's
Ayahuasca saved them \ud83c\udf3f
1 like
Reply
\"laurelineorsettiworldwide's
Spectacular! Inspirational!
1 like
Reply
Like
2.3K
Comment
10
Share
Save
Log in to like or comment.

English
Down chevron icon
\u00a9 2026 Instagram from Meta
\n {\"require\":[[\"CometSSRMergedContentInjector\",\"onPayloadReceived\",null,[{\"fizzRootId\":\"ssrb_root_content\",\"id\":\"render_pass_0\",\"payloadType\":\"LAST\",\"readyPreloaders\":[\"adp_PolarisLoggedOutDesktopWWWPostRootContentQueryRelayPreloader_6a5159be0652c5722640683\",\"adp_PolarisWWWLoggedOutDynamicLandingDialogQueryRelayPreloader_6a5159be065439d65551266\"],\"renderPassCount\":0,\"status\":\"success\"},{\"clientRenderErrors\":[],\"productRecoverableErrors\":[]}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_after_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_last_payload_0\"]]]}\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"} \ No newline at end of file From 467f51fd5c8a14e402adda613ff581a4dfc05197 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:50:43 +0530 Subject: [PATCH 113/160] chore(.gitignore): add Instagram test fixtures for unit tests --- surfsense_backend/.gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index bda5961fe..13c2d40e3 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -15,4 +15,7 @@ celerybeat-schedule.* celerybeat-schedule.dir celerybeat-schedule.bak /app/config/global_llm_config.yaml -app/templates/_generated/ \ No newline at end of file +app/templates/_generated/ + +/tests/unit/platforms/instagram/fixtures/post.json +/tests/unit/platforms/instagram/fixtures/profile.json \ No newline at end of file From d41ccb7edd70a1774f3c6c307bfebc7903da1345 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:56:29 +0530 Subject: [PATCH 114/160] feat(instagram): enhance Open Graph parsing for anonymous posts - Added support for extracting likes, comments, username, timestamp, and caption from Open Graph meta tags. - Implemented fallback mechanisms to ensure graceful degradation when expected data is missing. - Updated unit tests to validate new parsing logic and ensure robustness against unrecognized formats. --- .../platforms/instagram/parsers.py | 85 ++++++++++++++++--- .../unit/platforms/instagram/test_parsers.py | 35 +++++++- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index 55884537d..b4c2aeba3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -15,6 +15,7 @@ parity is additive. from __future__ import annotations +import html import json import re from datetime import UTC, datetime @@ -185,10 +186,70 @@ _LDJSON_RE = re.compile( _OG_RE = re.compile( r' on Instagram: ". +_OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL) + + +def _og_date_to_iso(value: str) -> str | None: + """``"July 9, 2026"`` -> ``"2026-07-09"`` (date-only; og carries no time).""" + try: + return datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat() + except ValueError: + return None + + +def _clean_caption(raw: str) -> str | None: + """HTML-unescape and unwrap the surrounding quotes off an og caption preview.""" + return html.unescape(raw).strip().strip('"').strip() or None + + +def _parse_og_meta(og: dict[str, str]) -> dict[str, Any]: + """Lift post fields out of Instagram's Open Graph tags (see module notes above). + + Caption + full name come from ``og:title``; counts + username + date from + ``og:description``. Every field is optional and independently guarded, so a + shape we don't recognise yields a partial (or empty) dict instead of raising. + """ + out: dict[str, Any] = {} + desc = og.get("description", "") + title = og.get("title", "") + + counts = _OG_COUNTS_RE.search(desc) + if counts: + out["likes"] = _html_int(counts.group(1)) + out["comments"] = _html_int(counts.group(2)) + + owner_date = _OG_OWNER_DATE_RE.search(desc) + if owner_date: + out["ownerUsername"] = owner_date.group(1).strip().lstrip("@") or None + out["timestamp"] = _og_date_to_iso(owner_date.group(2)) + + tm = _OG_TITLE_RE.match(title) + if tm: + out["ownerFullName"] = tm.group(1).strip() or None + out["caption"] = _clean_caption(tm.group(2)) + elif owner_date: + # No usable og:title: fall back to the caption after og:description's + # date prefix — still clean (the counts/username/date are stripped). + out["caption"] = _clean_caption(desc[owner_date.end():]) + return out def _html_int(value: Any) -> int | None: @@ -278,18 +339,19 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), blocks[0] if blocks else {}, ) + # ld+json wins when present (richer, structured); og fills every gap. On + # anonymous /p/ pages ld+json is absent, so og_meta is the de-facto source. + og_meta = _parse_og_meta(og) + counts = _ld_interaction(node) - if "likes" not in counts or "comments" not in counts: - m = _OG_COUNTS_RE.search(og.get("description", "")) - if m: - counts.setdefault("likes", _html_int(m.group(1)) or 0) - counts.setdefault("comments", _html_int(m.group(2)) or 0) + counts.setdefault("likes", og_meta.get("likes")) + counts.setdefault("comments", og_meta.get("comments")) caption = ( node.get("articleBody") or node.get("caption") or node.get("description") - or og.get("description") + or og_meta.get("caption") ) caption = caption if isinstance(caption, str) else None @@ -313,13 +375,14 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, - "hashtags": _HASHTAG_RE.findall(caption) if caption else [], - "mentions": _MENTION_RE.findall(caption) if caption else [], + "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], + "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], "url": url, "commentsCount": counts.get("comments"), "displayUrl": display_url, "videoUrl": video_url if is_video else None, "likesCount": counts.get("likes"), - "timestamp": node.get("uploadDate") or node.get("datePublished"), - "ownerUsername": _ld_author_username(node), + "timestamp": node.get("uploadDate") or node.get("datePublished") or og_meta.get("timestamp"), + "ownerUsername": _ld_author_username(node) or og_meta.get("ownerUsername"), + "ownerFullName": og_meta.get("ownerFullName"), } diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index a4c781776..d6be49da4 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -116,12 +116,17 @@ def test_parse_post_prefers_ldjson(): def test_parse_post_falls_back_to_og_meta(): + # Anonymous /p/ pages carry no ld+json; everything is lifted from the og + # tags. og:description gives counts + username + date; og:title gives the + # clean caption + full name. Entities in the caption are deduped. html = """ + + content="1,234 likes, 56 comments - natgeo on January 2, 2024: "a caption #wow #wow @buzz"" /> """ item = parse_post(html, url=_POST_URL, shortcode="Cabc") @@ -130,6 +135,34 @@ def test_parse_post_falls_back_to_og_meta(): assert item["commentsCount"] == 56 assert item["displayUrl"] == "https://cdn/i.jpg" assert item["type"] == "Video" + assert item["ownerUsername"] == "natgeo" + assert item["ownerFullName"] == "Nat Geo" + assert item["timestamp"] == "2024-01-02" # og carries date only, no time + assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped + assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution + assert item["mentions"] == ["buzz"] + + +def test_parse_post_og_degrades_without_crashing(): + # A shape we don't recognise (hidden likes / a non-English locale that + # slipped the en-US header) must yield a partial item with None fields, + # never an exception or a caption polluted with the counts/date prefix. + html = """ + + + + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None # og:image present -> still emits + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["likesCount"] is None + assert item["commentsCount"] is None + assert item["ownerUsername"] is None + assert item["timestamp"] is None + assert item["caption"] is None # unrecognised prefix -> no pollution def test_parse_post_returns_none_without_surfaces(): From 457be1871c7e09c5a5b31cd962494ac37c7869ce Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:06:04 +0530 Subject: [PATCH 115/160] feat(instagram): improve parsing of Instagram media IDs and mentions - Enhanced the regular expression for mentions to prevent trailing punctuation from being included in handles. - Added support for extracting media IDs from deep-link meta tags in anonymous posts. - Updated unit tests to validate the new media ID extraction and ensure proper handling of mentions. --- .../proprietary/platforms/instagram/parsers.py | 16 ++++++++++++++-- .../unit/platforms/instagram/test_parsers.py | 12 +++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index b4c2aeba3..f28ca58e0 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -23,7 +23,10 @@ from typing import Any _BASE = "https://www.instagram.com" _HASHTAG_RE = re.compile(r"#(\w+)") -_MENTION_RE = re.compile(r"@([\w.]+)") +# Instagram handles are letters/digits/period/underscore but never start or end +# with a period, so anchor both ends to alphanumerics/underscore — otherwise +# trailing sentence punctuation ("@hulu.") leaks into the handle. +_MENTION_RE = re.compile(r"@([A-Za-z0-9_](?:[A-Za-z0-9_.]*[A-Za-z0-9_])?)") _TYPE_MAP = { "GraphImage": "Image", "GraphVideo": "Video", @@ -205,6 +208,10 @@ _OG_OWNER_DATE_RE = re.compile( # og:title is the cleaner caption source (no counts/date prefix): the caption is # everything after " on Instagram: ". _OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL) +# The numeric media id (pk) rides in the App Link deep-link meta tags +# (al:ios:url / al:android:url = "instagram://media?id=") on anonymous pages, +# even though og:* and ld+json omit it. +_MEDIA_ID_RE = re.compile(r"instagram://media\?id=(\d+)") def _og_date_to_iso(value: str) -> str | None: @@ -370,8 +377,13 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d else None ) or og.get("image") + media_id = node.get("identifier") if isinstance(node.get("identifier"), str) else None + if media_id is None: + id_match = _MEDIA_ID_RE.search(html) + media_id = id_match.group(1) if id_match else None + return { - "id": node.get("identifier") if isinstance(node.get("identifier"), str) else None, + "id": media_id, "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index d6be49da4..73e043671 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -123,14 +123,16 @@ def test_parse_post_falls_back_to_og_meta(): + + content="Nat Geo on Instagram: "a caption #wow #wow @buzz."" /> + content="1,234 likes, 56 comments - natgeo on January 2, 2024: "a caption #wow #wow @buzz."" /> """ item = parse_post(html, url=_POST_URL, shortcode="Cabc") assert item is not None + assert item["id"] == "3938367641542741384" # numeric pk from al:ios:url meta assert item["likesCount"] == 1234 assert item["commentsCount"] == 56 assert item["displayUrl"] == "https://cdn/i.jpg" @@ -138,9 +140,9 @@ def test_parse_post_falls_back_to_og_meta(): assert item["ownerUsername"] == "natgeo" assert item["ownerFullName"] == "Nat Geo" assert item["timestamp"] == "2024-01-02" # og carries date only, no time - assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped - assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution - assert item["mentions"] == ["buzz"] + assert item["caption"] == "a caption #wow #wow @buzz." # @ -> @, unescaped + assert item["hashtags"] == ["wow"] # deduped, no @-as-#064 pollution + assert item["mentions"] == ["buzz"] # trailing sentence dot stripped def test_parse_post_og_degrades_without_crashing(): From 5cac6612c39dcb918deee0b0c8369f96c629b2cf Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:17 +0530 Subject: [PATCH 116/160] refactor(instagram): update platform schemas and scraper for tagged media --- .../app/proprietary/platforms/instagram/schemas.py | 2 +- .../app/proprietary/platforms/instagram/scraper.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py index 76ad81d12..63c76da7b 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -19,7 +19,7 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field -InstagramResultsType = Literal["posts", "details", "reels", "mentions"] +InstagramResultsType = Literal["posts", "details", "reels"] InstagramSearchType = Literal["profile", "user"] diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 381b40d5b..0aed53bb3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -16,7 +16,7 @@ Google-backed handle discovery. Login-walled surfaces (hashtag/place feeds, comment threads, IG's native keyword search) are deliberately absent. Flows are selected by ``resultsType``: -- ``posts`` / ``reels`` / ``mentions`` -> media items (profile feed, or a single +- ``posts`` / ``reels`` -> media items (profile feed, or a single ``/p/``/``/reel/`` page, or discovery search) - ``details`` -> profile metadata (by URL or discovery search) @@ -394,7 +394,7 @@ async def iter_instagram( yield item return - # posts / reels / mentions -> media feeds, de-duped by id across targets. + # posts / reels -> media feeds, de-duped by id across targets. jobs = [ _media_flow( r, input_model=input_model, cutoff=cutoff, per_target=per_target From 27d22a9a2a081ac5ddd7482cfa4fe1dee0a591f5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:20 +0530 Subject: [PATCH 117/160] refactor(instagram): drop mentions from scrape capability --- .../app/capabilities/instagram/scrape/__init__.py | 2 +- .../app/capabilities/instagram/scrape/definition.py | 4 ++-- .../app/capabilities/instagram/scrape/schemas.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/capabilities/instagram/scrape/__init__.py b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py index de8b3c7c4..be8086485 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/__init__.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py @@ -1,3 +1,3 @@ -"""``instagram.scrape`` verb: Instagram URLs / search terms → posts, reels, mentions.""" +"""``instagram.scrape`` verb: Instagram URLs / search terms → posts, reels.""" from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/scrape/definition.py b/surfsense_backend/app/capabilities/instagram/scrape/definition.py index 7b5d9769f..931ddfe01 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/definition.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/definition.py @@ -10,8 +10,8 @@ from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput INSTAGRAM_SCRAPE = Capability( name="instagram.scrape", description=( - "Scrape public Instagram posts, reels, or mentions from profile/post/" - "reel URLs, or discover public profiles via search queries." + "Scrape public Instagram posts or reels from profile/post/reel URLs, " + "or discover public profiles via search queries." ), input_schema=ScrapeInput, output_schema=ScrapeOutput, diff --git a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py index 03e03e55d..f30078d9b 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py @@ -43,9 +43,9 @@ class ScrapeInput(BaseModel): default="profile", description="Discovery kind (profile-only; hashtag/place are login-walled).", ) - result_type: Literal["posts", "reels", "mentions"] = Field( + result_type: Literal["posts", "reels"] = Field( default="posts", - description="Which feed to return. 'mentions' requires profile URLs.", + description="Which feed to return: 'posts' or 'reels'.", ) newer_than: str | None = Field( default=None, From ca7d23d33a6da5b92397a6567ecd88a34ab46b65 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:23 +0530 Subject: [PATCH 118/160] refactor(mcp): align Instagram scraper tool with capability changes --- .../mcp_server/features/scrapers/platforms/instagram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py index 9683e6581..28ad34e66 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -13,7 +13,7 @@ from ....core.workspace_context import WorkspaceContext, WorkspaceParam from ..annotations import SCRAPE from ..capability import run_scraper -ResultType = Literal["posts", "reels", "mentions"] +ResultType = Literal["posts", "reels"] SearchType = Literal["profile", "user"] @@ -50,7 +50,7 @@ def register( ] = "profile", result_type: Annotated[ ResultType, - Field(description="Which feed to return. 'mentions' needs profile URLs."), + Field(description="Which feed to return: 'posts' or 'reels'."), ] = "posts", max_items: Annotated[ int, Field(ge=1, description="Maximum items to return across sources.") From c001f4b16e724e198f21e2fc67b59ed3afb9d2df Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:25 +0530 Subject: [PATCH 119/160] feat(subagent): update Instagram subagent system prompt --- .../subagents/builtins/instagram/system_prompt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md index 617a376b7..42713cc38 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -12,7 +12,7 @@ Answer the delegated question from live Instagram data gathered with your verbs, -- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). Hashtag/place URLs are unsupported (login-walled). +- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled). - Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). - Profile metadata (follower counts, bio, post count): call `instagram_details`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. From b75ab81261933f0372ced1f9662614af37aa51eb Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:30 +0530 Subject: [PATCH 120/160] docs(web): refresh Instagram connector docs --- surfsense_web/content/docs/connectors/native/instagram.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_web/content/docs/connectors/native/instagram.mdx b/surfsense_web/content/docs/connectors/native/instagram.mdx index 5881ac832..ad2b14ebe 100644 --- a/surfsense_web/content/docs/connectors/native/instagram.mdx +++ b/surfsense_web/content/docs/connectors/native/instagram.mdx @@ -22,7 +22,7 @@ Provide exactly one source: `urls` **or** `search_queries` (never both). | `urls` | — | Profile, post, or reel URLs or bare profile IDs (max 20 sources per call) | | `search_queries` | — | Discovery keywords resolved to public profiles via Google; each returns up to `max_per_target` items | | `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | -| `result_type` | `posts` | Which feed to return: `posts`, `reels`, or `mentions` (`mentions` needs profile URLs) | +| `result_type` | `posts` | Which feed to return: `posts` or `reels` | | `newer_than` | — | Only return posts newer than this: `YYYY-MM-DD`, ISO timestamp, or relative (`2 months`), UTC | | `skip_pinned_posts` | `false` | Exclude pinned posts in posts mode | | `max_per_target` | `10` | Max results per URL or per discovered target | From 3e8f31152e36f099f90b75baaeb135d8f80d92df Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:34 +0530 Subject: [PATCH 121/160] feat(web): update Instagram marketing copy --- surfsense_web/lib/connectors-marketing/instagram.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 39a4a04cb..4c779b591 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -88,7 +88,7 @@ export const instagram: ConnectorPageContent = { { title: "Influencer vetting and outreach", description: - "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership, and surface who is already mentioning your brand.", + "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership.", }, ], @@ -161,7 +161,7 @@ export const instagram: ConnectorPageContent = { name: "result_type", type: "string", defaultValue: '"posts"', - description: "Which feed to return: posts, reels, or mentions. 'mentions' requires profile URLs.", + description: "Which feed to return: posts or reels.", }, { name: "newer_than", From b1fe739308a85e5f8e93252afa4ac0dc58a97e94 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:36 +0530 Subject: [PATCH 122/160] docs(instagram): update platform scraper README --- .../app/proprietary/platforms/instagram/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md index e4d7ca0f7..5bb64cdb2 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/README.md +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -26,7 +26,7 @@ Surfaces used: | Flow | Surface | Extractor | |---|---|---| | profile / details | `api/v1/users/web_profile_info/?username=…` (JSON) | `parse_profile` | -| profile feed (posts/reels/mentions) | the media embedded in the same profile JSON | `parse_media` | +| profile feed (posts/reels) | the media embedded in the same profile JSON | `parse_media` | | single post / reel | `/p//` (HTML: ld+json + og-meta) | `parse_post` | | profile discovery | Google `site:instagram.com ` | `resolve_url` | @@ -66,7 +66,7 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. comma-split queries) into targets and fans them out on a pool of warm proxy sessions (`fan_out`, 8-way). Each worker opens one sticky-IP session and warms `csrftoken`/`mid` once, reusing it across the sequential targets it pulls. -2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media items, +2. `resultsType` selects the flow: `posts`/`reels` → media items, `details` → profile metadata. Media items de-dupe by `id` across targets. - A **profile** target → `web_profile_info` JSON → `parse_media` over the embedded recent-media edges (feed) or `parse_profile` (details). From 69bf748c2ce693d4995dbe1bad557535ea0ec72a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:39 +0530 Subject: [PATCH 123/160] chore(.gitignore): ignore Instagram tagged test fixture --- surfsense_backend/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index 13c2d40e3..b8be5ae6c 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -18,4 +18,5 @@ celerybeat-schedule.bak app/templates/_generated/ /tests/unit/platforms/instagram/fixtures/post.json -/tests/unit/platforms/instagram/fixtures/profile.json \ No newline at end of file +/tests/unit/platforms/instagram/fixtures/profile.json +/tests/unit/platforms/instagram/fixtures/tagged.json \ No newline at end of file From cb4adab852dbfdb780e21a1106b2bf3b9647f569 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:10 +0530 Subject: [PATCH 124/160] feat(instagram): extract PolarisMedia relay JSON with carousel, tags, and location --- .../platforms/instagram/parsers.py | 372 ++++++++++++------ .../platforms/instagram/scraper.py | 7 +- 2 files changed, 264 insertions(+), 115 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index f28ca58e0..5c4e8ea03 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -35,6 +35,10 @@ _TYPE_MAP = { "XDTGraphVideo": "Video", "XDTGraphSidecar": "Sidecar", } +# Mobile v1 ``media_type``: 1 = image, 2 = video, 8 = carousel/sidecar. Used by +# the single-post relay parser (the embedded PolarisMedia blob uses this int, not +# the GraphQL ``__typename`` the profile feed uses). +_MEDIA_TYPE = {1: "Image", 2: "Video", 8: "Sidecar"} def _int(value: Any) -> int | None: @@ -108,6 +112,80 @@ def _shortcode(node: dict[str, Any]) -> str | None: return None +def _user_ref(user: Any) -> dict[str, Any] | None: + """A trimmed public-user dict (tagged users / coauthor producers), or None. + + Normalizes the two anonymous dialects: the profile feed nests the handle + under ``edge_media_to_tagged_user...node.user`` / ``coauthor_producers`` while + the single-post relay blob uses ``usertags.in[].user`` — both carry the same + scalar user fields, so this trims them to one shape. + """ + if not isinstance(user, dict): + return None + ref = { + "username": user.get("username"), + "fullName": user.get("full_name"), + "id": user.get("id") or user.get("pk"), + "isVerified": user.get("is_verified"), + "profilePicUrl": user.get("profile_pic_url"), + } + return ref if ref["username"] or ref["id"] else None + + +def _iv2_url(iv2: Any) -> str | None: + """First candidate URL from a mobile ``image_versions2`` container, or None.""" + if isinstance(iv2, dict): + cands = iv2.get("candidates") + if isinstance(cands, list) and cands and isinstance(cands[0], dict): + url = cands[0].get("url") + return url if isinstance(url, str) else None + return None + + +def _location_ref(loc: Any) -> tuple[str | None, str | None]: + """``(name, id)`` from a location node, or ``(None, None)``.""" + if isinstance(loc, dict): + lid = loc.get("id") or loc.get("pk") + return loc.get("name"), (str(lid) if lid is not None else None) + return None, None + + +def _feed_child(node: dict[str, Any]) -> dict[str, Any]: + """Map a profile-feed ``edge_sidecar_to_children`` child to a childPost dict.""" + dims = node.get("dimensions") if isinstance(node.get("dimensions"), dict) else {} + is_video = bool(node.get("is_video")) + return { + "id": node.get("id"), + "shortCode": node.get("shortcode"), + "type": "Video" if is_video else "Image", + "displayUrl": node.get("display_url"), + "videoUrl": node.get("video_url") if is_video else None, + "alt": node.get("accessibility_caption"), + "dimensionsHeight": _int(dims.get("height")), + "dimensionsWidth": _int(dims.get("width")), + } + + +def _relay_child(node: dict[str, Any]) -> dict[str, Any]: + """Map a single-post relay ``carousel_media`` child to a childPost dict.""" + mt = node.get("media_type") + vv = node.get("video_versions") + video_url = ( + vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None + ) + is_video = mt == 2 or bool(video_url) + return { + "id": node.get("id"), + "shortCode": node.get("code"), + "type": _MEDIA_TYPE.get(mt) or ("Video" if is_video else "Image"), + "displayUrl": _iv2_url(node.get("image_versions2")) or node.get("display_uri"), + "videoUrl": video_url, + "alt": node.get("accessibility_caption"), + "dimensionsHeight": _int(node.get("original_height")), + "dimensionsWidth": _int(node.get("original_width")), + } + + def parse_media(node: dict[str, Any]) -> dict[str, Any]: """Map a raw timeline/feed media node to a flat media item dict.""" code = _shortcode(node) @@ -116,6 +194,16 @@ def parse_media(node: dict[str, Any]) -> dict[str, Any]: owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} dims = node.get("dimensions") if isinstance(node.get("dimensions"), dict) else {} is_video = bool(node.get("is_video")) + children = _edges(node.get("edge_sidecar_to_children")) + tagged = [ + ref + for n in _edges(node.get("edge_media_to_tagged_user")) + if (ref := _user_ref(n.get("user"))) + ] + coauthors = [ + ref for c in (node.get("coauthor_producers") or []) if (ref := _user_ref(c)) + ] + loc_name, loc_id = _location_ref(node.get("location")) return { "id": node.get("id"), "type": _TYPE_MAP.get(typename) or ("Video" if is_video else "Image"), @@ -129,14 +217,25 @@ def parse_media(node: dict[str, Any]) -> dict[str, Any]: "dimensionsHeight": _int(dims.get("height")), "dimensionsWidth": _int(dims.get("width")), "displayUrl": node.get("display_url"), + "images": [c.get("display_url") for c in children if c.get("display_url")], + "childPosts": [_feed_child(c) for c in children], "videoUrl": node.get("video_url") if is_video else None, "alt": node.get("accessibility_caption"), "likesCount": _likes_count(node), "videoViewCount": _int(node.get("video_view_count")) if is_video else None, + "videoDuration": node.get("video_duration") if is_video else None, "timestamp": _utc_from_sec(node.get("taken_at_timestamp")), "ownerUsername": owner.get("username"), "ownerId": owner.get("id") or node.get("owner_id"), "ownerFullName": owner.get("full_name"), + "isPinned": bool(node.get("pinned_for_users")), + "productType": node.get("product_type"), + "paidPartnership": node.get("is_paid_partnership"), + "taggedUsers": tagged, + "coauthorProducers": coauthors, + "musicInfo": node.get("clips_music_attribution_info"), + "locationName": loc_name, + "locationId": loc_id, "isCommentsDisabled": node.get("comments_disabled"), } @@ -145,6 +244,9 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]: """Map a raw ``web_profile_info`` ``data.user`` to a flat profile item dict.""" username = user.get("username") latest = [parse_media(n) for n in _edges(user.get("edge_owner_to_timeline_media"))] + related = [ + ref for n in _edges(user.get("edge_related_profiles")) if (ref := _user_ref(n)) + ] return { "detailKind": "profile", "id": user.get("id"), @@ -164,32 +266,34 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]: "verified": user.get("is_verified"), "profilePicUrl": user.get("profile_pic_url"), "profilePicUrlHD": user.get("profile_pic_url_hd"), + "relatedProfiles": related, "latestPosts": latest, } # Anonymous single-post extraction (/p//, /reel//) -------- # -# Instagram serves logged-out visitors the post's public metadata inside the +# Instagram serves logged-out visitors the post's full metadata inside the # document itself, not via a JSON XHR (the ``?__a=1`` API 404s / login-walls for -# anonymous callers). Two durable, anonymous surfaces carry it: -# 1. ``', - re.IGNORECASE | re.DOTALL, +_APP_JSON_RE = re.compile( + r'', re.DOTALL ) _OG_RE = re.compile( r' int | None: return None -def _ldjson_blocks(html: str) -> list[dict[str, Any]]: - """Parse every ``application/ld+json`` script block into dicts.""" - out: list[dict[str, Any]] = [] - for raw in _LDJSON_RE.findall(html): - try: - data = json.loads(raw.strip()) - except (ValueError, TypeError): - continue - for node in data if isinstance(data, list) else [data]: - if isinstance(node, dict): - out.append(node) - return out - - def _og_tags(html: str) -> dict[str, str]: """Map ``og:`` -> content for the post document.""" return {k.lower(): v for k, v in _OG_RE.findall(html)} -def _ld_interaction(node: dict[str, Any]) -> dict[str, int]: - """Pull like/comment counts out of schema.org ``interactionStatistic``.""" - stats = node.get("interactionStatistic") - items = stats if isinstance(stats, list) else [stats] if stats else [] - out: dict[str, int] = {} - for stat in items: - if not isinstance(stat, dict): - continue - itype = str(stat.get("interactionType") or "") - count = _html_int(stat.get("userInteractionCount")) - if count is None: - continue - if "Like" in itype: - out["likes"] = count - elif "Comment" in itype: - out["comments"] = count - return out +def _find_media(root: Any, shortcode: str | None) -> dict[str, Any] | None: + """Depth-first search a JSON tree for the post's mobile-v1 media object. - -def _ld_author_username(node: dict[str, Any]) -> str | None: - """Owner handle from a schema.org ``author`` (alternateName / identifier).""" - author = node.get("author") - author = author[0] if isinstance(author, list) and author else author - if not isinstance(author, dict): - return None - for key in ("alternateName", "identifier", "name"): - val = author.get(key) - if isinstance(val, dict): - val = val.get("value") - if isinstance(val, str) and val.strip(): - return val.strip().lstrip("@") or None + Matches on ``code == shortcode`` (so a carousel *child* or a related post + can't be picked instead of the target) plus ``taken_at`` and an id, which + together uniquely identify the top-level ``PolarisMedia`` node. + """ + stack = [root] + while stack: + cur = stack.pop() + if isinstance(cur, dict): + if ( + cur.get("taken_at") is not None + and ("pk" in cur or "id" in cur) + and (shortcode is None or cur.get("code") == shortcode) + ): + return cur + stack.extend(cur.values()) + elif isinstance(cur, list): + stack.extend(cur) return None -def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> dict[str, Any] | None: +def _relay_media(html: str, shortcode: str | None) -> dict[str, Any] | None: + """Locate the embedded ``PolarisMedia`` object for this post, or ``None``. + + The logged-out media payload is inlined as one of ~40 ``application/json`` + script blocks. We only ``json.loads`` blocks that mention ``taken_at`` (and + the shortcode when known) so a single post fetch doesn't parse every blob. + """ + for raw in _APP_JSON_RE.findall(html): + if "taken_at" not in raw: + continue + if shortcode and shortcode not in raw: + continue + try: + data = json.loads(raw) + except (ValueError, TypeError): + continue + media = _find_media(data, shortcode) + if media is not None: + return media + return None + + +def _media_from_relay( + media: dict[str, Any], *, url: str, shortcode: str | None +) -> dict[str, Any]: + """Map an embedded mobile-v1 ``PolarisMedia`` object to a flat media item. + + Same output shape as :func:`parse_media` (so it flows through + ``InstagramMediaItem`` unchanged), sourced from the relay dialect + (``user``/``taken_at``/``usertags.in``/``carousel_media``/flat counts). + """ + mt = media.get("media_type") + cap = media.get("caption") + caption = ( + cap.get("text") if isinstance(cap, dict) else (cap if isinstance(cap, str) else None) + ) + carousel = media.get("carousel_media") + carousel = [c for c in carousel if isinstance(c, dict)] if isinstance(carousel, list) else [] + vv = media.get("video_versions") + video_url = ( + vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None + ) + is_video = mt == 2 or bool(video_url) + owner = media.get("user") if isinstance(media.get("user"), dict) else {} + tagged = [ + ref + for t in ((media.get("usertags") or {}).get("in") or []) + if isinstance(t, dict) and (ref := _user_ref(t.get("user"))) + ] + coauthors = [ + ref for c in (media.get("coauthor_producers") or []) if (ref := _user_ref(c)) + ] + loc_name, loc_id = _location_ref(media.get("location")) + # The relay ``id`` is ``POLARIS_``; strip the prefix so single-post ids + # match the numeric pk that og-fallback + the al:ios meta also yield. + ident = media.get("id") + if isinstance(ident, str) and ident.startswith("POLARIS_"): + ident = ident[len("POLARIS_") :] + pk = media.get("pk") + media_id = ident or (str(pk) if pk is not None else None) + return { + "id": media_id, + "type": _MEDIA_TYPE.get(mt) or ("Video" if is_video else "Image"), + "shortCode": media.get("code") or shortcode, + "caption": caption, + "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], + "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], + "url": url, + "commentsCount": _int(media.get("comment_count")), + "dimensionsHeight": _int(media.get("original_height")), + "dimensionsWidth": _int(media.get("original_width")), + "displayUrl": _iv2_url(media.get("image_versions2")) or media.get("display_uri"), + "images": [ + u + for c in carousel + if (u := _iv2_url(c.get("image_versions2")) or c.get("display_uri")) + ], + "childPosts": [_relay_child(c) for c in carousel], + "videoUrl": video_url, + "alt": media.get("accessibility_caption"), + "likesCount": _int(media.get("like_count")), + "videoViewCount": _int(media.get("view_count") or media.get("play_count")) + if is_video + else None, + "videoDuration": media.get("video_duration") if is_video else None, + "timestamp": _utc_from_sec(media.get("taken_at")), + "ownerUsername": owner.get("username"), + "ownerId": owner.get("id") or owner.get("pk"), + "ownerFullName": owner.get("full_name"), + "productType": media.get("product_type"), + "taggedUsers": tagged, + "coauthorProducers": coauthors, + "locationName": loc_name, + "locationId": loc_id, + } + + +def parse_post( + html: str | None, *, url: str, shortcode: str | None = None +) -> dict[str, Any] | None: """Map an anonymous ``/p//`` (or ``/reel/``) HTML page to a media dict. - Prefers the embedded schema.org ``ld+json`` block, falling back to Open Graph - meta tags for whatever it omits. Returns a dict shaped like - :func:`parse_media` (so it flows through ``InstagramMediaItem`` unchanged), or - ``None`` when the document carries neither surface (e.g. a login interstitial - slipped past the fetch-layer redirect check — the caller treats ``None`` as - "nothing to emit", never a silent success). + Prefers the embedded mobile-v1 ``PolarisMedia`` relay JSON (full fidelity), + falling back to the lossy Open Graph meta tags only if that blob is absent. + Returns a dict shaped like :func:`parse_media` (so it flows through + ``InstagramMediaItem`` unchanged), or ``None`` when the document carries + neither surface (e.g. a login interstitial slipped past the fetch-layer + redirect check — the caller treats ``None`` as "nothing to emit", never a + silent success). """ if not isinstance(html, str) or not html.strip(): return None - blocks = _ldjson_blocks(html) + + media = _relay_media(html, shortcode) + if media is not None: + return _media_from_relay(media, url=url, shortcode=shortcode) + + # Fallback: no embedded relay blob -> Open Graph meta only. og = _og_tags(html) - if not blocks and not og: + if not og: return None - - node = next( - (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), - blocks[0] if blocks else {}, - ) - # ld+json wins when present (richer, structured); og fills every gap. On - # anonymous /p/ pages ld+json is absent, so og_meta is the de-facto source. og_meta = _parse_og_meta(og) - - counts = _ld_interaction(node) - counts.setdefault("likes", og_meta.get("likes")) - counts.setdefault("comments", og_meta.get("comments")) - - caption = ( - node.get("articleBody") - or node.get("caption") - or node.get("description") - or og_meta.get("caption") - ) - caption = caption if isinstance(caption, str) else None - - video = node.get("video") - video = video[0] if isinstance(video, list) and video else video - video_url = ( - video.get("contentUrl") if isinstance(video, dict) else None - ) or og.get("video") + caption = og_meta.get("caption") + video_url = og.get("video") is_video = bool(video_url) or og.get("type") == "video.other" - - image = node.get("image") - image = image[0] if isinstance(image, list) and image else image - display_url = ( - image.get("url") if isinstance(image, dict) else image - if isinstance(image, str) - else None - ) or og.get("image") - - media_id = node.get("identifier") if isinstance(node.get("identifier"), str) else None - if media_id is None: - id_match = _MEDIA_ID_RE.search(html) - media_id = id_match.group(1) if id_match else None - + id_match = _MEDIA_ID_RE.search(html) return { - "id": media_id, + "id": id_match.group(1) if id_match else None, "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], "url": url, - "commentsCount": counts.get("comments"), - "displayUrl": display_url, + "commentsCount": og_meta.get("comments"), + "displayUrl": og.get("image"), "videoUrl": video_url if is_video else None, - "likesCount": counts.get("likes"), - "timestamp": node.get("uploadDate") or node.get("datePublished") or og_meta.get("timestamp"), - "ownerUsername": _ld_author_username(node) or og_meta.get("ownerUsername"), + "likesCount": og_meta.get("likes"), + "timestamp": og_meta.get("timestamp"), + "ownerUsername": og_meta.get("ownerUsername"), "ownerFullName": og_meta.get("ownerFullName"), } diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 0aed53bb3..a4c22bde7 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -255,9 +255,10 @@ async def _media_flow( return if resolved.kind in ("post", "reel"): # Single-post extraction: the anonymous ``?__a=1`` JSON API 404s/login- - # walls, but the public /p// document embeds the post's og-meta + - # ld+json, which parse_post reads. Numeric-ID URLs can't be keyed this - # way (the page needs the shortCode), so they're skipped upstream. + # walls, but the public /p// document embeds the mobile-v1 + # PolarisMedia JSON (og-meta fallback), which parse_post reads. Numeric-ID + # URLs can't be keyed this way (the page needs the shortCode), so they're + # skipped upstream. if resolved.numeric_post_id: return html = await fetch_html(f"p/{resolved.value}/") From 452fe30aa4606f53687757001e163502cb385b06 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:15 +0530 Subject: [PATCH 125/160] refactor(instagram): drop login-walled comment fields and note search_type aliasing --- .../app/proprietary/platforms/instagram/schemas.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py index 63c76da7b..930a2acb9 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -20,6 +20,9 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field InstagramResultsType = Literal["posts", "details", "reels"] +# Kept as distinct values for actor-spec parity, but under anonymous Google-backed +# discovery ``user`` and ``profile`` are aliases: both resolve to profile targets +# (IG's own hashtag/place/keyword search is login-walled). InstagramSearchType = Literal["profile", "user"] @@ -64,7 +67,12 @@ class _ItemBase(BaseModel): class InstagramMediaItem(_ItemBase): - """A post / reel / mention. One flat schema per the actor FAQ.""" + """A post or reel. One flat schema per the actor FAQ. + + ``firstComment``/``latestComments`` are intentionally absent: comment + *content* is login-walled (only the anonymous comment *count* is exposed, as + ``commentsCount``), so this scraper can never source them. + """ id: str | None = None type: Literal["Image", "Video", "Sidecar"] | None = None @@ -74,8 +82,6 @@ class InstagramMediaItem(_ItemBase): mentions: list[str] = Field(default_factory=list) url: str | None = None commentsCount: int | None = None - firstComment: str | None = None - latestComments: list[dict[str, Any]] = Field(default_factory=list) dimensionsHeight: int | None = None dimensionsWidth: int | None = None displayUrl: str | None = None From 4bc3d43b5639c24b5cac137c430ee2abeaa28c38 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:23 +0530 Subject: [PATCH 126/160] refactor(instagram): remove unused share-link redirect resolver --- .../proprietary/platforms/instagram/fetch.py | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index e2b7c5818..41f692bf1 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -297,25 +297,6 @@ async def _get_page(session: Any, url: str) -> Any: ) -async def resolve_redirect(url: str) -> str | None: - """Follow a ``share/`` short URL to its canonical target, or ``None``. - - ``share/`` links redirect to the real post/profile URL; the resolver records - the original as ``redirectedFromUrl``. Best-effort: returns the final URL - when the session exposes it, else ``None``. - """ - holder = _current_session.get() - if holder is None: - async with proxy_session(): - return await resolve_redirect(url) - with suppress(Exception): - page = await _get_page(holder.session, url) - final = getattr(page, "url", None) - if isinstance(final, str) and final and final != url: - return final - return None - - async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None: """GET an Instagram web endpoint through a warmed session; parse JSON. @@ -331,7 +312,8 @@ async def fetch_html(path: str, params: dict[str, Any] | None = None) -> str | N Same warm/rotate/backoff resilience as :func:`fetch_json` (a login-wall redirect still raises :class:`InstagramAccessBlockedError`), but hands back the raw HTML body for the pages that embed their data in the document - (``/p//`` og-meta / ld+json) instead of a JSON XHR endpoint. + (``/p//`` embedded PolarisMedia JSON / og-meta) instead of a JSON + XHR endpoint. """ return await _fetch(path, params, _page_text) From d3c65a37b148e7e5c0317a3d9ecd77842416fd52 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:28 +0530 Subject: [PATCH 127/160] refactor(instagram): drop unused slug field and mark share links unsupported --- .../app/proprietary/platforms/instagram/url_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py index aec46a678..08f5c6553 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py @@ -13,7 +13,8 @@ Normalization rules (from the reference spec): - Numeric post-ID URLs cannot be single-post-extracted anonymously (the HTML page keys on the shortCode), so they resolve with ``numeric_post_id`` set and the media flow skips them. -- ``share/`` redirect resolution is handled at fetch time (network), not here. +- ``share/`` links are unsupported (they need a network redirect to resolve to a + canonical post/profile URL); pass the resolved ``/p/`` or profile URL instead. """ from __future__ import annotations @@ -40,7 +41,6 @@ class ResolvedUrl: kind: ResolvedKind value: str url: str - slug: str | None = None numeric_post_id: bool = False From 75c3addd2bcd5083e1f4f5a57a9780fc916d7603 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:33 +0530 Subject: [PATCH 128/160] test(instagram): cover PolarisMedia relay parsing and enriched media fields --- .../unit/platforms/instagram/test_parsers.py | 144 +++++++++++++++--- 1 file changed, 122 insertions(+), 22 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index 73e043671..8cba74b3a 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -60,6 +60,59 @@ def test_parse_media_marks_video_type(): assert item["videoViewCount"] == 99 +def test_parse_media_extracts_sidecar_tags_location_pinned(): + # The anonymous profile feed node carries far more than the core fields: + # sidecar children, tagged users, coauthors, location, product type and pin + # state — all populated here from the real GraphQL key shapes. + node = { + "id": "1", + "shortcode": "Cabc", + "__typename": "GraphSidecar", + "taken_at_timestamp": 1_704_164_645, + "edge_media_to_caption": _edge([{"text": "x #tag @me"}]), + "pinned_for_users": [{"id": "9"}], + "product_type": "feed", + "location": {"id": "55", "name": "Paris"}, + "coauthor_producers": [{"username": "co", "id": "7"}], + "edge_media_to_tagged_user": _edge( + [{"user": {"username": "tg", "id": "3"}, "x": 0.1, "y": 0.2}] + ), + "edge_sidecar_to_children": _edge( + [ + { + "id": "c1", + "shortcode": "s1", + "display_url": "https://cdn/1.jpg", + "dimensions": {"height": 10, "width": 20}, + }, + { + "id": "c2", + "shortcode": "s2", + "is_video": True, + "video_url": "https://cdn/2.mp4", + "display_url": "https://cdn/2.jpg", + }, + ] + ), + } + item = parse_media(node) + assert item["type"] == "Sidecar" + assert item["isPinned"] is True + assert item["productType"] == "feed" + assert item["locationName"] == "Paris" + assert item["locationId"] == "55" + assert item["taggedUsers"][0]["username"] == "tg" + assert item["coauthorProducers"][0]["username"] == "co" + assert item["images"] == ["https://cdn/1.jpg", "https://cdn/2.jpg"] + assert len(item["childPosts"]) == 2 + assert item["childPosts"][1]["type"] == "Video" + assert item["childPosts"][1]["videoUrl"] == "https://cdn/2.mp4" + + +def test_parse_media_unpinned_is_false(): + assert parse_media({"id": "1"})["isPinned"] is False + + def test_parse_profile_flattens_counts_and_latest_posts(): user = { "id": "9", @@ -71,6 +124,9 @@ def test_parse_profile_flattens_counts_and_latest_posts(): "count": 2, "edges": [{"node": {"id": "p1", "shortcode": "A"}}], }, + "edge_related_profiles": _edge( + [{"username": "similar1", "id": "11"}, {"username": "similar2", "id": "12"}] + ), } item = parse_profile(user) assert item["detailKind"] == "profile" @@ -79,40 +135,79 @@ def test_parse_profile_flattens_counts_and_latest_posts(): assert item["followsCount"] == 50 assert item["postsCount"] == 2 assert len(item["latestPosts"]) == 1 + assert [p["username"] for p in item["relatedProfiles"]] == ["similar1", "similar2"] _POST_URL = "https://www.instagram.com/p/Cabc/" -def test_parse_post_prefers_ldjson(): - html = """ - - - - """ +def test_parse_post_prefers_relay_json(): + # Anonymous /p/ pages inline the mobile-v1 PolarisMedia object in an + # application/json script. It's the full-fidelity source (carousel children, + # tagged users, coauthors, location, precise timestamp), preferred over og. + media = { + "pk": "3938367641542741384", + "id": "POLARIS_3938367641542741384", + "code": "Cabc", + "taken_at": 1_704_164_645, + "media_type": 8, + "product_type": "carousel_container", + "like_count": 4200, + "comment_count": 37, + "accessibility_caption": "alt text", + "caption": {"text": "sunset over #bali with @friend @friend"}, + "user": {"username": "natgeo", "full_name": "Nat Geo", "id": "9"}, + "image_versions2": {"candidates": [{"url": "https://cdn/i.jpg"}]}, + "carousel_media": [ + { + "id": "m1", + "code": "c1", + "media_type": 1, + "image_versions2": {"candidates": [{"url": "https://cdn/c1.jpg"}]}, + "original_height": 1080, + "original_width": 1080, + }, + { + "id": "m2", + "code": "c2", + "media_type": 2, + "video_versions": [{"url": "https://cdn/c2.mp4"}], + "image_versions2": {"candidates": [{"url": "https://cdn/c2.jpg"}]}, + }, + ], + "usertags": {"in": [{"position": [0.5, 0.5], "user": {"username": "tagged1", "id": "77"}}]}, + "coauthor_producers": [{"username": "coauthor1", "id": "88", "is_verified": True}], + "location": {"id": "123", "name": "Bali"}, + } + html = ( + '" + ) item = parse_post(html, url=_POST_URL, shortcode="Cabc") assert item is not None - assert item["type"] == "Video" + assert item["id"] == "3938367641542741384" # POLARIS_ prefix stripped + assert item["type"] == "Sidecar" # media_type 8 assert item["shortCode"] == "Cabc" assert item["url"] == _POST_URL - assert item["ownerUsername"] == "natgeo" - assert item["caption"] == "sunset over #bali with @friend" + assert item["caption"] == "sunset over #bali with @friend @friend" assert item["hashtags"] == ["bali"] - assert item["mentions"] == ["friend"] + assert item["mentions"] == ["friend"] # deduped assert item["likesCount"] == 4200 assert item["commentsCount"] == 37 - assert item["videoUrl"] == "https://cdn/v.mp4" - assert item["timestamp"] == "2024-01-02T03:04:05Z" + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["timestamp"].startswith("2024-01-02T") # real epoch -> ISO w/ time + assert item["ownerUsername"] == "natgeo" + assert item["ownerFullName"] == "Nat Geo" + assert item["images"] == ["https://cdn/c1.jpg", "https://cdn/c2.jpg"] + assert len(item["childPosts"]) == 2 + assert item["childPosts"][1]["type"] == "Video" + assert item["childPosts"][1]["videoUrl"] == "https://cdn/c2.mp4" + assert item["taggedUsers"][0]["username"] == "tagged1" + assert item["coauthorProducers"][0]["username"] == "coauthor1" + assert item["locationName"] == "Bali" + assert item["locationId"] == "123" + assert item["productType"] == "carousel_container" def test_parse_post_falls_back_to_og_meta(): @@ -196,3 +291,8 @@ def test_fixture_post_maps(): item = parse_post(raw["html"], url=raw["url"], shortcode=raw.get("shortcode")) assert item is not None, "captured /p/ HTML produced no media item" assert item["url"] == raw["url"] + # The relay blob (not og-meta) should drive extraction: numeric id + a + # precise timestamp with a time component (og-only would be date-only). + assert item["id"] and item["id"].isdigit() + assert item["ownerUsername"] + assert item["timestamp"] and "T" in item["timestamp"] From 819486ac464a88e08e516f0e333e37c1e63a6b70 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:38 +0530 Subject: [PATCH 129/160] docs(instagram): document PolarisMedia extraction and richer media fields --- .../proprietary/platforms/instagram/README.md | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md index 5bb64cdb2..a3c6f01ba 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/README.md +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -27,9 +27,16 @@ Surfaces used: |---|---|---| | profile / details | `api/v1/users/web_profile_info/?username=…` (JSON) | `parse_profile` | | profile feed (posts/reels) | the media embedded in the same profile JSON | `parse_media` | -| single post / reel | `/p//` (HTML: ld+json + og-meta) | `parse_post` | +| single post / reel | `/p//` (embedded mobile-v1 `PolarisMedia` JSON, og-meta fallback) | `parse_post` | | profile discovery | Google `site:instagram.com ` | `resolve_url` | +All of these are richer than the core fields: the feed node and the single-post +relay blob both carry carousel children (`images`/`childPosts`), tagged users, +coauthor producers, location, product type, and pin state; `web_profile_info` +also carries related profiles. Comment **content** stays login-walled — only the +anonymous comment **count** (`commentsCount`) is exposed, so `firstComment` / +`latestComments` are intentionally absent from the item schema. + **Why anonymous-only is a hard constraint.** Live logged-out probes show that Instagram walls the interesting endpoints for anyone without a `sessionid` account cookie: `api/v1/tags/web_info/`, `api/v1/locations/web_info/`, the @@ -57,7 +64,7 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. | `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramProfile`) each with `to_output()`. | | `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json` (JSON) / `fetch_html` (HTML) sharing one resilient `_fetch(path, params, extract)` loop. | | `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`; non-Instagram (and hashtag/place) → `None`. Strips `_u/`, `profilecard/`; story → profile. | -| `parsers.py` | Pure mapping (`parse_media`, `parse_profile`, `parse_post` [ld+json/og], `_edges`). I/O-free. | +| `parsers.py` | Pure mapping (`parse_media`, `parse_profile`, `parse_post` [relay `PolarisMedia` JSON, og-meta fallback], `_edges`). I/O-free. | | `scraper.py` | Orchestrator: `_media_flow`/`_details_flow`/`_discover` (+ `_discover_via_google`), `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | ## How it works @@ -71,8 +78,9 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. - A **profile** target → `web_profile_info` JSON → `parse_media` over the embedded recent-media edges (feed) or `parse_profile` (details). - A **post/reel** target → `fetch_html("p//")` → `parse_post`, which - reads the page's `application/ld+json` (preferred) and Open Graph meta - (fallback). Numeric-ID post URLs are skipped (the page keys on the shortCode). + reads the embedded mobile-v1 `PolarisMedia` JSON (full fidelity) and falls + back to Open Graph meta only if that blob is absent. Numeric-ID post URLs are + skipped (the page keys on the shortCode). 3. `fetch_json` / `fetch_html` warm the session on first use, rotate the IP + re-warm on 401/403, back off on 429, return `None` on 404, and raise `InstagramAccessBlockedError` on a `/accounts/login/` redirect. @@ -103,12 +111,12 @@ Caveats: the `InstagramAccessBlockedError` path, not a bug. - `likesCount` is frequently withheld on anonymous responses (surfaces as `-1` or absent upstream); treat it as best-effort. -- **Single-post extraction** reads whatever the public `/p/` document embeds - (ld+json + og-meta). If Instagram strips both for a given post (private, taken - down, or a login interstitial), `parse_post` returns `None` — an honest empty, - never a fabricated item. ponytail: the embedded-blob shapes can drift; a live - probe that dumps the raw HTML pins them (see Testing) and any change is contained - to `parse_post`. +- **Single-post extraction** reads the mobile-v1 `PolarisMedia` object embedded in + the public `/p/` document (og-meta is a lossy fallback). If Instagram strips both + for a given post (private, taken down, or a login interstitial), `parse_post` + returns `None` — an honest empty, never a fabricated item. ponytail: the + embedded-blob shape can drift; a live probe that dumps the raw HTML pins it (see + Testing) and any change is contained to `_find_media` / `parse_post`. - The `$3.50 / 1k items` default meter assumes the proxy-bytes-per-item measured on the reference targets; re-measure with the scale harness before high-volume use. @@ -116,14 +124,15 @@ Caveats: - Offline unit tests: `tests/unit/platforms/instagram/` — `test_skeleton.py` (schema + URL resolver), `test_parsers.py` (mapping incl. `parse_post` - ld+json/og shapes; fixture-pinned tests skip when the fixture is absent), + relay-JSON/og shapes; fixture-pinned tests skip when the fixture is absent), `test_discovery.py` (Google-backed profile discovery with a fake `scrape_serps`), `test_fetch_resilience.py` (warm / rotate / backoff loop + fan-out with fake sessions, no network), `test_budget.py` (fair-share caps + de-dup). - Stress / accuracy harness (live, needs network + residential proxy): `scripts/stress/stress_instagram_scraper.py` — `--mode live-discovery` (profile discovery accuracy), `--mode probe-post` (dumps a real anonymous `/p/` payload - to `fixtures/post.json` and shows what `parse_post` extracted), and + to `fixtures/post.json` and shows what `parse_post` extracted), `--mode + probe-mentions` (settles that the tagged/`mentions` feed is login-walled), and `--mode accuracy` (field coverage across the profile + single-post flows). ```bash From 42f9f35ec7ca1ab2945f5cc2d9a80baf6514e29d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:45 +0530 Subject: [PATCH 130/160] docs(web): document new media fields and search_type aliasing --- surfsense_web/content/docs/connectors/native/instagram.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_web/content/docs/connectors/native/instagram.mdx b/surfsense_web/content/docs/connectors/native/instagram.mdx index ad2b14ebe..54b8781e0 100644 --- a/surfsense_web/content/docs/connectors/native/instagram.mdx +++ b/surfsense_web/content/docs/connectors/native/instagram.mdx @@ -13,7 +13,7 @@ Instagram login-walls hashtag feeds, place feeds, and comment threads for anonym POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/scrape ``` -Give it Instagram URLs (profile, post `/p/`, or reel `/reel/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, and owner. +Give it Instagram URLs (profile, post `/p/`, or reel `/reel/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, owner, carousel children, tagged users, coauthors, and location. Comment *content* is login-walled, so only the comment *count* is returned (never comment text). Provide exactly one source: `urls` **or** `search_queries` (never both). @@ -21,7 +21,7 @@ Provide exactly one source: `urls` **or** `search_queries` (never both). |-------|---------|-------------| | `urls` | — | Profile, post, or reel URLs or bare profile IDs (max 20 sources per call) | | `search_queries` | — | Discovery keywords resolved to public profiles via Google; each returns up to `max_per_target` items | -| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` (aliases — anonymous discovery resolves both to profiles) | | `result_type` | `posts` | Which feed to return: `posts` or `reels` | | `newer_than` | — | Only return posts newer than this: `YYYY-MM-DD`, ISO timestamp, or relative (`2 months`), UTC | | `skip_pinned_posts` | `false` | Exclude pinned posts in posts mode | @@ -53,7 +53,7 @@ Give it profile URLs (or search queries) and get profile metadata back: follower |-------|---------|-------------| | `urls` | — | Profile URLs or bare profile IDs (max 20) | | `search_queries` | — | Terms to discover public profiles for (provide these OR urls) | -| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` (aliases — anonymous discovery resolves both to profiles) | | `max_items` | `10` | Max detail items to return | Billing is per returned item. From ec3f47abf88a254640e49297ba7d1f65b6845cee Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:51 +0530 Subject: [PATCH 131/160] feat(web): surface carousel, tagged users, and location in connector marketing --- .../lib/connectors-marketing/instagram.tsx | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 4c779b591..d22400b7a 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -232,6 +232,21 @@ export const instagram: ConnectorPageContent = { type: "string", description: "ISO timestamp for when the post was published.", }, + { + name: "images / childPosts", + type: "string[] / object[]", + description: "Carousel (sidecar) children: each child's media URL and metadata.", + }, + { + name: "taggedUsers / coauthorProducers", + type: "object[]", + description: "Users tagged in the media and any co-authors credited on it.", + }, + { + name: "locationName / locationId / productType / isPinned", + type: "string / boolean", + description: "Location tag, product type (feed/clips), and whether the post is pinned.", + }, ], }, @@ -254,7 +269,7 @@ export const instagram: ConnectorPageContent = { { question: "Can I scrape hashtags, places, or comments?", answer: - "No. Instagram login-walls hashtag feeds, place feeds, and comment threads for logged-out visitors, so SurfSense does not offer them. The API focuses on what is reliably public and anonymous: profiles, posts, and reels.", + "No. Instagram login-walls hashtag feeds, place feeds, and comment threads for logged-out visitors, so SurfSense does not offer them. You still get each post's comment count, just not the comment text. The API focuses on what is reliably public and anonymous: profiles, posts, and reels.", }, ], From cb256729a3c5b23b9d6fc4673bc48cd8eabc0461 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 04:38:42 +0530 Subject: [PATCH 132/160] chore: Enhance SurfSense MCP server documentation and toolset by adding TikTok scraping capabilities, including comments, user search, and trending features. Updated README and documentation to reflect the addition of 24 native tools. --- surfsense_mcp/README.md | 4 +++- surfsense_web/app/(home)/mcp-server/page.tsx | 3 +++ surfsense_web/content/docs/how-to/mcp-server.mdx | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index 5bac5f8b2..95ac293d9 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -21,7 +21,9 @@ Connect it two ways: **Scrapers (all platforms)** - `surfsense_web_crawl`, `surfsense_google_search`, `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, - `surfsense_tiktok_scrape`, + `surfsense_instagram_scrape`, `surfsense_instagram_details`, + `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, + `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews` - `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` — retrieve past results in full (useful when a large result was truncated inline) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 3d00c44c9..ce4caf992 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -96,6 +96,9 @@ const TOOL_GROUPS = [ "surfsense_instagram_scrape", "surfsense_instagram_details", "surfsense_tiktok_scrape", + "surfsense_tiktok_comments", + "surfsense_tiktok_user_search", + "surfsense_tiktok_trending", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index 782476d07..71d55ea6c 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 21 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 24 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -133,7 +133,7 @@ Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a pro } ``` -Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 21 tools should appear with a green dot. +Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 24 tools should appear with a green dot. @@ -257,14 +257,14 @@ For self-host (stdio), all settings are environment variables passed by the clie - **401 errors** — the API key is wrong or expired; create a new one. - **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. - **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. -- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 21 tools register without needing a backend. +- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 24 tools register without needing a backend. ## Tools reference | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From 6e0b8516dcf7a085ad6294e11662b81d78d9c023 Mon Sep 17 00:00:00 2001 From: Benebo7 Date: Sat, 11 Jul 2026 14:38:16 -0300 Subject: [PATCH 133/160] fix: correct invalid placeholder values in backed .env.example --- surfsense_backend/.env.example | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 2c0cf38ff..3d2355460 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -120,8 +120,10 @@ STRIPE_RECONCILIATION_BATCH_SIZE=100 # BACKEND_URL=https://api.yourdomain.com # Auth -AUTH_TYPE=GOOGLE or LOCAL -REGISTRATION_ENABLED=TRUE or FALSE +# AUTH_TYPE: GOOGLE or LOCAL +AUTH_TYPE=LOCAL +# REGISTRATION_ENABLED: TRUE or FALSE +REGISTRATION_ENABLED=TRUE # For Google Auth Only GOOGLE_OAUTH_CLIENT_ID=924507538m GOOGLE_OAUTH_CLIENT_SECRET=GOCSV From a6c6b4b3d834e7c112712d87c8f5580f0341634d Mon Sep 17 00:00:00 2001 From: Benebo7 Date: Sat, 11 Jul 2026 14:40:04 -0300 Subject: [PATCH 134/160] docs: Improve contributor Docker setup guide and fix broken link --- CONTRIBUTING.md | 2 +- .../docs/docker-installation/index.mdx | 40 +++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 493f24c02..1998bd74e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ We follow a **branch protection model** to keep `main` stable: ``` 3. **Choose your setup method**: - - **Docker Setup**: Follow the [Docker Setup Guide](./DOCKER_SETUP.md) + - **Docker Setup**: Follow the [Building from Source (Contributors)](https://www.surfsense.com/docs/docker-installation#building-from-source-contributors) section of the Docker Installation guide - **Manual Setup**: Follow the [Installation Guide](https://www.surfsense.com/docs/) 4. **Configure services**: diff --git a/surfsense_web/content/docs/docker-installation/index.mdx b/surfsense_web/content/docs/docker-installation/index.mdx index a2db4e8f5..de65e3699 100644 --- a/surfsense_web/content/docs/docker-installation/index.mdx +++ b/surfsense_web/content/docs/docker-installation/index.mdx @@ -143,14 +143,48 @@ To update SurfSense, see [Updating](/docs/docker-installation/updating). ## Building from Source (Contributors) -If you're contributing to SurfSense and want to build the images from your local checkout, use the dev compose file: +If you're contributing to SurfSense and want to build the images from your local checkout, use the dev compose file. Unlike the production setup above, there's no bundled Caddy proxy — each service publishes its own port directly, and you need **three** separate `.env` files instead of one. + +**Requirements:** Docker with the `buildx` plugin (needed for the multi-stage `Dockerfile`; a plain `docker build` without BuildKit fails). Check with `docker buildx version` — if missing, install `docker-buildx` (or `docker-buildx-plugin` on some distros). ```bash -cd SurfSense/docker +git clone https://github.com//SurfSense.git +cd SurfSense +cp docker/.env.example docker/.env +cp surfsense_backend/.env.example surfsense_backend/.env +cp surfsense_web/.env.example surfsense_web/.env +``` + +`docker/.env` only configures Docker Compose variable substitution — it is **not** passed into the containers. The backend and frontend read their own `.env` files instead (wired via `env_file:` in `docker-compose.dev.yml`). + +Edit before starting: + +- **`SECRET_KEY`** in `surfsense_backend/.env` — required, generate with `openssl rand -base64 32`. Note this is separate from `SECRET_KEY` in `docker/.env`; the dev compose file does not forward one to the other. + +Everything else in the three example files (auth type, Stripe, connector OAuth credentials, messaging bots, proxy settings, etc.) is optional — only needed if you're testing that specific integration. + +```bash +cd docker docker compose -f docker-compose.dev.yml up --build ``` -It builds the backend and frontend from source, publishes raw service ports for debugging, and includes pgAdmin at [http://localhost:5050](http://localhost:5050). There's also a `docker-compose.deps-only.yml` that runs just the dependencies (Postgres, Redis, zero-cache) in Docker while you run the backend and frontend natively — see [Manual Installation](/docs/manual-installation#alternative-let-docker-manage-all-dependencies). +| Service | Port | Notes | +|---------|------|-------| +| `frontend` | `localhost:3000` | Next.js dev server | +| `backend` | `localhost:8000` | FastAPI, `/ready` for healthcheck | +| `pgadmin` | `localhost:5050` | Postgres GUI | +| `zero-cache` | `localhost:4848` (ws) | Real-time sync | +| `celery_worker` | — | Background task processing, no exposed port | +| `celery_beat` | — | Periodic task scheduler, no exposed port | +| `otel-lgtm` | `localhost:3001` | Grafana + Loki + Tempo + Mimir bundle, for tracing/metrics | + +Observability (`otel-lgtm`) isn't needed for regular dev work — it's the heaviest non-essential service, so skip it if you're not debugging traces/metrics: + +```bash +docker compose -f docker-compose.dev.yml up --build db redis zero-cache backend celery_worker celery_beat frontend +``` + +There's also a `docker-compose.deps-only.yml` that runs just the dependencies (Postgres, Redis, zero-cache) in Docker while you run the backend and frontend natively — see [Manual Installation](/docs/manual-installation#alternative-let-docker-manage-all-dependencies). ## Troubleshooting From a172e9c1624ce5ebf095333d8bf2410eb9758c8c Mon Sep 17 00:00:00 2001 From: zzh Date: Sun, 12 Jul 2026 23:25:02 +0800 Subject: [PATCH 135/160] Add raw OpenAI-compatible provider without /v1 normalization --- .../app/services/provider_registry.py | 9 +++++++++ .../unit/services/test_model_connections.py | 16 ++++++++++++++++ .../model-connections/default-connect-form.tsx | 5 ++++- .../model-connections/provider-metadata.tsx | 6 ++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/services/provider_registry.py b/surfsense_backend/app/services/provider_registry.py index 67d1c4db4..6392e38c3 100644 --- a/surfsense_backend/app/services/provider_registry.py +++ b/surfsense_backend/app/services/provider_registry.py @@ -87,6 +87,15 @@ REGISTRY: dict[str, ProviderSpec] = { "bearer", "OpenAI-compatible provider", ), + "openai_compatible_raw": ProviderSpec( + Transport.NATIVE, + "openai", + "none", + None, + True, + "bearer", + "OpenAI-compatible raw endpoint", + ), "lm_studio": ProviderSpec( Transport.OPENAI_COMPATIBLE, "openai", diff --git a/surfsense_backend/tests/unit/services/test_model_connections.py b/surfsense_backend/tests/unit/services/test_model_connections.py index b4e7c18d7..bb5ca318e 100644 --- a/surfsense_backend/tests/unit/services/test_model_connections.py +++ b/surfsense_backend/tests/unit/services/test_model_connections.py @@ -64,6 +64,22 @@ def test_openai_compatible_resolver_uses_explicit_api_base() -> None: assert ensure_v1("http://example.com/v1") == "http://example.com/v1" +def test_openai_compatible_raw_resolver_does_not_append_v1() -> None: + model, kwargs = to_litellm( + { + "provider": "openai_compatible_raw", + "base_url": "https://ark.cn-beijing.volces.com/api/v3", + "api_key": "ark-key", + "extra": {}, + }, + "ep-20260101000000-test", + ) + + assert model == "openai/ep-20260101000000-test" + assert kwargs["api_base"] == "https://ark.cn-beijing.volces.com/api/v3" + assert kwargs["api_key"] == "ark-key" + + def test_ollama_resolver_uses_native_api_base() -> None: model, kwargs = to_litellm( { diff --git a/surfsense_web/components/settings/model-connections/default-connect-form.tsx b/surfsense_web/components/settings/model-connections/default-connect-form.tsx index e3111202d..6ca96ced2 100644 --- a/surfsense_web/components/settings/model-connections/default-connect-form.tsx +++ b/surfsense_web/components/settings/model-connections/default-connect-form.tsx @@ -9,7 +9,10 @@ function baseUrlHint(provider: string) { return "For local servers, use host.docker.internal instead of localhost."; } if (provider === "openai_compatible") { - return "Enter the full endpoint URL."; + return "Enter the full endpoint URL. This provider expects a /v1-compatible endpoint."; + } + if (provider === "openai_compatible_raw") { + return "Enter the exact chat-completions API base URL. SurfSense will not append /v1."; } if (provider === "openai" || provider === "anthropic" || provider === "openrouter") { return "Override only if you route through a proxy or gateway."; diff --git a/surfsense_web/components/settings/model-connections/provider-metadata.tsx b/surfsense_web/components/settings/model-connections/provider-metadata.tsx index 8b8a877b9..0a5cc53c8 100644 --- a/surfsense_web/components/settings/model-connections/provider-metadata.tsx +++ b/surfsense_web/components/settings/model-connections/provider-metadata.tsx @@ -10,6 +10,7 @@ export const PROVIDER_ORDER = [ "ollama_chat", "lm_studio", "openai_compatible", + "openai_compatible_raw", ]; export const PROVIDER_DISPLAY: Record< @@ -37,6 +38,11 @@ export const PROVIDER_DISPLAY: Record< subtitle: "OpenAI-compatible endpoint", iconKey: "custom", }, + openai_compatible_raw: { + name: "OpenAI-Compatible Raw", + subtitle: "Use the exact base URL, no /v1 is appended", + iconKey: "custom", + }, openrouter: { name: "OpenRouter", subtitle: "OpenRouter", From 2ff7ea4cb6b1e11857e2404e8850d2ff9641fbdd Mon Sep 17 00:00:00 2001 From: Thibault Jaigu Date: Mon, 13 Jul 2026 09:42:30 +0100 Subject: [PATCH 136/160] feat: add Requesty as a model provider Add Requesty (https://requesty.ai), an OpenAI-compatible LLM router, as a model provider by mirroring the existing OpenRouter integration. Backend: - app/services/requesty_model_normalizer.py: normalizes Requesty's /v1/models catalogue, mapping its flat capability booleans (supports_tool_calling/ supports_vision/supports_image_generation) and context_window field onto the shared normalized shape (Requesty differs from OpenRouter's architecture + supported_parameters + context_length layout) - provider_registry.py: Requesty ProviderSpec (OpenAI-compatible, base URL https://router.requesty.ai/v1, REQUESTY_API_KEY bearer auth) - model_connection_service.py: key verification + live model discovery - quality_score.py: Requesty score entry - unit tests mirroring the OpenRouter normalizer coverage Frontend: - Requesty provider icon + registration, metadata entry, and base-url hint Signed-off-by: Thibault Jaigu --- .../app/services/model_connection_service.py | 15 ++- .../app/services/provider_registry.py | 10 ++ .../app/services/quality_score.py | 1 + .../app/services/requesty_model_normalizer.py | 123 ++++++++++++++++++ .../test_requesty_model_normalizer.py | 107 +++++++++++++++ .../components/icons/providers/index.ts | 1 + .../components/icons/providers/requesty.svg | 1 + .../default-connect-form.tsx | 7 +- .../model-connections/provider-metadata.tsx | 7 + surfsense_web/lib/provider-icons.tsx | 3 + 10 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/services/requesty_model_normalizer.py create mode 100644 surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py create mode 100644 surfsense_web/components/icons/providers/requesty.svg diff --git a/surfsense_backend/app/services/model_connection_service.py b/surfsense_backend/app/services/model_connection_service.py index cdfd1d725..54d0a3c3f 100644 --- a/surfsense_backend/app/services/model_connection_service.py +++ b/surfsense_backend/app/services/model_connection_service.py @@ -15,6 +15,7 @@ from app.db import Connection, Model, ModelSource from app.services.model_resolver import ensure_v1, to_litellm from app.services.openrouter_model_normalizer import normalize_openrouter_models from app.services.provider_registry import Transport, provider_label, spec_for +from app.services.requesty_model_normalizer import normalize_requesty_models logger = logging.getLogger(__name__) @@ -148,7 +149,7 @@ async def verify_connection(conn: Connection) -> VerifyResult: if spec.transport == Transport.OLLAMA and base_url: url = f"{base_url.rstrip('/')}/api/version" - elif spec.discovery in {"openai_models", "openrouter"} and base_url: + elif spec.discovery in {"openai_models", "openrouter", "requesty"} and base_url: url = f"{ensure_v1(base_url)}/models" elif spec.discovery == "anthropic_models" and base_url: url = f"{base_url.rstrip('/')}/models" @@ -363,6 +364,16 @@ async def _openrouter_models(conn: Connection) -> list[dict[str, Any]]: return normalize_openrouter_models(response.json().get("data", [])) +async def _requesty_models(conn: Connection) -> list[dict[str, Any]]: + base_url = _base_url_or_default(conn) or "https://router.requesty.ai/v1" + async with httpx.AsyncClient(timeout=DISCOVERY_TIMEOUT_SECONDS) as client: + response = await client.get( + f"{ensure_v1(base_url)}/models", headers=_auth_headers(conn) + ) + response.raise_for_status() + return normalize_requesty_models(response.json().get("data", [])) + + def _litellm_static_models(conn: Connection) -> list[dict[str, Any]]: provider = conn.provider prefix = spec_for(provider).litellm_prefix or provider @@ -446,6 +457,8 @@ async def discover_models(conn: Connection) -> list[dict[str, Any]]: results = await _ollama_tags_then_show(conn) elif spec.discovery == "openrouter": results = await _openrouter_models(conn) + elif spec.discovery == "requesty": + results = await _requesty_models(conn) elif spec.discovery == "anthropic_models": results = await _discover_anthropic_models(conn) elif spec.discovery == "openai_models": diff --git a/surfsense_backend/app/services/provider_registry.py b/surfsense_backend/app/services/provider_registry.py index 67d1c4db4..3461500e6 100644 --- a/surfsense_backend/app/services/provider_registry.py +++ b/surfsense_backend/app/services/provider_registry.py @@ -23,6 +23,7 @@ DiscoveryKind = Literal[ "anthropic_models", "bedrock_models", "openrouter", + "requesty", "static", "none", ] @@ -78,6 +79,15 @@ REGISTRY: dict[str, ProviderSpec] = { "bearer", "OpenRouter", ), + "requesty": ProviderSpec( + Transport.OPENAI_COMPATIBLE, + "openai", + "requesty", + "https://router.requesty.ai/v1", + False, + "bearer", + "Requesty", + ), "openai_compatible": ProviderSpec( Transport.OPENAI_COMPATIBLE, "openai", diff --git a/surfsense_backend/app/services/quality_score.py b/surfsense_backend/app/services/quality_score.py index 737dd7c2f..1aa3e7eda 100644 --- a/surfsense_backend/app/services/quality_score.py +++ b/surfsense_backend/app/services/quality_score.py @@ -123,6 +123,7 @@ PROVIDER_PRESTIGE_YAML: dict[str, int] = { "perplexity": 28, "bedrock": 28, "openrouter": 25, + "requesty": 25, "ollama_chat": 12, "custom": 12, } diff --git a/surfsense_backend/app/services/requesty_model_normalizer.py b/surfsense_backend/app/services/requesty_model_normalizer.py new file mode 100644 index 000000000..7710eb174 --- /dev/null +++ b/surfsense_backend/app/services/requesty_model_normalizer.py @@ -0,0 +1,123 @@ +"""Shared Requesty model normalization. + +Requesty (https://router.requesty.ai) is an OpenAI-compatible LLM router. +Its ``/v1/models`` catalogue carries richer, Requesty-specific capability +metadata than a generic OpenAI-compatible ``/models`` response, so keep all +Requesty filtering and capability extraction here -- mirroring +``openrouter_model_normalizer`` -- so GLOBAL catalogue generation and BYOK +discovery agree. + +Unlike OpenRouter, Requesty exposes capabilities as flat booleans +(``supports_tool_calling`` / ``supports_reasoning`` / ``supports_vision`` / +``supports_image_generation``) rather than an ``architecture`` block plus a +``supported_parameters`` array, and it reports context size as +``context_window`` rather than ``context_length``. This module maps those +fields onto the same normalized shape the rest of the backend consumes. +""" + +from __future__ import annotations + +from typing import Any + +from app.db import ModelSource + +MIN_CONTEXT_LENGTH = 100_000 + +EXCLUDED_PROVIDER_SLUGS: set[str] = {"amazon"} +EXCLUDED_MODEL_IDS: set[str] = set() +EXCLUDED_MODEL_SUFFIXES: tuple[str, ...] = ("-deep-research",) + + +def is_image_output_model(model: dict[str, Any]) -> bool: + return bool(model.get("supports_image_generation")) + + +def is_text_output_model(model: dict[str, Any]) -> bool: + # Requesty entries are chat-completion models (``api == "chat"``). Treat a + # model as text output whenever it is not an image-generation model. + return not is_image_output_model(model) + + +def supports_image_input(model: dict[str, Any]) -> bool: + return bool(model.get("supports_vision")) + + +def supports_tool_calling(model: dict[str, Any]) -> bool: + return bool(model.get("supports_tool_calling")) + + +def has_sufficient_context(model: dict[str, Any]) -> bool: + return int(model.get("context_window") or 0) >= MIN_CONTEXT_LENGTH + + +def is_compatible_provider(model: dict[str, Any]) -> bool: + model_id = str(model.get("id") or "") + slug = model_id.split("/", 1)[0] if "/" in model_id else "" + return slug not in EXCLUDED_PROVIDER_SLUGS + + +def is_allowed_model(model: dict[str, Any]) -> bool: + model_id = str(model.get("id") or "") + if model_id in EXCLUDED_MODEL_IDS: + return False + base_id = model_id.split(":")[0] + return not base_id.endswith(EXCLUDED_MODEL_SUFFIXES) + + +def is_requesty_chat_model(model: dict[str, Any]) -> bool: + return ( + "/" in str(model.get("id") or "") + and is_text_output_model(model) + and supports_tool_calling(model) + and has_sufficient_context(model) + and is_compatible_provider(model) + and is_allowed_model(model) + ) + + +def is_requesty_image_model(model: dict[str, Any]) -> bool: + return ( + "/" in str(model.get("id") or "") + and is_image_output_model(model) + and is_compatible_provider(model) + and is_allowed_model(model) + ) + + +def normalize_requesty_models( + raw_models: list[dict[str, Any]], +) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for model in raw_models: + if not is_requesty_chat_model(model): + continue + model_id = str(model.get("id") or "") + normalized.append( + { + "model_id": model_id, + "display_name": model.get("name") or model_id, + "source": ModelSource.DISCOVERED, + "supports_chat": True, + "max_input_tokens": model.get("context_window"), + "supports_image_input": supports_image_input(model), + "supports_tools": supports_tool_calling(model), + "supports_image_generation": False, + "metadata": model, + } + ) + return normalized + + +__all__ = [ + "MIN_CONTEXT_LENGTH", + "has_sufficient_context", + "is_allowed_model", + "is_compatible_provider", + "is_image_output_model", + "is_requesty_chat_model", + "is_requesty_image_model", + "is_text_output_model", + "normalize_requesty_models", + "supports_image_input", + "supports_tool_calling", +] diff --git a/surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py b/surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py new file mode 100644 index 000000000..36b867923 --- /dev/null +++ b/surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py @@ -0,0 +1,107 @@ +"""Unit tests for Requesty model normalization. + +Mirrors the OpenRouter normalizer coverage but exercises Requesty's flat +boolean capability fields (``supports_tool_calling`` / ``supports_vision``) +and ``context_window`` sizing. +""" + +from __future__ import annotations + +import pytest + +from app.services.requesty_model_normalizer import ( + is_requesty_chat_model, + is_requesty_image_model, + normalize_requesty_models, + supports_image_input, + supports_tool_calling, +) + +pytestmark = pytest.mark.unit + + +def _requesty_model( + *, + model_id: str, + context_window: int = 128_000, + tools: bool = True, + vision: bool = False, + image_generation: bool = False, + name: str | None = None, +) -> dict: + """Return a synthetic Requesty ``/v1/models`` entry. + + Only the fields the normalizer inspects are populated; the live payload + carries many more (pricing, ``supports_caching``, ``description``, ...). + """ + return { + "id": model_id, + "name": name or model_id, + "api": "chat", + "object": "model", + "context_window": context_window, + "supports_tool_calling": tools, + "supports_vision": vision, + "supports_image_generation": image_generation, + } + + +def test_chat_model_requires_slash_tools_and_context(): + assert is_requesty_chat_model(_requesty_model(model_id="openai/gpt-4o-mini")) + assert not is_requesty_chat_model( + _requesty_model(model_id="openai/gpt-4o-mini", tools=False) + ) + assert not is_requesty_chat_model( + _requesty_model(model_id="openai/gpt-4o-mini", context_window=8_000) + ) + assert not is_requesty_chat_model(_requesty_model(model_id="bare-model")) + + +def test_excluded_provider_slug_is_filtered(): + assert not is_requesty_chat_model( + _requesty_model(model_id="amazon/nova-pro-v1") + ) + + +def test_image_generation_models_excluded_from_chat_and_flagged(): + image_model = _requesty_model( + model_id="google/gemini-2.5-flash-image", image_generation=True + ) + assert not is_requesty_chat_model(image_model) + assert is_requesty_image_model(image_model) + + +def test_capability_helpers_read_flat_booleans(): + model = _requesty_model( + model_id="anthropic/claude-sonnet-4-5", vision=True, tools=True + ) + assert supports_image_input(model) is True + assert supports_tool_calling(model) is True + + +def test_normalize_maps_context_window_and_capabilities(): + normalized = normalize_requesty_models( + [ + _requesty_model( + model_id="openai/gpt-4o-mini", + context_window=128_000, + vision=True, + name="GPT-4o mini", + ), + _requesty_model(model_id="openai/gpt-4o-mini", tools=False), + _requesty_model( + model_id="black-forest-labs/flux", image_generation=True + ), + ] + ) + + assert len(normalized) == 1 + entry = normalized[0] + assert entry["model_id"] == "openai/gpt-4o-mini" + assert entry["display_name"] == "GPT-4o mini" + assert entry["supports_chat"] is True + assert entry["max_input_tokens"] == 128_000 + assert entry["supports_image_input"] is True + assert entry["supports_tools"] is True + assert entry["supports_image_generation"] is False + assert entry["metadata"]["id"] == "openai/gpt-4o-mini" diff --git a/surfsense_web/components/icons/providers/index.ts b/surfsense_web/components/icons/providers/index.ts index 5c8276e62..f03fbec68 100644 --- a/surfsense_web/components/icons/providers/index.ts +++ b/surfsense_web/components/icons/providers/index.ts @@ -27,6 +27,7 @@ export { default as PerplexityIcon } from "./perplexity.svg"; export { default as QwenIcon } from "./qwen.svg"; export { default as RecraftIcon } from "./recraft.svg"; export { default as ReplicateIcon } from "./replicate.svg"; +export { default as RequestyIcon } from "./requesty.svg"; export { default as SambaNovaIcon } from "./sambanova.svg"; export { default as TogetherAiIcon } from "./togetherai.svg"; export { default as VertexAiIcon } from "./vertexai.svg"; diff --git a/surfsense_web/components/icons/providers/requesty.svg b/surfsense_web/components/icons/providers/requesty.svg new file mode 100644 index 000000000..a804601b6 --- /dev/null +++ b/surfsense_web/components/icons/providers/requesty.svg @@ -0,0 +1 @@ + diff --git a/surfsense_web/components/settings/model-connections/default-connect-form.tsx b/surfsense_web/components/settings/model-connections/default-connect-form.tsx index e3111202d..91f31f2f0 100644 --- a/surfsense_web/components/settings/model-connections/default-connect-form.tsx +++ b/surfsense_web/components/settings/model-connections/default-connect-form.tsx @@ -11,7 +11,12 @@ function baseUrlHint(provider: string) { if (provider === "openai_compatible") { return "Enter the full endpoint URL."; } - if (provider === "openai" || provider === "anthropic" || provider === "openrouter") { + if ( + provider === "openai" || + provider === "anthropic" || + provider === "openrouter" || + provider === "requesty" + ) { return "Override only if you route through a proxy or gateway."; } return undefined; diff --git a/surfsense_web/components/settings/model-connections/provider-metadata.tsx b/surfsense_web/components/settings/model-connections/provider-metadata.tsx index 8b8a877b9..e6c0e8cd8 100644 --- a/surfsense_web/components/settings/model-connections/provider-metadata.tsx +++ b/surfsense_web/components/settings/model-connections/provider-metadata.tsx @@ -7,6 +7,7 @@ export const PROVIDER_ORDER = [ "bedrock", "azure", "openrouter", + "requesty", "ollama_chat", "lm_studio", "openai_compatible", @@ -43,6 +44,12 @@ export const PROVIDER_DISPLAY: Record< iconKey: "openrouter", defaultBaseUrl: "https://openrouter.ai/api/v1", }, + requesty: { + name: "Requesty", + subtitle: "Requesty", + iconKey: "requesty", + defaultBaseUrl: "https://router.requesty.ai/v1", + }, vertex_ai: { name: "Gemini", subtitle: "Google Cloud Vertex AI", iconKey: "vertex_ai" }, }; diff --git a/surfsense_web/lib/provider-icons.tsx b/surfsense_web/lib/provider-icons.tsx index d3e799720..20016fdf2 100644 --- a/surfsense_web/lib/provider-icons.tsx +++ b/surfsense_web/lib/provider-icons.tsx @@ -29,6 +29,7 @@ import { QwenIcon, RecraftIcon, ReplicateIcon, + RequestyIcon, SambaNovaIcon, TogetherAiIcon, VertexAiIcon, @@ -117,6 +118,8 @@ export function getProviderIcon( return ; case "REPLICATE": return ; + case "REQUESTY": + return ; case "SAMBANOVA": return ; case "TOGETHER_AI": From 0d663358842295dd2b15607a70f323236343d85c Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 14:15:38 -0700 Subject: [PATCH 137/160] Update metadata and hero section text to reflect rebranding of SurfSense as NotebookLM for competitive intelligence. Adjusted titles and descriptions for improved clarity and alignment with new positioning. --- surfsense_web/app/layout.tsx | 6 +++--- surfsense_web/components/homepage/hero-section.tsx | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index d9e915d0f..fff38e83b 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -50,7 +50,7 @@ export const metadata: Metadata = { alternates: { canonical: "https://www.surfsense.com", }, - title: "SurfSense - Competitive Intelligence Platform for AI Agents", + title: "SurfSense - NotebookLM for Competitive Intelligence", description: "SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.", keywords: [ @@ -68,7 +68,7 @@ export const metadata: Metadata = { "SurfSense", ], openGraph: { - title: "SurfSense - Competitive Intelligence Platform for AI Agents", + title: "SurfSense - NotebookLM for Competitive Intelligence", description: "SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.", url: "https://www.surfsense.com", @@ -86,7 +86,7 @@ export const metadata: Metadata = { }, twitter: { card: "summary_large_image", - title: "SurfSense - Competitive Intelligence Platform for AI Agents", + title: "SurfSense - NotebookLM for Competitive Intelligence", description: "SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.", creator: "@SurfSenseAI", diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index fc8356041..05db2781f 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -708,7 +708,7 @@ export function HeroSection() { "relative mt-4 max-w-4xl text-left text-4xl font-bold tracking-tight text-balance text-neutral-900 sm:text-5xl md:text-6xl dark:text-neutral-50" )} > - Give your AI agents competitive intelligence. + NotebookLM for competitive intelligence research.
@@ -717,9 +717,10 @@ export function HeroSection() { "relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400" )} > - SurfSense is an open-source competitive intelligence platform. Your AI agents monitor - competitors, track rankings, and listen to your market with live data from platforms - like Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web. + SurfSense is an open-source competitive intelligence platform, like NotebookLM but with + live scraping connectors. Your AI agents monitor competitors, track rankings, and listen + to your market with live data from platforms like Reddit, YouTube, Instagram, TikTok, + Google Maps, Google Search, and the open web.

From b5cefd25ef376f2140a1b42d5ca1c276310db68e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 138/160] feat(chat): add module-level chat stream store for resumable streaming --- surfsense_web/lib/chat/stream-engine/store.ts | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/store.ts diff --git a/surfsense_web/lib/chat/stream-engine/store.ts b/surfsense_web/lib/chat/stream-engine/store.ts new file mode 100644 index 000000000..44627663f --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/store.ts @@ -0,0 +1,177 @@ +import type { ThreadMessageLike } from "@assistant-ui/react"; +import { createTokenUsageStore } from "@/components/assistant-ui/token-usage-context"; +import type { PendingInterruptState } from "@/features/chat-messages/hitl"; + +/** + * Durable, per-thread streaming state for a single in-flight chat turn. + * + * Lives at module scope (see {@link chatStreamStore}) so a running turn's + * messages / interrupts survive the chat page unmounting during in-app + * navigation. The React tree consumes it through ``useChatStream`` via + * ``useSyncExternalStore`` (state lives outside the render cycle). + */ +export interface ThreadStreamState { + threadId: number; + messages: ThreadMessageLike[]; + isRunning: boolean; + pendingInterrupts: PendingInterruptState[]; +} + +type Listener = () => void; + +/** + * Module-level singleton external store for chat streaming. + * + * Single-active-stream invariant: at most one turn streams at a time, + * tracked by {@link active}. Per-thread state is still keyed by threadId so + * the currently-streaming thread and the currently-viewed thread can differ + * (e.g. a stream finishes while the user is on another chat). + * + * ponytail: unbounded ``states`` map is bounded in practice by + * ``clearInactive`` (called on navigation) + ``clear`` (called after the DB + * re-hydrates a finished turn). Upgrade path if that ever leaks: LRU cap. + */ +class ChatStreamStore { + private states = new Map(); + private listeners = new Set(); + + /** Shared, cross-navigation token-usage store (one instance app-wide). */ + readonly tokenUsage = createTokenUsageStore(); + + /** The one in-flight turn's abort handle, or null when idle. */ + private active: { threadId: number; controller: AbortController } | null = null; + + /** Timestamp of the last explicit cancel, for the THREAD_BUSY retry window. */ + recentCancelRequestedAt = 0; + + subscribe = (listener: Listener): (() => void) => { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }; + + private notify(): void { + for (const l of this.listeners) l(); + } + + /** Snapshot for ``useSyncExternalStore``; stable ref between mutations. */ + getSnapshot = (threadId: number | null): ThreadStreamState | null => { + if (threadId == null) return null; + return this.states.get(threadId) ?? null; + }; + + isRunning(threadId: number | null): boolean { + if (threadId == null) return false; + return this.states.get(threadId)?.isRunning ?? false; + } + + getMessages(threadId: number): ThreadMessageLike[] { + return this.states.get(threadId)?.messages ?? []; + } + + getPendingInterrupts(threadId: number): PendingInterruptState[] { + return this.states.get(threadId)?.pendingInterrupts ?? []; + } + + private ensure(threadId: number): ThreadStreamState { + let s = this.states.get(threadId); + if (!s) { + s = { threadId, messages: [], isRunning: false, pendingInterrupts: [] }; + this.states.set(threadId, s); + } + return s; + } + + private commit(threadId: number, next: ThreadStreamState): void { + this.states.set(threadId, next); + this.notify(); + } + + /** Seed a thread's state at the start of a fresh turn (running=true). */ + begin(threadId: number, messages: ThreadMessageLike[]): void { + this.commit(threadId, { threadId, messages, isRunning: true, pendingInterrupts: [] }); + } + + setMessages(threadId: number, updater: (prev: ThreadMessageLike[]) => ThreadMessageLike[]): void { + const prev = this.ensure(threadId); + const messages = updater(prev.messages); + if (messages === prev.messages) return; + this.commit(threadId, { ...prev, messages }); + } + + setRunning(threadId: number, running: boolean): void { + const prev = this.ensure(threadId); + if (prev.isRunning === running) return; + this.commit(threadId, { ...prev, isRunning: running }); + } + + setPendingInterrupts( + threadId: number, + updater: (prev: PendingInterruptState[]) => PendingInterruptState[] + ): void { + const prev = this.ensure(threadId); + const pendingInterrupts = updater(prev.pendingInterrupts); + if (pendingInterrupts === prev.pendingInterrupts) return; + this.commit(threadId, { ...prev, pendingInterrupts }); + } + + /** + * A thread whose overlay must survive DB re-hydration / navigation: it is + * either streaming or paused awaiting a HITL decision (the pending + * interrupts + interrupt cards live only in the overlay). + */ + private isPinned(s: ThreadStreamState): boolean { + return s.isRunning || s.pendingInterrupts.length > 0; + } + + /** Drop a thread's overlay once the DB is authoritative. No-op while pinned. */ + clear(threadId: number): void { + const s = this.states.get(threadId); + if (!s || this.isPinned(s)) return; + this.states.delete(threadId); + this.notify(); + } + + /** Evict every non-pinned thread except ``exceptThreadId`` (memory bound). */ + clearInactive(exceptThreadId: number | null): void { + let changed = false; + for (const [id, s] of this.states) { + if (id === exceptThreadId || this.isPinned(s)) continue; + this.states.delete(id); + changed = true; + } + if (changed) this.notify(); + } + + // ---- active-stream lifecycle ------------------------------------------- + + /** Register a new in-flight turn, aborting any previous one first. */ + beginActive(threadId: number, controller: AbortController): void { + this.abortActive(); + this.active = { threadId, controller }; + } + + get activeThreadId(): number | null { + return this.active?.threadId ?? null; + } + + /** Clear the active handle iff it still points at ``controller``. */ + clearActive(controller: AbortController): void { + if (this.active?.controller === controller) this.active = null; + } + + /** Abort the in-flight turn's fetch (client disconnect, not a server stop). */ + abortActive(): void { + if (this.active) { + this.active.controller.abort(); + this.active = null; + } + } + + markRecentCancel(): void { + this.recentCancelRequestedAt = Date.now(); + } +} + +export const chatStreamStore = new ChatStreamStore(); From 0fb577b71ff7075a0cafe770ee7e469f31f2d347 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 139/160] feat(chat): add useChatStream hook over the stream store --- .../lib/chat/stream-engine/use-chat-stream.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/use-chat-stream.ts diff --git a/surfsense_web/lib/chat/stream-engine/use-chat-stream.ts b/surfsense_web/lib/chat/stream-engine/use-chat-stream.ts new file mode 100644 index 000000000..ccfb32481 --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/use-chat-stream.ts @@ -0,0 +1,18 @@ +"use client"; + +import { useCallback, useSyncExternalStore } from "react"; +import { chatStreamStore, type ThreadStreamState } from "./store"; + +/** + * Subscribe to the durable streaming state for a given thread. + * + * Returns the live ``ThreadStreamState`` while a turn is streaming (or + * pending re-hydration from the DB), or ``null`` when the thread has no + * active overlay — in which case the page falls back to its DB-hydrated + * messages. State lives in the module-level {@link chatStreamStore}, so it + * survives the chat page unmounting during in-app navigation. + */ +export function useChatStream(threadId: number | null): ThreadStreamState | null { + const getSnapshot = useCallback(() => chatStreamStore.getSnapshot(threadId), [threadId]); + return useSyncExternalStore(chatStreamStore.subscribe, getSnapshot, () => null); +} From 9984a88d6d65c5fd319008ac3f97c74d3b377415 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 140/160] feat(chat): extract shared stream-engine helpers --- .../lib/chat/stream-engine/helpers.ts | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/helpers.ts diff --git a/surfsense_web/lib/chat/stream-engine/helpers.ts b/surfsense_web/lib/chat/stream-engine/helpers.ts new file mode 100644 index 000000000..afa1851e1 --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/helpers.ts @@ -0,0 +1,154 @@ +import { z } from "zod"; +import type { MentionedDocumentInfo } from "@/atoms/chat/mentioned-documents.atom"; +import type { ToolUIGate } from "@/lib/chat/streaming-state"; + +/** + * Every tool call renders a card. The sentinel ``"all"`` matches every tool + * — the legacy ``BASE_TOOLS_WITH_UI`` allowlist was dropped so unknown tool + * calls route through the generic ``ToolFallback``. Persisted payload size + * stays bounded because the backend's ``format_thinking_step`` summarisation + * and the ``result_length``-only default for unknown tools keep the JSON + * from ballooning. + */ +export const TOOLS_WITH_UI_ALL: ToolUIGate = "all"; + +export const TURN_CANCELLING_INITIAL_DELAY_MS = 200; +export const TURN_CANCELLING_BACKOFF_FACTOR = 2; +export const TURN_CANCELLING_MAX_DELAY_MS = 1500; +export const RECENT_CANCEL_WINDOW_MS = 5_000; + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function computeFallbackTurnCancellingRetryDelay(attempt: number): number { + const safeAttempt = Math.max(1, attempt); + const raw = + TURN_CANCELLING_INITIAL_DELAY_MS * TURN_CANCELLING_BACKOFF_FACTOR ** (safeAttempt - 1); + return Math.min(raw, TURN_CANCELLING_MAX_DELAY_MS); +} + +/** + * Generate a synthetic ``toolCallId`` for an action_request that has no + * matching streamed tool-call card (HITL-blocked subagent calls don't surface + * as tool-call events). Suffixes a counter when the base id is already taken + * — sequential interrupts for the same tool name otherwise collide on + * ``interrupt-${name}-${i}`` and crash assistant-ui with a duplicate-key error. + */ +export function freshSynthToolCallId( + toolCallIndices: Map, + toolName: string, + index: number +): string { + const base = `interrupt-${toolName}-${index}`; + if (!toolCallIndices.has(base)) return base; + let n = 1; + while (toolCallIndices.has(`${base}-${n}`)) n++; + return `${base}-${n}`; +} + +/** + * Pair each ``action_request`` to a unique pending tool-call card, preserving + * order so ``decisions[i]`` lines up with ``action_requests[i]`` on the wire. + * + * Same-name bundles (e.g. three ``create_jira_issue``) used to collapse onto + * one card because the matcher keyed by name; this consumes each card via the + * ``claimed`` set and walks forward in DOM order. + */ +export function pairBundleToolCallIds( + toolCallIndices: Map, + contentParts: Array<{ + type: string; + toolName?: string; + result?: unknown; + }>, + actionRequests: ReadonlyArray<{ name: string }> +): Array { + const claimed = new Set(); + const paired: Array = []; + for (const action of actionRequests) { + let matched: string | null = null; + for (const [tcId, idx] of toolCallIndices) { + if (claimed.has(tcId)) continue; + const part = contentParts[idx]; + if (!part || part.type !== "tool-call" || part.toolName !== action.name) continue; + const result = part.result as Record | undefined | null; + if (result == null || (result.__interrupt__ === true && !result.__decided__)) { + matched = tcId; + claimed.add(tcId); + break; + } + } + paired.push(matched); + } + return paired; +} + +/** + * Zod schema for mentioned document info (for type-safe parsing). + * + * ``kind`` defaults to ``"doc"`` so messages persisted before folder + * mentions existed deserialise unchanged. + */ +const MentionedDocumentInfoSchema = z.object({ + id: z.number(), + title: z.string(), + document_type: z.string().optional(), + kind: z + .union([z.literal("doc"), z.literal("folder"), z.literal("connector"), z.literal("thread")]) + .optional() + .default("doc"), + connector_type: z.string().optional(), + account_name: z.string().optional(), +}); + +const MentionedDocumentsPartSchema = z.object({ + type: z.literal("mentioned-documents"), + documents: z.array(MentionedDocumentInfoSchema), +}); + +/** + * Extract mentioned documents from message content (type-safe with Zod). + */ +export function extractMentionedDocuments(content: unknown): MentionedDocumentInfo[] { + if (!Array.isArray(content)) return []; + + for (const part of content) { + const result = MentionedDocumentsPartSchema.safeParse(part); + if (result.success) { + return result.data.documents.map((doc) => { + if (doc.kind === "connector") { + return { + id: doc.id, + title: doc.title, + kind: "connector", + connector_type: doc.connector_type ?? doc.document_type ?? "UNKNOWN", + account_name: doc.account_name ?? doc.title, + }; + } + if (doc.kind === "folder") { + return { + id: doc.id, + title: doc.title, + kind: "folder", + }; + } + if (doc.kind === "thread") { + return { + id: doc.id, + title: doc.title, + kind: "thread", + }; + } + return { + id: doc.id, + title: doc.title, + document_type: doc.document_type ?? "UNKNOWN", + kind: "doc", + }; + }); + } + } + + return []; +} From 204b3aa56cdcab49c6eaab3a27e3a019b8a8a90a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 141/160] feat(chat): add stream engine driving turns into the store --- .../lib/chat/stream-engine/engine.ts | 1433 +++++++++++++++++ 1 file changed, 1433 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/engine.ts diff --git a/surfsense_web/lib/chat/stream-engine/engine.ts b/surfsense_web/lib/chat/stream-engine/engine.ts new file mode 100644 index 000000000..a4a0c02a7 --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/engine.ts @@ -0,0 +1,1433 @@ +import type { AppendMessage, ThreadMessageLike } from "@assistant-ui/react"; +import { getDefaultStore } from "jotai"; +import type { Dispatch, SetStateAction } from "react"; +import { toast } from "sonner"; +import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; +import { disabledToolsAtom } from "@/atoms/agent-tools/agent-tools.atoms"; +import { + deriveMentionedPayload, + type MentionedDocumentInfo, + mentionedDocumentsAtom, + messageDocumentsMapAtom, + submittedMentionsAtom, +} from "@/atoms/chat/mentioned-documents.atom"; +import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; +import { setPremiumAlertForThreadAtom } from "@/atoms/chat/premium-alert.atom"; +import { type AgentCreatedDocument, agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; +import { updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import type { HitlDecision, PendingInterruptState } from "@/features/chat-messages/hitl"; +import { + applyActionLogSse, + applyActionLogUpdatedSse, + markActionRevertedInCache, +} from "@/hooks/use-agent-actions-query"; +import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; +import { authenticatedFetch } from "@/lib/auth-fetch"; +import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; +import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; +import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; +import { createStreamFlushHelpers } from "@/lib/chat/stream-flush"; +import { consumeSseEvents, processSharedStreamEvent } from "@/lib/chat/stream-pipeline"; +import { + applyTurnIdToAssistantMessageList, + mergeChatTurnIdIntoMessage, + readStreamedChatTurnId, + readStreamedMessageId, +} from "@/lib/chat/stream-side-effects"; +import { + addToolCall, + buildContentForUI, + type ContentPartsState, + type FrameBatchedUpdater, + type ThinkingStepData, + updateToolCall, +} from "@/lib/chat/streaming-state"; +import { + appendMessage, + createThread, + getRegenerateUrl, + type ThreadListItem, + type ThreadListResponse, + type ThreadRecord, +} from "@/lib/chat/thread-persistence"; +import { + extractUserTurnForNewChatApi, + type NewChatUserImagePayload, +} from "@/lib/chat/user-turn-api-parts"; +import { buildBackendUrl } from "@/lib/env-config"; +import { + trackChatBlocked, + trackChatCreated, + trackChatErrorDetailed, + trackChatMessageSent, + trackChatResponseReceived, +} from "@/lib/posthog/events"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { + computeFallbackTurnCancellingRetryDelay, + freshSynthToolCallId, + pairBundleToolCallIds, + RECENT_CANCEL_WINDOW_MS, + sleep, + TOOLS_WITH_UI_ALL, +} from "./helpers"; +import { chatStreamStore } from "./store"; + +const jotaiStore = getDefaultStore(); +const tokenUsageStore = chatStreamStore.tokenUsage; +const toolsWithUI = TOOLS_WITH_UI_ALL; + +/** + * Display-only setters the page provides while it is mounted. After the chat + * page unmounts (in-app navigation) these are stale but harmless — the stream + * keeps writing to the module {@link chatStreamStore}, and the page re-derives + * its thread from the URL when it remounts. + */ +export interface EngineView { + setThreadId: Dispatch>; + setCurrentThread: Dispatch>; +} + +/** Route/view context the page passes into every engine call. */ +export interface EngineContext { + workspaceId: number; + /** Currently viewed thread id (``activeThreadId`` in the page). */ + threadId: number | null; + /** The page's current displayed messages — history/slice seed. */ + priorMessages: ThreadMessageLike[]; + view: EngineView; +} + +// --------------------------------------------------------------------------- +// Error handling (relocated from the chat page) +// --------------------------------------------------------------------------- + +async function persistAssistantErrorMessage({ + threadId, + assistantMsgId, + text, +}: { + threadId: number | null; + assistantMsgId: string; + text: string; +}): Promise { + if (threadId != null) { + chatStreamStore.setMessages(threadId, (prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, content: [{ type: "text", text }] } : m)) + ); + } + + if (!threadId) return; + + // Persist only temporary assistant placeholders to avoid duplicate rows + // when the message already has a database-backed ID. + if (!assistantMsgId.startsWith("msg-assistant-")) return; + + try { + const savedMessage = await appendMessage(threadId, { + role: "assistant", + content: [{ type: "text", text }], + }); + const newMsgId = `msg-${savedMessage.id}`; + tokenUsageStore.rename(assistantMsgId, newMsgId); + chatStreamStore.setMessages(threadId, (prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)) + ); + } catch (persistErr) { + console.error("Failed to persist assistant error message:", persistErr); + } +} + +async function handleChatFailure({ + error, + flow, + threadId, + assistantMsgId, + workspaceId, +}: { + error: unknown; + flow: ChatFlow; + threadId: number | null; + assistantMsgId: string; + workspaceId: number; +}): Promise { + const normalized = classifyChatError({ + error, + flow, + context: { workspaceId, threadId }, + }); + + const logger = + normalized.severity === "error" + ? console.error + : normalized.severity === "warn" + ? console.warn + : console.info; + logger(`[chat-engine] ${flow} ${normalized.kind}:`, error); + + const telemetryPayload = { + flow, + kind: normalized.kind, + error_code: normalized.errorCode, + severity: normalized.severity, + is_expected: normalized.isExpected, + message: normalized.userMessage, + }; + if (normalized.telemetryEvent === "chat_blocked") { + trackChatBlocked(workspaceId, threadId, telemetryPayload); + } else { + trackChatErrorDetailed(workspaceId, threadId, telemetryPayload); + } + + if (normalized.channel === "silent") { + return; + } + + if (normalized.channel === "pinned_inline") { + if (threadId) { + const currentUser = jotaiStore.get(currentUserAtom).data; + jotaiStore.set(setPremiumAlertForThreadAtom, { + threadId, + message: normalized.userMessage, + userId: currentUser?.id ?? null, + }); + } + if (normalized.assistantMessage) { + await persistAssistantErrorMessage({ + threadId, + assistantMsgId, + text: normalized.assistantMessage, + }); + } + return; + } + + if (normalized.channel === "inline") { + if (normalized.assistantMessage) { + await persistAssistantErrorMessage({ + threadId, + assistantMsgId, + text: normalized.assistantMessage, + }); + } + toast.error(normalized.userMessage); + return; + } + + toast.error(normalized.userMessage); +} + +async function handleStreamTerminalError({ + error, + flow, + threadId, + assistantMsgId, + accepted, + workspaceId, + onAbort, + onPreAcceptFailure, + onAcceptedStreamError, +}: { + error: unknown; + flow: ChatFlow; + threadId: number | null; + assistantMsgId: string; + accepted: boolean; + workspaceId: number; + onAbort?: () => Promise; + onPreAcceptFailure?: () => Promise; + onAcceptedStreamError?: () => Promise; +}): Promise { + if (error instanceof Error && error.name === "AbortError") { + await onAbort?.(); + return; + } + + if (!accepted) { + await onPreAcceptFailure?.(); + } else { + await onAcceptedStreamError?.(); + } + + await handleChatFailure({ + error: !accepted ? tagPreAcceptSendFailure(error) : error, + flow, + threadId, + assistantMsgId: accepted ? assistantMsgId : "no-persist-assistant", + workspaceId, + }); +} + +async function fetchWithTurnCancellingRetry(runFetch: () => Promise): Promise { + const maxAttempts = 4; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const response = await runFetch(); + if (response.ok) { + return response; + } + const error = await toHttpResponseError(response); + const withMeta = error as Error & { errorCode?: string; retryAfterMs?: number }; + const isTurnCancelling = withMeta.errorCode === "TURN_CANCELLING"; + const isRecentThreadBusyAfterCancel = + withMeta.errorCode === "THREAD_BUSY" && + Date.now() - chatStreamStore.recentCancelRequestedAt <= RECENT_CANCEL_WINDOW_MS; + if ((isTurnCancelling || isRecentThreadBusyAfterCancel) && attempt < maxAttempts) { + const waitMs = withMeta.retryAfterMs ?? computeFallbackTurnCancellingRetryDelay(attempt); + await sleep(waitMs); + continue; + } + throw error; + } + + throw Object.assign(new Error("Turn cancellation retry limit exceeded"), { + errorCode: "TURN_CANCELLING", + }); +} + +// --------------------------------------------------------------------------- +// Cancel +// --------------------------------------------------------------------------- + +/** + * Cancel the single in-flight turn. Targets the active stream's OWNER thread + * (not the currently-viewed thread) so cancel works even after navigation and + * for the lazy-create case. + */ +export async function cancelActiveTurn(): Promise { + const threadId = chatStreamStore.activeThreadId; + if (threadId) { + try { + const response = await authenticatedFetch( + buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), + { method: "POST" } + ); + if (response.ok) { + const payload = (await response.json()) as { error_code?: string }; + if (payload.error_code === "TURN_CANCELLING") { + chatStreamStore.markRecentCancel(); + } + } + } catch (error) { + console.warn("[chat-engine] Failed to signal cancel-active-turn:", error); + } + chatStreamStore.setRunning(threadId, false); + } + chatStreamStore.abortActive(); +} + +// --------------------------------------------------------------------------- +// New chat turn +// --------------------------------------------------------------------------- + +export async function startNewChat(ctx: EngineContext, message: AppendMessage): Promise { + const { workspaceId, threadId, priorMessages, view } = ctx; + + // Supersede any previous in-flight turn. + chatStreamStore.abortActive(); + + // Prefer the submit-time snapshot; fall back to the live atom for the + // send-button path. + const submittedSnapshot = jotaiStore.get(submittedMentionsAtom); + jotaiStore.set(submittedMentionsAtom, null); + const mentionedDocuments = jotaiStore.get(mentionedDocumentsAtom); + const activeMentions = submittedSnapshot ?? mentionedDocuments; + const mentionPayload = deriveMentionedPayload(activeMentions); + if (activeMentions.length > 0) { + jotaiStore.set(mentionedDocumentsAtom, []); + } + + const pendingUserImageUrls = jotaiStore.get(pendingUserImageDataUrlsAtom); + const urlsSnapshot = [...pendingUserImageUrls]; + const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot); + + if (!userQuery.trim() && userImages.length === 0) return; + + const localFilesystemEnabled = + jotaiStore.get(agentFlagsAtom).data?.enable_desktop_local_filesystem === true; + const disabledTools = jotaiStore.get(disabledToolsAtom); + const currentUser = jotaiStore.get(currentUserAtom).data; + + // Resolve filesystem selection BEFORE any optimistic UI / lazy thread + // creation so an unsatisfied "Local Folder" requirement bails cleanly + // with nothing to roll back and no thread left stuck "running". + let selection: Awaited>; + try { + selection = await getAgentFilesystemSelection(workspaceId, { localFilesystemEnabled }); + } catch (error) { + await handleChatFailure({ + error: tagPreAcceptSendFailure(error), + flow: "new", + threadId, + assistantMsgId: "no-persist-assistant", + workspaceId, + }); + return; + } + if ( + selection.filesystem_mode === "desktop_local_folder" && + (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0) + ) { + toast.error("Select a local folder before using Local Folder mode."); + return; + } + + // Lazy thread creation: create thread on first message if it doesn't exist. + let currentThreadId = threadId; + let isNewThread = false; + if (!currentThreadId) { + try { + const newThread = await createThread(workspaceId, "New Chat"); + currentThreadId = newThread.id; + view.setThreadId(currentThreadId); + view.setCurrentThread(newThread); + queryClient.setQueryData(cacheKeys.threads.detail(newThread.id), newThread); + queryClient.setQueryData(cacheKeys.threads.messages(newThread.id), { messages: [] }); + + trackChatCreated(workspaceId, currentThreadId); + + isNewThread = true; + // Update URL silently using browser API (not router.replace) to avoid + // interrupting the ongoing fetch/streaming with React navigation. + window.history.replaceState( + null, + "", + `/dashboard/${workspaceId}/new-chat/${currentThreadId}` + ); + } catch (error) { + console.error("[chat-engine] Failed to create thread:", error); + await handleChatFailure({ + error: tagPreAcceptSendFailure(error), + flow: "new", + threadId: currentThreadId, + assistantMsgId: "no-persist-assistant", + workspaceId, + }); + return; + } + } + + // Seed the durable per-thread overlay with the pre-turn conversation and + // flip it to running so the page renders the live stream. + const streamThreadId = currentThreadId; + chatStreamStore.begin(streamThreadId, priorMessages); + + if (urlsSnapshot.length > 0) { + jotaiStore.set(pendingUserImageDataUrlsAtom, (prev) => + prev.filter((u) => !urlsSnapshot.includes(u)) + ); + } + + // Add user message to state. Mutable because the SSE + // ``data-user-message-id`` handler renames this optimistic id to the + // canonical ``msg-{db_id}`` once ``persist_user_turn`` resolves. + let userMsgId = `msg-user-${Date.now()}`; + + const authorMetadata = currentUser + ? { + custom: { + author: { + displayName: currentUser.display_name ?? null, + avatarUrl: currentUser.avatar_url ?? null, + }, + }, + } + : undefined; + + const existingImageUrls = new Set( + message.content + .filter( + (p): p is { type: "image"; image: string } => + typeof p === "object" && p !== null && "type" in p && p.type === "image" && "image" in p + ) + .map((p) => p.image) + ); + const extraImageParts = urlsSnapshot + .filter((u) => !existingImageUrls.has(u)) + .map((image) => ({ type: "image" as const, image })); + const userDisplayContent = [...message.content, ...extraImageParts]; + + const userMessage: ThreadMessageLike = { + id: userMsgId, + role: "user", + content: userDisplayContent, + createdAt: new Date(), + metadata: authorMetadata, + }; + chatStreamStore.setMessages(streamThreadId, (prev) => [...prev, userMessage]); + + trackChatMessageSent(workspaceId, streamThreadId, { + hasAttachments: userImages.length > 0, + hasMentionedDocuments: + mentionPayload.document_ids.length > 0 || + mentionPayload.folder_ids.length > 0 || + mentionPayload.connector_ids.length > 0, + messageLength: userQuery.length, + }); + + // Collect unique mention chips for display & persistence. + const allMentionedDocs: MentionedDocumentInfo[] = []; + const seenDocKeys = new Set(); + for (const doc of activeMentions) { + const key = getMentionDocKey(doc); + if (seenDocKeys.has(key)) continue; + seenDocKeys.add(key); + allMentionedDocs.push(doc); + } + + if (allMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => ({ + ...prev, + [userMsgId]: allMentionedDocs, + })); + } + + const controller = new AbortController(); + chatStreamStore.beginActive(streamThreadId, controller); + + // Prepare assistant message. Mutable for the same reason as ``userMsgId``. + let assistantMsgId = `msg-assistant-${Date.now()}`; + const currentThinkingSteps = new Map(); + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + currentReasoningPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts } = contentPartsState; + let wasInterrupted = false; + let newAccepted = false; + let streamBatcher: FrameBatchedUpdater | null = null; + + try { + // Build message history for context. + const messageHistory = priorMessages + .filter((m) => m.role === "user" || m.role === "assistant") + .map((m) => { + let text = ""; + for (const part of m.content) { + if (typeof part === "object" && part.type === "text" && "text" in part) { + text += part.text; + } + } + return { role: m.role, content: text }; + }) + .filter((m) => m.content.length > 0); + + const hasDocumentIds = mentionPayload.document_ids.length > 0; + const hasFolderIds = mentionPayload.folder_ids.length > 0; + const hasConnectorIds = mentionPayload.connector_ids.length > 0; + const hasThreadIds = mentionPayload.thread_ids.length > 0; + + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + chat_id: streamThreadId, + user_query: userQuery.trim(), + workspace_id: workspaceId, + filesystem_mode: selection.filesystem_mode, + client_platform: selection.client_platform, + local_filesystem_mounts: selection.local_filesystem_mounts, + messages: messageHistory, + mentioned_document_ids: hasDocumentIds ? mentionPayload.document_ids : undefined, + mentioned_folder_ids: hasFolderIds ? mentionPayload.folder_ids : undefined, + mentioned_connector_ids: hasConnectorIds ? mentionPayload.connector_ids : undefined, + mentioned_connectors: hasConnectorIds ? mentionPayload.connectors : undefined, + mentioned_thread_ids: hasThreadIds ? mentionPayload.thread_ids : undefined, + mentioned_documents: allMentionedDocs.length > 0 ? allMentionedDocs : undefined, + disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + ...(userImages.length > 0 ? { user_images: userImages } : {}), + }), + signal: controller.signal, + }) + ); + + if (!response.ok) { + throw await toHttpResponseError(response); + } + newAccepted = true; + chatStreamStore.setMessages(streamThreadId, (prev) => [ + ...prev, + { + id: assistantMsgId, + role: "assistant", + content: [{ type: "text", text: "" }], + createdAt: new Date(), + }, + ]); + + const flushMessages = () => { + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + }; + const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); + streamBatcher = batcher; + + await consumeSseEvents(response, async (parsed) => { + if ( + processSharedStreamEvent(parsed, { + contentPartsState, + toolsWithUI, + currentThinkingSteps, + scheduleFlush, + forceFlush, + onTokenUsage: (data) => { + tokenUsageStore.set(assistantMsgId, data); + }, + onTurnStatus: (data) => { + if (data.status === "cancelling") { + chatStreamStore.markRecentCancel(); + } + }, + }) + ) { + return; + } + switch (parsed.type) { + case "data-thread-title-update": { + const titleData = parsed.data as { threadId: number; title: string }; + if (titleData?.title && titleData?.threadId === streamThreadId) { + view.setCurrentThread((prev) => (prev ? { ...prev, title: titleData.title } : prev)); + jotaiStore.set(updateChatTabTitleAtom, { + chatId: streamThreadId, + title: titleData.title, + }); + queryClient.setQueriesData( + { queryKey: ["threads", String(workspaceId)] }, + (old) => { + if (!old) return old; + const updateTitle = (list: ThreadListItem[]) => + list.map((t) => + t.id === titleData.threadId ? { ...t, title: titleData.title } : t + ); + return { + ...old, + threads: updateTitle(old.threads), + archived_threads: updateTitle(old.archived_threads), + }; + } + ); + } + break; + } + + case "data-documents-updated": { + const docEvent = parsed.data as { + action: string; + document: AgentCreatedDocument; + }; + if (docEvent?.document?.id) { + jotaiStore.set(agentCreatedDocumentsAtom, (prev) => { + if (prev.some((d) => d.id === docEvent.document.id)) return prev; + return [...prev, docEvent.document]; + }); + } + break; + } + + case "data-interrupt-request": { + wasInterrupted = true; + const interruptData = parsed.data as Record; + const actionRequests = (interruptData.action_requests ?? []) as Array<{ + name: string; + args: Record; + }>; + const paired = pairBundleToolCallIds( + contentPartsState.toolCallIndices, + contentPartsState.contentParts, + actionRequests + ); + const bundleToolCallIds: string[] = []; + for (let i = 0; i < actionRequests.length; i++) { + const action = actionRequests[i]; + let targetTcId = paired[i]; + if (!targetTcId) { + targetTcId = freshSynthToolCallId(contentPartsState.toolCallIndices, action.name, i); + addToolCall( + contentPartsState, + toolsWithUI, + targetTcId, + action.name, + action.args, + true + ); + } + updateToolCall(contentPartsState, targetTcId, { + result: { __interrupt__: true, ...interruptData }, + }); + bundleToolCallIds.push(targetTcId); + } + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + // ``tool_call_id`` is stamped on the backend by + // ``checkpointed_subagent_middleware``. Without it we can't + // address the paused subagent on resume — skip rather than + // fabricate a synthetic key. + const interruptId = String(interruptData.tool_call_id ?? ""); + if (interruptId) { + const incoming: PendingInterruptState = { + interruptId, + threadId: streamThreadId, + assistantMsgId, + interruptData, + bundleToolCallIds, + }; + chatStreamStore.setPendingInterrupts(streamThreadId, (prev) => { + const without = prev.filter((p) => p.interruptId !== interruptId); + return [...without, incoming]; + }); + } + break; + } + + case "data-action-log": { + applyActionLogSse(queryClient, streamThreadId, workspaceId, parsed.data); + break; + } + + case "data-action-log-updated": { + applyActionLogUpdatedSse( + queryClient, + streamThreadId, + parsed.data.id, + parsed.data.reversible + ); + break; + } + + case "data-turn-info": { + const turnId = readStreamedChatTurnId(parsed.data); + if (turnId) { + chatStreamStore.setMessages(streamThreadId, (prev) => + applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) + ); + } + break; + } + + case "data-user-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newUserMsgId = `msg-${parsedMsg.messageId}`; + const oldUserMsgId = userMsgId; + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldUserMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) + : m + ) + ); + if (allMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => { + if (!(oldUserMsgId in prev)) { + return { ...prev, [newUserMsgId]: allMentionedDocs }; + } + const { [oldUserMsgId]: _removed, ...rest } = prev; + return { ...rest, [newUserMsgId]: allMentionedDocs }; + }); + } + userMsgId = newUserMsgId; + if (isNewThread) { + queryClient.invalidateQueries({ + queryKey: ["threads", String(workspaceId)], + }); + } + break; + } + + case "data-assistant-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newAssistantMsgId = `msg-${parsedMsg.messageId}`; + const oldAssistantMsgId = assistantMsgId; + tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldAssistantMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) + : m + ) + ); + chatStreamStore.setPendingInterrupts(streamThreadId, (prev) => + prev.map((p) => + p.assistantMsgId === oldAssistantMsgId + ? { ...p, assistantMsgId: newAssistantMsgId } + : p + ) + ); + assistantMsgId = newAssistantMsgId; + break; + } + } + }); + + batcher.flush(); + + if (contentParts.length > 0 && !wasInterrupted) { + trackChatResponseReceived(workspaceId, streamThreadId); + } + } catch (error) { + streamBatcher?.dispose(); + await handleStreamTerminalError({ + error, + flow: "new", + threadId: streamThreadId, + assistantMsgId, + accepted: newAccepted, + workspaceId, + onPreAcceptFailure: async () => { + // Pre-accept failure means the BE never accepted the request — no + // server-side persistence ran. Roll back the optimistic UI. + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.filter((m) => m.id !== userMsgId) + ); + jotaiStore.set(messageDocumentsMapAtom, (prev) => { + if (!(userMsgId in prev)) return prev; + const { [userMsgId]: _removed, ...rest } = prev; + return rest; + }); + }, + }); + } finally { + chatStreamStore.setRunning(streamThreadId, false); + chatStreamStore.clearActive(controller); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(streamThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(streamThreadId), + }); + } +} + +// --------------------------------------------------------------------------- +// Resume (HITL decisions) +// --------------------------------------------------------------------------- + +export async function resumeChat( + ctx: EngineContext, + decisions: Array<{ + type: string; + message?: string; + edited_action?: { name: string; args: Record }; + }> +): Promise { + const { workspaceId, threadId } = ctx; + if (threadId == null) return; + const pendingInterrupts = chatStreamStore.getPendingInterrupts(threadId); + if (pendingInterrupts.length === 0) return; + + const resumeThreadId = pendingInterrupts[0].threadId; + let assistantMsgId = pendingInterrupts[0].assistantMsgId; + const allBundleToolCallIds = pendingInterrupts.flatMap((p) => p.bundleToolCallIds); + chatStreamStore.setPendingInterrupts(resumeThreadId, () => []); + chatStreamStore.setRunning(resumeThreadId, true); + + const controller = new AbortController(); + chatStreamStore.beginActive(resumeThreadId, controller); + + const localFilesystemEnabled = + jotaiStore.get(agentFlagsAtom).data?.enable_desktop_local_filesystem === true; + const disabledTools = jotaiStore.get(disabledToolsAtom); + + const currentThinkingSteps = new Map(); + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + currentReasoningPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts, toolCallIndices } = contentPartsState; + let resumeAccepted = false; + let streamBatcher: FrameBatchedUpdater | null = null; + + const existingMsg = chatStreamStore + .getMessages(resumeThreadId) + .find((m) => m.id === assistantMsgId); + if (existingMsg && Array.isArray(existingMsg.content)) { + contentPartsState.suppressStepSeparators = true; + for (const part of existingMsg.content) { + if (typeof part === "object" && part !== null) { + const p = part as Record; + if (p.type === "text") { + contentParts.push({ type: "text", text: String(p.text ?? "") }); + contentPartsState.currentTextPartIndex = contentParts.length - 1; + } else if (p.type === "tool-call") { + toolCallIndices.set(String(p.toolCallId), contentParts.length); + contentParts.push({ + type: "tool-call", + toolCallId: String(p.toolCallId), + toolName: String(p.toolName), + args: (p.args as Record) ?? {}, + result: p.result as unknown, + ...(typeof p.argsText === "string" ? { argsText: p.argsText } : {}), + ...(typeof p.langchainToolCallId === "string" + ? { langchainToolCallId: p.langchainToolCallId } + : {}), + ...(p.metadata && typeof p.metadata === "object" + ? { metadata: p.metadata as Record } + : {}), + }); + contentPartsState.currentTextPartIndex = -1; + } else if (p.type === "data-thinking-steps") { + const stepsData = p.data as { steps: ThinkingStepData[] } | undefined; + contentParts.push({ + type: "data-thinking-steps", + data: { steps: stepsData?.steps ?? [] }, + }); + for (const step of stepsData?.steps ?? []) { + currentThinkingSteps.set(step.id, step); + } + } + } + } + } + + // Apply each decision to its own card by toolCallId so mixed bundles + // (approve/edit/reject) do not collapse onto ``decisions[0]``. + const decisionByTcId = new Map(); + const tcIds = allBundleToolCallIds; + if (decisions.length === tcIds.length) { + for (let i = 0; i < tcIds.length; i++) decisionByTcId.set(tcIds[i], decisions[i]); + } + if (decisionByTcId.size > 0) { + for (const part of contentParts) { + if (part.type !== "tool-call") continue; + const tcId = part.toolCallId as string | undefined; + const d = tcId ? decisionByTcId.get(tcId) : undefined; + if (!d) continue; + if (typeof part.result !== "object" || part.result === null) continue; + if (!("__interrupt__" in (part.result as Record))) continue; + const decided = d.type; + if (decided === "edit" && d.edited_action) { + const mergedArgs = { ...part.args, ...d.edited_action.args }; + part.args = mergedArgs; + part.argsText = JSON.stringify(mergedArgs, null, 2); + } + part.result = { + ...(part.result as Record), + __decided__: decided, + }; + } + } + + try { + const selection = await getAgentFilesystemSelection(workspaceId, { localFilesystemEnabled }); + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + workspace_id: workspaceId, + decisions, + disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + filesystem_mode: selection.filesystem_mode, + client_platform: selection.client_platform, + local_filesystem_mounts: selection.local_filesystem_mounts, + }), + signal: controller.signal, + }) + ); + + if (!response.ok) { + throw await toHttpResponseError(response); + } + resumeAccepted = true; + + const flushMessages = () => { + chatStreamStore.setMessages(resumeThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + }; + const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); + streamBatcher = batcher; + + await consumeSseEvents(response, async (parsed) => { + if ( + processSharedStreamEvent(parsed, { + contentPartsState, + toolsWithUI, + currentThinkingSteps, + scheduleFlush, + forceFlush, + onTokenUsage: (data) => { + tokenUsageStore.set(assistantMsgId, data); + }, + onTurnStatus: (data) => { + if (data.status === "cancelling") { + chatStreamStore.markRecentCancel(); + } + }, + }) + ) { + return; + } + switch (parsed.type) { + case "data-interrupt-request": { + const interruptData = parsed.data as Record; + const actionRequests = (interruptData.action_requests ?? []) as Array<{ + name: string; + args: Record; + }>; + const paired = pairBundleToolCallIds( + contentPartsState.toolCallIndices, + contentPartsState.contentParts, + actionRequests + ); + const bundleToolCallIds: string[] = []; + for (let i = 0; i < actionRequests.length; i++) { + const action = actionRequests[i]; + let targetTcId = paired[i]; + if (!targetTcId) { + targetTcId = freshSynthToolCallId(contentPartsState.toolCallIndices, action.name, i); + addToolCall( + contentPartsState, + toolsWithUI, + targetTcId, + action.name, + action.args, + true + ); + } + updateToolCall(contentPartsState, targetTcId, { + result: { __interrupt__: true, ...interruptData }, + }); + bundleToolCallIds.push(targetTcId); + } + chatStreamStore.setMessages(resumeThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + { + const interruptId = String(interruptData.tool_call_id ?? ""); + if (interruptId) { + const incoming: PendingInterruptState = { + interruptId, + threadId: resumeThreadId, + assistantMsgId, + interruptData, + bundleToolCallIds, + }; + chatStreamStore.setPendingInterrupts(resumeThreadId, (prev) => { + const without = prev.filter((p) => p.interruptId !== interruptId); + return [...without, incoming]; + }); + } + } + break; + } + + case "data-action-log": { + applyActionLogSse(queryClient, resumeThreadId, workspaceId, parsed.data); + break; + } + + case "data-action-log-updated": { + applyActionLogUpdatedSse( + queryClient, + resumeThreadId, + parsed.data.id, + parsed.data.reversible + ); + break; + } + + case "data-turn-info": { + const turnId = readStreamedChatTurnId(parsed.data); + if (turnId) { + chatStreamStore.setMessages(resumeThreadId, (prev) => + applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) + ); + } + break; + } + + case "data-assistant-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newAssistantMsgId = `msg-${parsedMsg.messageId}`; + const oldAssistantMsgId = assistantMsgId; + tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); + chatStreamStore.setMessages(resumeThreadId, (prev) => + prev.map((m) => + m.id === oldAssistantMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) + : m + ) + ); + assistantMsgId = newAssistantMsgId; + break; + } + } + }); + + batcher.flush(); + } catch (error) { + streamBatcher?.dispose(); + await handleStreamTerminalError({ + error, + flow: "resume", + threadId: resumeThreadId, + assistantMsgId, + accepted: resumeAccepted, + workspaceId, + }); + } finally { + chatStreamStore.setRunning(resumeThreadId, false); + chatStreamStore.clearActive(controller); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(resumeThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(resumeThreadId), + }); + } +} + +// --------------------------------------------------------------------------- +// Regenerate (edit / reload) +// --------------------------------------------------------------------------- + +export async function regenerateChat( + ctx: EngineContext, + newUserQuery: string | null, + editExtras?: { + userMessageContent: ThreadMessageLike["content"]; + userImages: NewChatUserImagePayload[]; + sourceUserMessageId?: string; + }, + editFromPosition?: { + fromMessageId?: number | null; + revertActions?: boolean; + } +): Promise { + const { workspaceId, threadId, priorMessages } = ctx; + if (!threadId) { + toast.error("Cannot regenerate: no active chat thread"); + return; + } + const streamThreadId = threadId; + + const isEdit = newUserQuery !== null; + + // Supersede any previous in-flight turn. + chatStreamStore.abortActive(); + + const messageDocumentsMap = jotaiStore.get(messageDocumentsMapAtom); + const localFilesystemEnabled = + jotaiStore.get(agentFlagsAtom).data?.enable_desktop_local_filesystem === true; + const disabledTools = jotaiStore.get(disabledToolsAtom); + + // Extract the original user query BEFORE removing messages (reload mode). + let userQueryToDisplay: string | undefined; + let originalUserMessageContent: ThreadMessageLike["content"] | null = null; + let originalUserMessageMetadata: ThreadMessageLike["metadata"] | undefined; + let sourceUserMessageId: string | undefined = editExtras?.sourceUserMessageId; + + if (!isEdit) { + const lastUserMessage = [...priorMessages].reverse().find((m) => m.role === "user"); + if (lastUserMessage) { + sourceUserMessageId = lastUserMessage.id; + originalUserMessageContent = lastUserMessage.content; + originalUserMessageMetadata = lastUserMessage.metadata; + for (const part of lastUserMessage.content) { + if (typeof part === "object" && part.type === "text" && "text" in part) { + userQueryToDisplay = part.text; + break; + } + } + } + } else { + userQueryToDisplay = newUserQuery; + } + + // Seed the durable overlay with the pre-regenerate conversation and flip + // to running so the page renders the live stream. + chatStreamStore.begin(streamThreadId, priorMessages); + + const controller = new AbortController(); + chatStreamStore.beginActive(streamThreadId, controller); + + let userMsgId = `msg-user-${Date.now()}`; + let assistantMsgId = `msg-assistant-${Date.now()}`; + const currentThinkingSteps = new Map(); + + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + currentReasoningPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts } = contentPartsState; + let regenerateAccepted = false; + let streamBatcher: FrameBatchedUpdater | null = null; + + const userMessage: ThreadMessageLike = { + id: userMsgId, + role: "user", + content: isEdit + ? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }]) + : originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }], + createdAt: new Date(), + metadata: isEdit ? undefined : originalUserMessageMetadata, + }; + const sourceMentionedDocs = + sourceUserMessageId && messageDocumentsMap[sourceUserMessageId] + ? messageDocumentsMap[sourceUserMessageId] + : []; + try { + const selection = await getAgentFilesystemSelection(workspaceId, { localFilesystemEnabled }); + const regenerateDocIds = sourceMentionedDocs.filter((d) => d.kind === "doc").map((d) => d.id); + const regenerateFolderIds = sourceMentionedDocs + .filter((d) => d.kind === "folder") + .map((d) => d.id); + const regenerateConnectors = sourceMentionedDocs.filter((d) => d.kind === "connector"); + const regenerateThreadIds = sourceMentionedDocs + .filter((d) => d.kind === "thread") + .map((d) => d.id); + + const requestBody: Record = { + workspace_id: workspaceId, + user_query: newUserQuery, + disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + filesystem_mode: selection.filesystem_mode, + client_platform: selection.client_platform, + local_filesystem_mounts: selection.local_filesystem_mounts, + mentioned_document_ids: regenerateDocIds.length > 0 ? regenerateDocIds : undefined, + mentioned_folder_ids: regenerateFolderIds.length > 0 ? regenerateFolderIds : undefined, + mentioned_connector_ids: + regenerateConnectors.length > 0 ? regenerateConnectors.map((d) => d.id) : undefined, + mentioned_connectors: regenerateConnectors.length > 0 ? regenerateConnectors : undefined, + mentioned_thread_ids: regenerateThreadIds.length > 0 ? regenerateThreadIds : undefined, + mentioned_documents: sourceMentionedDocs.length > 0 ? sourceMentionedDocs : undefined, + }; + if (isEdit) { + requestBody.user_images = editExtras?.userImages ?? []; + } + if (editFromPosition?.fromMessageId != null) { + requestBody.from_message_id = editFromPosition.fromMessageId; + if (editFromPosition.revertActions) { + requestBody.revert_actions = true; + } + } + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(getRegenerateUrl(streamThreadId), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(requestBody), + signal: controller.signal, + }) + ); + + if (!response.ok) { + throw await toHttpResponseError(response); + } + regenerateAccepted = true; + + chatStreamStore.setMessages(streamThreadId, (prev) => { + let base = prev; + if (editFromPosition?.fromMessageId != null) { + const targetId = `msg-${editFromPosition.fromMessageId}`; + const sliceIndex = prev.findIndex((m) => m.id === targetId); + if (sliceIndex >= 0) { + base = prev.slice(0, sliceIndex); + } + } else if (prev.length >= 2) { + base = prev.slice(0, -2); + } + return [ + ...base, + userMessage, + { + id: assistantMsgId, + role: "assistant", + content: [{ type: "text", text: "" }], + createdAt: new Date(), + }, + ]; + }); + if (sourceMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => ({ + ...prev, + [userMsgId]: sourceMentionedDocs, + })); + } + + const flushMessages = () => { + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + }; + const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); + streamBatcher = batcher; + + await consumeSseEvents(response, async (parsed) => { + if ( + processSharedStreamEvent(parsed, { + contentPartsState, + toolsWithUI, + currentThinkingSteps, + scheduleFlush, + forceFlush, + onTokenUsage: (data) => { + tokenUsageStore.set(assistantMsgId, data); + }, + onTurnStatus: (data) => { + if (data.status === "cancelling") { + chatStreamStore.markRecentCancel(); + } + }, + }) + ) { + return; + } + switch (parsed.type) { + case "data-action-log": { + applyActionLogSse(queryClient, streamThreadId, workspaceId, parsed.data); + break; + } + + case "data-action-log-updated": { + applyActionLogUpdatedSse( + queryClient, + streamThreadId, + parsed.data.id, + parsed.data.reversible + ); + break; + } + + case "data-turn-info": { + const turnId = readStreamedChatTurnId(parsed.data); + if (turnId) { + chatStreamStore.setMessages(streamThreadId, (prev) => + applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) + ); + } + break; + } + + case "data-user-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newUserMsgId = `msg-${parsedMsg.messageId}`; + const oldUserMsgId = userMsgId; + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldUserMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) + : m + ) + ); + if (sourceMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => { + if (!(oldUserMsgId in prev)) { + return { ...prev, [newUserMsgId]: sourceMentionedDocs }; + } + const { [oldUserMsgId]: _removed, ...rest } = prev; + return { ...rest, [newUserMsgId]: sourceMentionedDocs }; + }); + } + userMsgId = newUserMsgId; + break; + } + + case "data-assistant-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newAssistantMsgId = `msg-${parsedMsg.messageId}`; + const oldAssistantMsgId = assistantMsgId; + tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldAssistantMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) + : m + ) + ); + assistantMsgId = newAssistantMsgId; + break; + } + + case "data-revert-results": { + const summary = parsed.data; + const failureCount = + summary.failed + summary.not_reversible + (summary.permission_denied ?? 0); + if (failureCount > 0) { + toast.warning( + `Pre-revert: ${summary.reverted}/${summary.total} undone, ${failureCount} could not be rolled back.` + ); + } else if (summary.reverted > 0) { + toast.success( + summary.reverted === 1 + ? "Reverted 1 downstream action before regenerating." + : `Reverted ${summary.reverted} downstream actions before regenerating.` + ); + } + for (const r of summary.results) { + if (r.status === "reverted" || r.status === "already_reverted") { + markActionRevertedInCache( + queryClient, + streamThreadId, + r.action_id, + r.new_action_id ?? null + ); + } + } + break; + } + } + }); + + batcher.flush(); + + if (contentParts.length > 0) { + trackChatResponseReceived(workspaceId, streamThreadId); + } + } catch (error) { + streamBatcher?.dispose(); + await handleStreamTerminalError({ + error, + flow: "regenerate", + threadId: streamThreadId, + assistantMsgId, + accepted: regenerateAccepted, + workspaceId, + }); + } finally { + chatStreamStore.setRunning(streamThreadId, false); + chatStreamStore.clearActive(controller); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(streamThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(streamThreadId), + }); + } +} + +export type { HitlDecision }; From 5c3aa72c1a825f28e6ec1cae5b2287eb6ef54ddc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 142/160] feat(chat): add ActiveChatStreamRunner to persist streams across nav --- .../chat/active-chat-stream-runner.tsx | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 surfsense_web/components/chat/active-chat-stream-runner.tsx diff --git a/surfsense_web/components/chat/active-chat-stream-runner.tsx b/surfsense_web/components/chat/active-chat-stream-runner.tsx new file mode 100644 index 000000000..da86c0aa8 --- /dev/null +++ b/surfsense_web/components/chat/active-chat-stream-runner.tsx @@ -0,0 +1,29 @@ +"use client"; + +import { useEffect } from "react"; +import { chatStreamStore } from "@/lib/chat/stream-engine/store"; + +/** + * Persistent, render-null host for app-wide chat-stream lifecycle. + * + * Mounted in the workspace shell (``LayoutDataProvider``), it persists across + * in-app navigation between workspace routes (``/new-chat`` -> ``/chats`` -> + * ``/automations`` and doc-tab switches) and only unmounts on real teardown + * (workspace change / app teardown). On that teardown it aborts the single + * in-flight turn — this replaces the old chat-page unmount abort, which was + * what killed streams on ordinary navigation. + * + * NOTE: the ``hitl-decision`` bridge and ``useChatSessionStateSync`` stay in + * the chat page: both need the currently-viewed thread id (HITL resume can + * only be triggered from the page's approval UI), which the shell-level runner + * does not cleanly have. + */ +export function ActiveChatStreamRunner() { + useEffect(() => { + return () => { + chatStreamStore.abortActive(); + }; + }, []); + + return null; +} From 5e453cefee886d95cdd24aa90049b769cfb7792a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 143/160] feat(chat): mount ActiveChatStreamRunner in the workspace shell --- .../components/layout/providers/LayoutDataProvider.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index f9c2c9072..81f0d974f 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -18,6 +18,7 @@ import { workspacesAtom } from "@/atoms/workspaces/workspace-query.atoms"; import { ActionLogDialog } from "@/components/agent-action-log/action-log-dialog"; import { AnnouncementSpotlight } from "@/components/announcements/AnnouncementSpotlight"; import { AnnouncementsDialog } from "@/components/announcements/AnnouncementsDialog"; +import { ActiveChatStreamRunner } from "@/components/chat/active-chat-stream-runner"; import { AlertDialog, AlertDialogAction, @@ -644,6 +645,9 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider return ( <> + {/* Persistent host: keeps an in-flight chat turn streaming across + in-app navigation and aborts it only on workspace teardown. */} + Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 144/160] refactor(chat): consume stream store and delegate turns to the engine --- .../new-chat/[[...chat_id]]/page.tsx | 2218 ++--------------- 1 file changed, 236 insertions(+), 1982 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx index 4bb355049..f5b857ce1 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx @@ -7,14 +7,11 @@ import { useExternalStoreRuntime, } from "@assistant-ui/react"; import { useQueryClient } from "@tanstack/react-query"; -import { useAtomValue, useSetAtom, useStore } from "jotai"; +import { useAtomValue, useSetAtom } from "jotai"; import dynamic from "next/dynamic"; import { useParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; -import { z } from "zod"; -import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; -import { disabledToolsAtom } from "@/atoms/agent-tools/agent-tools.atoms"; import { clearTargetCommentIdAtom, currentThreadAtom, @@ -22,24 +19,15 @@ import { setTargetCommentIdAtom, } from "@/atoms/chat/current-thread.atom"; import { - deriveMentionedPayload, type MentionedDocumentInfo, mentionedDocumentsAtom, messageDocumentsMapAtom, - submittedMentionsAtom, } from "@/atoms/chat/mentioned-documents.atom"; -import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; -import { - clearPlanOwnerRegistry, - // extractWriteTodosFromContent, -} from "@/atoms/chat/plan-state.atom"; -import { setPremiumAlertForThreadAtom } from "@/atoms/chat/premium-alert.atom"; +import { clearPlanOwnerRegistry } from "@/atoms/chat/plan-state.atom"; import { closeReportPanelAtom } from "@/atoms/chat/report-panel.atom"; -import { type AgentCreatedDocument, agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; import { closeEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { membersAtom } from "@/atoms/members/members-query.atoms"; -import { removeChatTabAtom, syncChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import { removeChatTabAtom, syncChatTabAtom } from "@/atoms/tabs/tabs.atom"; import { EditMessageDialog, type EditMessageDialogChoice, @@ -47,7 +35,6 @@ import { import { StepSeparatorDataUI } from "@/components/assistant-ui/step-separator"; import { Thread } from "@/components/assistant-ui/thread"; import { - createTokenUsageStore, type TokenUsageData, TokenUsageProvider, } from "@/components/assistant-ui/token-usage-context"; @@ -60,64 +47,31 @@ import { type PendingInterruptState, } from "@/features/chat-messages/hitl"; import { TimelineDataUI } from "@/features/chat-messages/timeline"; -import { - applyActionLogSse, - applyActionLogUpdatedSse, - markActionRevertedInCache, - useAgentActionsQuery, -} from "@/hooks/use-agent-actions-query"; +import { useAgentActionsQuery } from "@/hooks/use-agent-actions-query"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesSync } from "@/hooks/use-messages-sync"; import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; -import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; -import { authenticatedFetch } from "@/lib/auth-fetch"; -import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; -import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; -import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; import { convertToThreadMessage, reconcileInterruptedAssistantMessages, } from "@/lib/chat/message-utils"; -import { createStreamFlushHelpers } from "@/lib/chat/stream-flush"; -import { consumeSseEvents, processSharedStreamEvent } from "@/lib/chat/stream-pipeline"; import { - applyTurnIdToAssistantMessageList, - mergeChatTurnIdIntoMessage, - readStreamedChatTurnId, - readStreamedMessageId, -} from "@/lib/chat/stream-side-effects"; -import { - addToolCall, - buildContentForUI, - type ContentPartsState, - type FrameBatchedUpdater, - type ThinkingStepData, - type ToolUIGate, - updateToolCall, -} from "@/lib/chat/streaming-state"; -import { - appendMessage, - createThread, - getRegenerateUrl, - type ThreadListItem, - type ThreadListResponse, - type ThreadRecord, -} from "@/lib/chat/thread-persistence"; + cancelActiveTurn, + type EngineContext, + regenerateChat, + resumeChat, + startNewChat, +} from "@/lib/chat/stream-engine/engine"; +import { extractMentionedDocuments } from "@/lib/chat/stream-engine/helpers"; +import { chatStreamStore } from "@/lib/chat/stream-engine/store"; +import { useChatStream } from "@/lib/chat/stream-engine/use-chat-stream"; +import type { ThreadRecord } from "@/lib/chat/thread-persistence"; import { extractUserTurnForNewChatApi, type NewChatUserImagePayload, } from "@/lib/chat/user-turn-api-parts"; -import { buildBackendUrl } from "@/lib/env-config"; import { NotFoundError } from "@/lib/error"; -import { - trackChatBlocked, - trackChatCreated, - trackChatErrorDetailed, - trackChatMessageSent, - trackChatResponseReceived, -} from "@/lib/posthog/events"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; const MobileEditorPanel = dynamic( () => @@ -148,156 +102,8 @@ const MobileArtifactsPanel = dynamic( { ssr: false } ); -/** - * Generate a synthetic ``toolCallId`` for an action_request that has no - * matching streamed tool-call card (HITL-blocked subagent calls don't surface - * as tool-call events). Suffixes a counter when the base id is already taken - * — sequential interrupts for the same tool name otherwise collide on - * ``interrupt-${name}-${i}`` and crash assistant-ui with a duplicate-key error. - */ -function freshSynthToolCallId( - toolCallIndices: Map, - toolName: string, - index: number -): string { - const base = `interrupt-${toolName}-${index}`; - if (!toolCallIndices.has(base)) return base; - let n = 1; - while (toolCallIndices.has(`${base}-${n}`)) n++; - return `${base}-${n}`; -} - -/** - * Pair each ``action_request`` to a unique pending tool-call card, preserving - * order so ``decisions[i]`` lines up with ``action_requests[i]`` on the wire. - * - * Same-name bundles (e.g. three ``create_jira_issue``) used to collapse onto - * one card because the matcher keyed by name; this consumes each card via the - * ``claimed`` set and walks forward in DOM order. - */ -function pairBundleToolCallIds( - toolCallIndices: Map, - contentParts: Array<{ - type: string; - toolName?: string; - result?: unknown; - }>, - actionRequests: ReadonlyArray<{ name: string }> -): Array { - const claimed = new Set(); - const paired: Array = []; - for (const action of actionRequests) { - let matched: string | null = null; - for (const [tcId, idx] of toolCallIndices) { - if (claimed.has(tcId)) continue; - const part = contentParts[idx]; - if (!part || part.type !== "tool-call" || part.toolName !== action.name) continue; - const result = part.result as Record | undefined | null; - if (result == null || (result.__interrupt__ === true && !result.__decided__)) { - matched = tcId; - claimed.add(tcId); - break; - } - } - paired.push(matched); - } - return paired; -} - -/** - * Zod schema for mentioned document info (for type-safe parsing). - * - * ``kind`` defaults to ``"doc"`` so messages persisted before folder - * mentions existed deserialise unchanged. - */ -const MentionedDocumentInfoSchema = z.object({ - id: z.number(), - title: z.string(), - document_type: z.string().optional(), - kind: z - .union([z.literal("doc"), z.literal("folder"), z.literal("connector"), z.literal("thread")]) - .optional() - .default("doc"), - connector_type: z.string().optional(), - account_name: z.string().optional(), -}); - -const MentionedDocumentsPartSchema = z.object({ - type: z.literal("mentioned-documents"), - documents: z.array(MentionedDocumentInfoSchema), -}); - -/** - * Extract mentioned documents from message content (type-safe with Zod) - */ -function extractMentionedDocuments(content: unknown): MentionedDocumentInfo[] { - if (!Array.isArray(content)) return []; - - for (const part of content) { - const result = MentionedDocumentsPartSchema.safeParse(part); - if (result.success) { - return result.data.documents.map((doc) => { - if (doc.kind === "connector") { - return { - id: doc.id, - title: doc.title, - kind: "connector", - connector_type: doc.connector_type ?? doc.document_type ?? "UNKNOWN", - account_name: doc.account_name ?? doc.title, - }; - } - if (doc.kind === "folder") { - return { - id: doc.id, - title: doc.title, - kind: "folder", - }; - } - if (doc.kind === "thread") { - return { - id: doc.id, - title: doc.title, - kind: "thread", - }; - } - return { - id: doc.id, - title: doc.title, - document_type: doc.document_type ?? "UNKNOWN", - kind: "doc", - }; - }); - } - } - - return []; -} - -/** - * Every tool call renders a card. The legacy - * ``BASE_TOOLS_WITH_UI`` allowlist used to drop unknown tool calls on the - * floor; we now route everything through ``ToolFallback``. Persisted - * payload size stays bounded because the backend's - * ``format_thinking_step`` summarisation and the - * ``result_length``-only default for unknown tools (see - * ``stream_new_chat.py``) keep the JSON from ballooning. - */ -const TOOLS_WITH_UI_ALL: ToolUIGate = "all"; -const TURN_CANCELLING_INITIAL_DELAY_MS = 200; -const TURN_CANCELLING_BACKOFF_FACTOR = 2; -const TURN_CANCELLING_MAX_DELAY_MS = 1500; -const RECENT_CANCEL_WINDOW_MS = 5_000; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function computeFallbackTurnCancellingRetryDelay(attempt: number): number { - const safeAttempt = Math.max(1, attempt); - const raw = - TURN_CANCELLING_INITIAL_DELAY_MS * TURN_CANCELLING_BACKOFF_FACTOR ** (safeAttempt - 1); - return Math.min(raw, TURN_CANCELLING_MAX_DELAY_MS); -} +/** Stable empty reference so idle threads don't re-render the interrupt provider. */ +const EMPTY_PENDING_INTERRUPTS: PendingInterruptState[] = []; function parseUrlChatId(id: string | string[] | undefined): number { let parsed = 0; @@ -372,103 +178,37 @@ export default function NewChatPage() { const activeThreadId = urlChatId > 0 ? urlChatId : threadId; const handledLoadErrorThreadRef = useRef(null); const [currentThread, setCurrentThread] = useState(null); + // DB-hydrated messages for the viewed thread (idle display). While a turn + // is streaming, the live overlay in ``chatStreamStore`` takes precedence + // (see ``displayMessages``) so it survives this page unmounting on nav. const [messages, setMessages] = useState([]); - const [isRunning, setIsRunning] = useState(false); - const [tokenUsageStore] = useState(() => createTokenUsageStore()); - const abortControllerRef = useRef(null); - const recentCancelRequestedAtRef = useRef(0); - // One entry per paused subagent, in receipt order (which matches the - // backend's ``state.interrupts`` traversal — and therefore the order - // ``slice_decisions_by_tool_call`` consumes on resume). Cleared on submit - // or on a fresh user turn. - const [pendingInterrupts, setPendingInterrupts] = useState([]); - // Per-card staged decisions held until every pending card has submitted, - // at which point we batch them into one ``hitl-decision`` event in the - // same order as ``pendingInterrupts``. Using a ref because partial - // progress should not re-render the page. - const stagedDecisionsByInterruptIdRef = useRef>(new Map()); - const toolsWithUI = TOOLS_WITH_UI_ALL; + + // Durable, cross-navigation streaming state for the viewed thread. + const streamState = useChatStream(activeThreadId); + const isRunning = streamState?.isRunning ?? false; + const pendingInterrupts = streamState?.pendingInterrupts ?? EMPTY_PENDING_INTERRUPTS; + // Live overlay while a turn is streaming / awaiting HITL; DB-hydrated + // messages once the overlay is cleared (the hydration effect drops it only + // after the DB catches up, so there is no finish->refetch gap). + const displayMessages = streamState ? streamState.messages : messages; + + // One shared token-usage store, alive across navigation. + const tokenUsageStore = chatStreamStore.tokenUsage; + const setMessageDocumentsMap = useSetAtom(messageDocumentsMapAtom); - - const persistAssistantErrorMessage = useCallback( - async ({ - threadId, - assistantMsgId, - text, - }: { - threadId: number | null; - assistantMsgId: string; - text: string; - }) => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { - ...m, - content: [{ type: "text", text }], - } - : m - ) - ); - - if (!threadId) return; - - // Persist only temporary assistant placeholders to avoid duplicate rows - // when the message already has a database-backed ID. - if (!assistantMsgId.startsWith("msg-assistant-")) return; - - try { - const savedMessage = await appendMessage(threadId, { - role: "assistant", - content: [{ type: "text", text }], - }); - const newMsgId = `msg-${savedMessage.id}`; - tokenUsageStore.rename(assistantMsgId, newMsgId); - setMessages((prev) => - prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)) - ); - } catch (persistErr) { - console.error("Failed to persist assistant error message:", persistErr); - } - }, - [tokenUsageStore] - ); - - // NOTE: ``persistUserTurn`` / ``persistAssistantTurn`` callbacks - // were removed in the SSE-based message ID handshake refactor. - // ``stream_new_chat`` and ``stream_resume_chat`` now persist both - // the user and assistant rows server-side via - // ``persist_user_turn`` / ``persist_assistant_shell`` and emit - // ``data-user-message-id`` / ``data-assistant-message-id`` SSE - // events; the consumers below rename the optimistic ids in real - // time. ``persistAssistantErrorMessage`` (above) is intentionally - // kept — it is the pre-stream-error fallback fired when the - // server NEVER accepted the request, and the BE has nothing to - // persist in that case. - - // Get disabled tools from the tool toggle UI - const disabledTools = useAtomValue(disabledToolsAtom); - - const jotaiStore = useStore(); - const mentionedDocuments = useAtomValue(mentionedDocumentsAtom); - const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); const setMentionedDocuments = useSetAtom(mentionedDocumentsAtom); const currentThreadState = useAtomValue(currentThreadAtom); const setCurrentThreadMetadata = useSetAtom(setCurrentThreadMetadataAtom); - const setPremiumAlertForThread = useSetAtom(setPremiumAlertForThreadAtom); const setTargetCommentId = useSetAtom(setTargetCommentIdAtom); const clearTargetCommentId = useSetAtom(clearTargetCommentIdAtom); const closeReportPanel = useSetAtom(closeReportPanelAtom); const closeEditorPanel = useSetAtom(closeEditorPanelAtom); const syncChatTab = useSetAtom(syncChatTabAtom); - const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom); const removeChatTab = useSetAtom(removeChatTabAtom); - const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom); - const pendingUserImageUrls = useAtomValue(pendingUserImageDataUrlsAtom); - const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom); - // Edit dialog state. Holds the message id being edited and - // the (already extracted) regenerate args so we can resume the edit - // after the user picks "revert all" / "continue" / "cancel". + + // Edit dialog state. Holds the message id being edited and the (already + // extracted) regenerate args so we can resume the edit after the user picks + // "revert all" / "continue" / "cancel". const [editDialogState, setEditDialogState] = useState<{ fromMessageId: number; userQuery: string | null; @@ -478,14 +218,17 @@ export default function NewChatPage() { downstreamTotalCount: number; } | null>(null); - // Get current user for author info in shared chats - const { data: currentUser } = useAtomValue(currentUserAtom); - const { data: agentFlags } = useAtomValue(agentFlagsAtom); - const localFilesystemEnabled = agentFlags?.enable_desktop_local_filesystem === true; + // Per-card staged decisions held until every pending card has submitted, at + // which point we batch them into one ``hitl-decision`` event in the same + // order as ``pendingInterrupts``. A ref because partial progress should not + // re-render the page. + const stagedDecisionsByInterruptIdRef = useRef>(new Map()); + const threadDetailQuery = useThreadDetail(activeThreadId); const threadMessagesQuery = useThreadMessages(activeThreadId); - // Live collaboration: sync session state and messages via Zero + // Live collaboration: sync session state and messages via Zero. Kept on the + // page because "AI responding" reflects the currently-viewed thread. useChatSessionStateSync(activeThreadId); const { data: membersData } = useAtomValue(membersAtom); @@ -498,10 +241,6 @@ export default function NewChatPage() { content: unknown; author_id: string | null; created_at: string; - // Forwarded so ``convertToThreadMessage`` can rebuild the - // ``metadata.custom.chatTurnId`` on the - // ``ThreadMessageLike``. Required by the inline Revert - // button's per-turn fallback. turn_id?: string | null; }[] ) => { @@ -520,7 +259,6 @@ export default function NewChatPage() { return reconcileInterruptedAssistantMessages(syncedMessages).map((msg) => { const member = msg.author_id ? (memberById.get(msg.author_id) ?? null) : null; - // Preserve existing author info if member lookup fails (e.g., cloned chats) const existingMsg = prevById.get(`msg-${msg.id}`); const existingAuthor = existingMsg?.metadata?.custom?.author as | { displayName?: string | null; avatarUrl?: string | null } @@ -535,10 +273,6 @@ export default function NewChatPage() { created_at: msg.created_at, author_display_name: member?.user_display_name ?? existingAuthor?.displayName ?? null, author_avatar_url: member?.user_avatar_url ?? existingAuthor?.avatarUrl ?? null, - // Forward the per-turn correlation id so the - // inline Revert button's ``(chat_turn_id, - // tool_name, position)`` fallback survives the - // post-stream Zero re-sync. turn_id: msg.turn_id ?? null, }); }); @@ -556,169 +290,33 @@ export default function NewChatPage() { return Number.isNaN(parsed) ? 0 : parsed; }, [params.workspace_id]); - // Unified store for agent-action rows (the same react-query cache - // the agent-actions dialog, the inline Revert button, and the - // per-turn Revert button all read). Hydrates from - // ``GET /threads/{id}/actions`` and is updated incrementally by the - // SSE handlers + revert-batch results below — no atom side-channel. + // Unified store for agent-action rows (react-query cache). Used by the + // edit pre-flight to count reversible downstream actions. const { items: agentActionItems } = useAgentActionsQuery(activeThreadId); - const handleChatFailure = useCallback( - async ({ - error, - flow, - threadId, - assistantMsgId, - }: { - error: unknown; - flow: ChatFlow; - threadId: number | null; - assistantMsgId: string; - }) => { - const normalized = classifyChatError({ - error, - flow, - context: { - workspaceId: workspaceId, - threadId, - }, - }); + // Latest displayed messages, read by the engine wrappers at call time so + // history/slice seeds stay fresh without re-creating the callbacks. + const messagesRef = useRef(displayMessages); + messagesRef.current = displayMessages; - const logger = - normalized.severity === "error" - ? console.error - : normalized.severity === "warn" - ? console.warn - : console.info; - logger(`[NewChatPage] ${flow} ${normalized.kind}:`, error); - - const telemetryPayload = { - flow, - kind: normalized.kind, - error_code: normalized.errorCode, - severity: normalized.severity, - is_expected: normalized.isExpected, - message: normalized.userMessage, - }; - if (normalized.telemetryEvent === "chat_blocked") { - trackChatBlocked(workspaceId, threadId, telemetryPayload); - } else { - trackChatErrorDetailed(workspaceId, threadId, telemetryPayload); - } - - if (normalized.channel === "silent") { - return; - } - - if (normalized.channel === "pinned_inline") { - if (threadId) { - setPremiumAlertForThread({ - threadId, - message: normalized.userMessage, - userId: currentUser?.id ?? null, - }); - } - if (normalized.assistantMessage) { - await persistAssistantErrorMessage({ - threadId, - assistantMsgId, - text: normalized.assistantMessage, - }); - } - return; - } - - if (normalized.channel === "inline") { - if (normalized.assistantMessage) { - await persistAssistantErrorMessage({ - threadId, - assistantMsgId, - text: normalized.assistantMessage, - }); - } - toast.error(normalized.userMessage); - return; - } - - toast.error(normalized.userMessage); - }, - [currentUser?.id, persistAssistantErrorMessage, workspaceId, setPremiumAlertForThread] + const buildCtx = useCallback( + (): EngineContext => ({ + workspaceId, + threadId: activeThreadId, + priorMessages: messagesRef.current, + view: { setThreadId, setCurrentThread }, + }), + [workspaceId, activeThreadId] ); - const handleStreamTerminalError = useCallback( - async ({ - error, - flow, - threadId, - assistantMsgId, - accepted, - onAbort, - onPreAcceptFailure, - onAcceptedStreamError, - }: { - error: unknown; - flow: ChatFlow; - threadId: number | null; - assistantMsgId: string; - accepted: boolean; - onAbort?: () => Promise; - onPreAcceptFailure?: () => Promise; - onAcceptedStreamError?: () => Promise; - }) => { - if (error instanceof Error && error.name === "AbortError") { - await onAbort?.(); - return; - } - - if (!accepted) { - await onPreAcceptFailure?.(); - } else { - await onAcceptedStreamError?.(); - } - - await handleChatFailure({ - error: !accepted ? tagPreAcceptSendFailure(error) : error, - flow, - threadId, - assistantMsgId: accepted ? assistantMsgId : "no-persist-assistant", - }); - }, - [handleChatFailure] - ); - - const fetchWithTurnCancellingRetry = useCallback(async (runFetch: () => Promise) => { - const maxAttempts = 4; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const response = await runFetch(); - if (response.ok) { - return response; - } - const error = await toHttpResponseError(response); - const withMeta = error as Error & { errorCode?: string; retryAfterMs?: number }; - const isTurnCancelling = withMeta.errorCode === "TURN_CANCELLING"; - const isRecentThreadBusyAfterCancel = - withMeta.errorCode === "THREAD_BUSY" && - Date.now() - recentCancelRequestedAtRef.current <= RECENT_CANCEL_WINDOW_MS; - if ((isTurnCancelling || isRecentThreadBusyAfterCancel) && attempt < maxAttempts) { - const waitMs = withMeta.retryAfterMs ?? computeFallbackTurnCancellingRetryDelay(attempt); - await sleep(waitMs); - continue; - } - throw error; - } - - throw Object.assign(new Error("Turn cancellation retry limit exceeded"), { - errorCode: "TURN_CANCELLING", - }); - }, []); - const hydratedMessagesRef = useRef<{ threadId: number | null; data: typeof threadMessagesQuery.data; }>({ threadId: null, data: undefined }); - // Reset thread-local runtime state on route/workspace changes. Data fetching - // is handled by React Query below so the chat shell can render immediately. + // Reset thread-local runtime state on route/workspace changes. The durable + // streaming overlay is preserved for any still-running thread (and the newly + // viewed thread) via ``clearInactive`` so an in-flight turn survives nav. useEffect(() => { const nextThreadId = urlChatId > 0 ? urlChatId : null; handledLoadErrorThreadRef.current = null; @@ -732,13 +330,11 @@ export default function NewChatPage() { clearPlanOwnerRegistry(); closeReportPanel(); closeEditorPanel(); - // Note: agent-action data is keyed by threadId in react-query so - // switching threads naturally swaps caches; no explicit reset. + chatStreamStore.clearInactive(nextThreadId); }, [ urlChatId, setMentionedDocuments, setMessageDocumentsMap, - tokenUsageStore, closeReportPanel, closeEditorPanel, ]); @@ -773,6 +369,7 @@ export default function NewChatPage() { return; } + // Per-thread gate: never overwrite the live overlay of a running turn. if (isRunning) { return; } @@ -800,13 +397,18 @@ export default function NewChatPage() { } setMessageDocumentsMap(restoredDocsMap); hydratedMessagesRef.current = { threadId: activeThreadId, data: messagesResponse }; + + // The DB is now authoritative for this thread — drop the streaming + // overlay so we render DB messages (no-op while running / HITL-pending). + if (loadedMessages.length >= chatStreamStore.getMessages(activeThreadId).length) { + chatStreamStore.clear(activeThreadId); + } }, [ activeThreadId, isRunning, messages.length, setMessageDocumentsMap, threadMessagesQuery.data, - tokenUsageStore, ]); useEffect(() => { @@ -838,8 +440,7 @@ export default function NewChatPage() { threadMessagesQuery.error, ]); - // Prefetch document titles for @ mention picker - // Runs when user lands on page so data is ready when they type @ + // Prefetch document titles for @ mention picker so data is ready on type. useEffect(() => { if (!workspaceId) return; @@ -856,10 +457,7 @@ export default function NewChatPage() { }); }, [workspaceId, queryClient]); - // Handle scroll to comment from URL query params (e.g., from inbox item click) - // Read from window.location.search inside the effect instead of subscribing via - // useSearchParams() — avoids re-rendering this heavy component tree on every - // unrelated query-string change. (Vercel Best Practice: rerender-defer-reads 5.2) + // Handle scroll to comment from URL query params (e.g., from inbox click). useEffect(() => { const readAndApplyCommentId = () => { const params = new URLSearchParams(window.location.search); @@ -874,10 +472,8 @@ export default function NewChatPage() { readAndApplyCommentId(); - // Also respond to SPA navigations (back/forward) that change the query string window.addEventListener("popstate", readAndApplyCommentId); - // Cleanup on unmount or when navigating away return () => { window.removeEventListener("popstate", readAndApplyCommentId); clearTargetCommentId(); @@ -919,934 +515,158 @@ export default function NewChatPage() { setCurrentThreadMetadata, ]); - // Cleanup on unmount - abort any in-flight requests - useEffect(() => { - return () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - }; - }, []); - - // Cancel ongoing request - const cancelRun = useCallback(async () => { - if (threadId) { - try { - const response = await authenticatedFetch( - buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), - { - method: "POST", - } - ); - if (response.ok) { - const payload = (await response.json()) as { - error_code?: string; - }; - if (payload.error_code === "TURN_CANCELLING") { - recentCancelRequestedAtRef.current = Date.now(); - } - } - } catch (error) { - console.warn("[NewChatPage] Failed to signal cancel-active-turn:", error); - } - } - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - setIsRunning(false); - }, [threadId]); - // Handle new message from user const onNew = useCallback( + (message: AppendMessage) => startNewChat(buildCtx(), message), + [buildCtx] + ); + + // Cancel the in-flight turn (targets the active stream's owner thread). + const onCancel = useCallback(async () => { + await cancelActiveTurn(); + }, []); + + // Convert message (pass through since already in correct format) + const convertMessage = useCallback( + (message: ThreadMessageLike): ThreadMessageLike => message, + [] + ); + + // Handle editing a message - truncates history and regenerates with new + // query. When ``message.sourceId`` is set we pin ``from_message_id`` so the + // backend rewinds to the right checkpoint, and prompt the user to revert / + // continue / cancel before regenerating. + const onEdit = useCallback( async (message: AppendMessage) => { - // Abort any previous streaming request to prevent race conditions - // when user sends a second query while the first is still streaming - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; + const { userQuery, userImages } = extractUserTurnForNewChatApi(message, []); + const queryForApi = userQuery.trim(); + if (!queryForApi && userImages.length === 0) { + toast.error("Cannot edit with empty message"); + return; } - // Prefer the submit-time snapshot; fall back to the live atom - // for the send-button path. - const submittedSnapshot = jotaiStore.get(submittedMentionsAtom); - jotaiStore.set(submittedMentionsAtom, null); - const activeMentions = submittedSnapshot ?? mentionedDocuments; - const mentionPayload = deriveMentionedPayload(activeMentions); - if (activeMentions.length > 0) { - setMentionedDocuments([]); + const userMessageContent = message.content as unknown as ThreadMessageLike["content"]; + + const sourceId = (message as { sourceId?: string }).sourceId; + const fromMessageId = + sourceId && /^msg-\d+$/.test(sourceId) + ? Number.parseInt(sourceId.replace(/^msg-/, ""), 10) + : null; + + if (fromMessageId == null) { + await regenerateChat(buildCtx(), queryForApi, { + userMessageContent, + userImages, + sourceUserMessageId: sourceId, + }); + return; } - const urlsSnapshot = [...pendingUserImageUrls]; - const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot); - - if (!userQuery.trim() && userImages.length === 0) return; - - // Lazy thread creation: create thread on first message if it doesn't exist - let currentThreadId = threadId; - let isNewThread = false; - if (!currentThreadId) { - try { - const newThread = await createThread(workspaceId, "New Chat"); - currentThreadId = newThread.id; - setThreadId(currentThreadId); - // Set currentThread so share button in header appears immediately - setCurrentThread(newThread); - queryClient.setQueryData(cacheKeys.threads.detail(newThread.id), newThread); - queryClient.setQueryData(cacheKeys.threads.messages(newThread.id), { messages: [] }); - - // Track chat creation - trackChatCreated(workspaceId, currentThreadId); - - isNewThread = true; - // Update URL silently using browser API (not router.replace) to avoid - // interrupting the ongoing fetch/streaming with React navigation - window.history.replaceState( - null, - "", - `/dashboard/${workspaceId}/new-chat/${currentThreadId}` - ); - } catch (error) { - console.error("[NewChatPage] Failed to create thread:", error); - await handleChatFailure({ - error: tagPreAcceptSendFailure(error), - flow: "new", - threadId: currentThreadId, - assistantMsgId: "no-persist-assistant", - }); - return; + const msgs = messagesRef.current; + const editedIndex = msgs.findIndex((m) => m.id === `msg-${fromMessageId}`); + let downstreamReversibleCount = 0; + let downstreamTotalCount = 0; + if (editedIndex >= 0) { + const downstream = msgs.slice(editedIndex + 1); + downstreamTotalCount = downstream.length; + const seenTurns = new Set(); + const downstreamTurnIds = new Set(); + for (const m of downstream) { + const meta = (m.metadata ?? {}) as { custom?: { chatTurnId?: string } }; + const tid = meta.custom?.chatTurnId; + if (!tid || seenTurns.has(tid)) continue; + seenTurns.add(tid); + downstreamTurnIds.add(tid); + } + for (const a of agentActionItems) { + if (!a.chat_turn_id || !downstreamTurnIds.has(a.chat_turn_id)) continue; + if ( + a.reversible && + (a.reverted_by_action_id === null || a.reverted_by_action_id === undefined) && + !a.is_revert_action && + (a.error === null || a.error === undefined) + ) { + downstreamReversibleCount += 1; + } } } - if (urlsSnapshot.length > 0) { - setPendingUserImageUrls((prev) => prev.filter((u) => !urlsSnapshot.includes(u))); + if (downstreamReversibleCount === 0) { + await regenerateChat( + buildCtx(), + queryForApi, + { userMessageContent, userImages, sourceUserMessageId: sourceId }, + { fromMessageId, revertActions: false } + ); + return; } - // Add user message to state. Mutable because the SSE - // ``data-user-message-id`` handler (below) renames this - // optimistic id to the canonical ``msg-{db_id}`` once the - // backend's ``persist_user_turn`` resolves the row, and - // the in-stream flush / interrupt closures need to see - // the post-rename value via this live ``let`` binding. - let userMsgId = `msg-user-${Date.now()}`; - - // Always include author metadata so the UI layer can decide visibility - const authorMetadata = currentUser - ? { - custom: { - author: { - displayName: currentUser.display_name ?? null, - avatarUrl: currentUser.avatar_url ?? null, - }, - }, - } - : undefined; - - const existingImageUrls = new Set( - message.content - .filter( - (p): p is { type: "image"; image: string } => - typeof p === "object" && - p !== null && - "type" in p && - p.type === "image" && - "image" in p - ) - .map((p) => p.image) - ); - const extraImageParts = urlsSnapshot - .filter((u) => !existingImageUrls.has(u)) - .map((image) => ({ type: "image" as const, image })); - const userDisplayContent = [...message.content, ...extraImageParts]; - - const userMessage: ThreadMessageLike = { - id: userMsgId, - role: "user", - content: userDisplayContent, - createdAt: new Date(), - metadata: authorMetadata, - }; - setMessages((prev) => [...prev, userMessage]); - - // Track message sent - trackChatMessageSent(workspaceId, currentThreadId, { - hasAttachments: userImages.length > 0, - hasMentionedDocuments: - mentionPayload.document_ids.length > 0 || - mentionPayload.folder_ids.length > 0 || - mentionPayload.connector_ids.length > 0, - messageLength: userQuery.length, + setEditDialogState({ + fromMessageId, + userQuery: queryForApi, + userMessageContent, + userImages, + downstreamReversibleCount, + downstreamTotalCount, }); + }, + [buildCtx, agentActionItems] + ); - // Collect unique mention chips for display & persistence. - // The ``kind`` field is forwarded to the backend - // so the persisted ``mentioned-documents`` content part - // can render the correct chip type on reload. - const allMentionedDocs: MentionedDocumentInfo[] = []; - const seenDocKeys = new Set(); - for (const doc of activeMentions) { - const key = getMentionDocKey(doc); - if (seenDocKeys.has(key)) continue; - seenDocKeys.add(key); - allMentionedDocs.push(doc); + const handleApprovalSubmit = useCallback( + (interruptId: string, decisions: HitlDecision[]) => { + // Stage this card's decisions; only fire the resume once every pending + // card in the current turn has submitted, so the backend slicer sees a + // single concatenated decisions list. + stagedDecisionsByInterruptIdRef.current.set(interruptId, decisions); + if (stagedDecisionsByInterruptIdRef.current.size < pendingInterrupts.length) { + return; } - - if (allMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => ({ - ...prev, - [userMsgId]: allMentionedDocs, - })); - } - - // Start streaming response - setIsRunning(true); - const controller = new AbortController(); - abortControllerRef.current = controller; - - // Prepare assistant message. Mutable for the same reason - // as ``userMsgId`` above — the ``data-assistant-message-id`` - // SSE handler reassigns this once - // ``persist_assistant_shell`` returns its canonical id. - let assistantMsgId = `msg-assistant-${Date.now()}`; - const currentThinkingSteps = new Map(); - const contentPartsState: ContentPartsState = { - contentParts: [], - currentTextPartIndex: -1, - currentReasoningPartIndex: -1, - toolCallIndices: new Map(), - }; - const { contentParts } = contentPartsState; - let wasInterrupted = false; - let newAccepted = false; - let streamBatcher: FrameBatchedUpdater | null = null; - - try { - const selection = await getAgentFilesystemSelection(workspaceId, { - localFilesystemEnabled, - }); - if ( - selection.filesystem_mode === "desktop_local_folder" && - (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0) - ) { - toast.error("Select a local folder before using Local Folder mode."); + const ordered: HitlDecision[] = []; + for (const pi of pendingInterrupts) { + const staged = stagedDecisionsByInterruptIdRef.current.get(pi.interruptId); + if (!staged) { return; } - - // Build message history for context - const messageHistory = messages - .filter((m) => m.role === "user" || m.role === "assistant") - .map((m) => { - let text = ""; - for (const part of m.content) { - if (typeof part === "object" && part.type === "text" && "text" in part) { - text += part.text; - } - } - return { role: m.role, content: text }; - }) - .filter((m) => m.content.length > 0); - - // Backend expects each mention kind in its own payload bucket. - const hasDocumentIds = mentionPayload.document_ids.length > 0; - const hasFolderIds = mentionPayload.folder_ids.length > 0; - const hasConnectorIds = mentionPayload.connector_ids.length > 0; - const hasThreadIds = mentionPayload.thread_ids.length > 0; - - const response = await fetchWithTurnCancellingRetry(() => - authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - chat_id: currentThreadId, - user_query: userQuery.trim(), - workspace_id: workspaceId, - filesystem_mode: selection.filesystem_mode, - client_platform: selection.client_platform, - local_filesystem_mounts: selection.local_filesystem_mounts, - messages: messageHistory, - mentioned_document_ids: hasDocumentIds ? mentionPayload.document_ids : undefined, - mentioned_folder_ids: hasFolderIds ? mentionPayload.folder_ids : undefined, - mentioned_connector_ids: hasConnectorIds ? mentionPayload.connector_ids : undefined, - mentioned_connectors: hasConnectorIds ? mentionPayload.connectors : undefined, - mentioned_thread_ids: hasThreadIds ? mentionPayload.thread_ids : undefined, - // Full mention metadata so the backend can persist a - // ``mentioned-documents`` ContentPart on the user message. - mentioned_documents: allMentionedDocs.length > 0 ? allMentionedDocs : undefined, - disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, - ...(userImages.length > 0 ? { user_images: userImages } : {}), - }), - signal: controller.signal, - }) - ); - - if (!response.ok) { - throw await toHttpResponseError(response); - } - newAccepted = true; - setMessages((prev) => [ - ...prev, - { - id: assistantMsgId, - role: "assistant", - content: [{ type: "text", text: "" }], - createdAt: new Date(), - }, - ]); - - const flushMessages = () => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - }; - const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); - streamBatcher = batcher; - - await consumeSseEvents(response, async (parsed) => { - if ( - processSharedStreamEvent(parsed, { - contentPartsState, - toolsWithUI, - currentThinkingSteps, - scheduleFlush, - forceFlush, - onTokenUsage: (data) => { - tokenUsageStore.set(assistantMsgId, data); - }, - onTurnStatus: (data) => { - if (data.status === "cancelling") { - recentCancelRequestedAtRef.current = Date.now(); - } - }, - }) - ) { - return; - } - switch (parsed.type) { - case "data-thread-title-update": { - const titleData = parsed.data as { threadId: number; title: string }; - if (titleData?.title && titleData?.threadId === currentThreadId) { - setCurrentThread((prev) => (prev ? { ...prev, title: titleData.title } : prev)); - updateChatTabTitle({ chatId: currentThreadId, title: titleData.title }); - queryClient.setQueriesData( - { queryKey: ["threads", String(workspaceId)] }, - (old) => { - if (!old) return old; - const updateTitle = (list: ThreadListItem[]) => - list.map((t) => - t.id === titleData.threadId ? { ...t, title: titleData.title } : t - ); - return { - ...old, - threads: updateTitle(old.threads), - archived_threads: updateTitle(old.archived_threads), - }; - } - ); - } - break; - } - - case "data-documents-updated": { - const docEvent = parsed.data as { - action: string; - document: AgentCreatedDocument; - }; - if (docEvent?.document?.id) { - setAgentCreatedDocuments((prev) => { - if (prev.some((d) => d.id === docEvent.document.id)) return prev; - return [...prev, docEvent.document]; - }); - } - break; - } - - case "data-interrupt-request": { - wasInterrupted = true; - const interruptData = parsed.data as Record; - const actionRequests = (interruptData.action_requests ?? []) as Array<{ - name: string; - args: Record; - }>; - const paired = pairBundleToolCallIds( - contentPartsState.toolCallIndices, - contentPartsState.contentParts, - actionRequests - ); - const bundleToolCallIds: string[] = []; - for (let i = 0; i < actionRequests.length; i++) { - const action = actionRequests[i]; - let targetTcId = paired[i]; - if (!targetTcId) { - targetTcId = freshSynthToolCallId( - contentPartsState.toolCallIndices, - action.name, - i - ); - addToolCall( - contentPartsState, - toolsWithUI, - targetTcId, - action.name, - action.args, - true - ); - } - updateToolCall(contentPartsState, targetTcId, { - result: { __interrupt__: true, ...interruptData }, - }); - bundleToolCallIds.push(targetTcId); - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - if (currentThreadId) { - // ``tool_call_id`` is stamped on the backend by - // ``checkpointed_subagent_middleware``. Without it we - // can't address the paused subagent on resume — skip - // rather than fabricate a synthetic key. - const interruptId = String(interruptData.tool_call_id ?? ""); - if (interruptId) { - const incoming: PendingInterruptState = { - interruptId, - threadId: currentThreadId, - assistantMsgId, - interruptData, - bundleToolCallIds, - }; - setPendingInterrupts((prev) => { - const without = prev.filter((p) => p.interruptId !== interruptId); - return [...without, incoming]; - }); - } - } - break; - } - - case "data-action-log": { - applyActionLogSse(queryClient, currentThreadId, workspaceId, parsed.data); - break; - } - - case "data-action-log-updated": { - applyActionLogUpdatedSse( - queryClient, - currentThreadId, - parsed.data.id, - parsed.data.reversible - ); - break; - } - - case "data-turn-info": { - const turnId = readStreamedChatTurnId(parsed.data); - if (turnId) { - setMessages((prev) => - applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) - ); - } - break; - } - - case "data-user-message-id": { - // Server-authoritative user message id resolved by - // ``persist_user_turn`` (or recovered via ON CONFLICT). - // Rename the optimistic ``msg-user-XXX`` placeholder to - // the canonical ``msg-{db_id}`` so DB-id-gated UI - // (comments, edit-from-this-message) unlocks immediately, - // migrate the local mentioned-documents map, and reassign - // the closure variable so all downstream - // ``m.id === userMsgId`` checks see the new value. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newUserMsgId = `msg-${parsedMsg.messageId}`; - const oldUserMsgId = userMsgId; - setMessages((prev) => - prev.map((m) => - m.id === oldUserMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) - : m - ) - ); - if (allMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => { - if (!(oldUserMsgId in prev)) { - return { ...prev, [newUserMsgId]: allMentionedDocs }; - } - const { [oldUserMsgId]: _removed, ...rest } = prev; - return { ...rest, [newUserMsgId]: allMentionedDocs }; - }); - } - userMsgId = newUserMsgId; - if (isNewThread) { - // First user-side row landed in ``new_chat_messages``; - // refresh the sidebar so the freshly-bumped - // ``thread.updated_at`` reorders this thread. - queryClient.invalidateQueries({ - queryKey: ["threads", String(workspaceId)], - }); - } - break; - } - - case "data-assistant-message-id": { - // Server-authoritative assistant message id resolved - // by ``persist_assistant_shell``. Rename the optimistic - // id, migrate ``tokenUsageStore`` so any pending - // ``data-token-usage`` payload binds to the new id, - // remap any in-flight ``pendingInterrupts`` entries, - // and reassign the closure variable so the in-stream - // flush callback (line ~1074) keeps writing to the - // renamed message. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newAssistantMsgId = `msg-${parsedMsg.messageId}`; - const oldAssistantMsgId = assistantMsgId; - tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); - setMessages((prev) => - prev.map((m) => - m.id === oldAssistantMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) - : m - ) - ); - setPendingInterrupts((prev) => - prev.map((p) => - p.assistantMsgId === oldAssistantMsgId - ? { ...p, assistantMsgId: newAssistantMsgId } - : p - ) - ); - assistantMsgId = newAssistantMsgId; - break; - } - } - }); - - batcher.flush(); - - // Server-authoritative persistence: ``stream_new_chat`` - // already wrote the user row in ``persist_user_turn`` - // (the FE renamed the optimistic id mid-stream via - // ``data-user-message-id``) and finalises the assistant - // row in ``finalize_assistant_turn`` from a shielded - // ``finally`` block. Nothing left for the FE to persist - // here — track the response and unblock the UI. - if (contentParts.length > 0 && !wasInterrupted) { - trackChatResponseReceived(workspaceId, currentThreadId); - } - } catch (error) { - streamBatcher?.dispose(); - await handleStreamTerminalError({ - error, - flow: "new", - threadId: currentThreadId, - assistantMsgId, - accepted: newAccepted, - // Server-side ``finalize_assistant_turn`` runs from a - // shielded ``anyio.CancelScope(shield=True)`` finally - // block, so partial content (incl. abort-mid-stream) - // is already persisted by the BE for the assistant - // row, and ``persist_user_turn`` ran before any LLM - // call. The FE's only remaining responsibility on - // abort / accepted-stream-error is to surface the - // error toast (handled by ``handleStreamTerminalError`` - // itself). - onPreAcceptFailure: async () => { - // Pre-accept failure means the BE never accepted the - // request — no server-side persistence ran. Roll - // back the optimistic UI insertions we made before - // the fetch so the user message and any local - // mentioned-docs metadata don't linger. - setMessages((prev) => prev.filter((m) => m.id !== userMsgId)); - setMessageDocumentsMap((prev) => { - if (!(userMsgId in prev)) return prev; - const { [userMsgId]: _removed, ...rest } = prev; - return rest; - }); - }, - }); - } finally { - setIsRunning(false); - abortControllerRef.current = null; - if (currentThreadId) { - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.messages(currentThreadId), - }); - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.detail(currentThreadId), - }); - } + ordered.push(...staged); } - }, - [ - threadId, - workspaceId, - messages, - jotaiStore, - mentionedDocuments, - setMentionedDocuments, - setMessageDocumentsMap, - setAgentCreatedDocuments, - queryClient, - currentUser, - localFilesystemEnabled, - disabledTools, - updateChatTabTitle, - tokenUsageStore, - pendingUserImageUrls, - setPendingUserImageUrls, - fetchWithTurnCancellingRetry, - handleStreamTerminalError, - handleChatFailure, - ] - ); - - const handleResume = useCallback( - async ( - decisions: Array<{ - type: string; - message?: string; - edited_action?: { name: string; args: Record }; - }> - ) => { - if (pendingInterrupts.length === 0) return; - // All cards in this turn share the same threadId/assistantMsgId - // (they're siblings of one parent agent step), so reading from - // the first entry is safe. - const resumeThreadId = pendingInterrupts[0].threadId; - // Destructured separately as ``let`` so the SSE - // ``data-assistant-message-id`` handler (resume always - // allocates a fresh server-side row) can rename it to - // the canonical ``msg-{db_id}`` mid-stream. - let assistantMsgId = pendingInterrupts[0].assistantMsgId; - // Concatenate every card's tool-call ids in pendingInterrupts order; - // this matches the ``decisions`` ordering produced by - // ``handleApprovalSubmit`` and the backend slicer's traversal of - // ``state.interrupts``. - const allBundleToolCallIds = pendingInterrupts.flatMap((p) => p.bundleToolCallIds); - setPendingInterrupts([]); stagedDecisionsByInterruptIdRef.current.clear(); - setIsRunning(true); - - const controller = new AbortController(); - abortControllerRef.current = controller; - - const currentThinkingSteps = new Map(); - - const contentPartsState: ContentPartsState = { - contentParts: [], - currentTextPartIndex: -1, - currentReasoningPartIndex: -1, - toolCallIndices: new Map(), - }; - const { contentParts, toolCallIndices } = contentPartsState; - let resumeAccepted = false; - let streamBatcher: FrameBatchedUpdater | null = null; - - const existingMsg = messages.find((m) => m.id === assistantMsgId); - if (existingMsg && Array.isArray(existingMsg.content)) { - // See ``ContentPartsState.suppressStepSeparators`` doc. - contentPartsState.suppressStepSeparators = true; - for (const part of existingMsg.content) { - if (typeof part === "object" && part !== null) { - const p = part as Record; - if (p.type === "text") { - contentParts.push({ type: "text", text: String(p.text ?? "") }); - contentPartsState.currentTextPartIndex = contentParts.length - 1; - } else if (p.type === "tool-call") { - toolCallIndices.set(String(p.toolCallId), contentParts.length); - contentParts.push({ - type: "tool-call", - toolCallId: String(p.toolCallId), - toolName: String(p.toolName), - args: (p.args as Record) ?? {}, - result: p.result as unknown, - // argsText: assistant-ui prefers it over - // JSON.stringify(args), so restoring it keeps - // pretty-printed JSON across reloads. - ...(typeof p.argsText === "string" ? { argsText: p.argsText } : {}), - ...(typeof p.langchainToolCallId === "string" - ? { langchainToolCallId: p.langchainToolCallId } - : {}), - // metadata: spanId / thinkingStepId drive the - // timeline's step↔tool join. Dropping these - // here orphans every rehydrated tool-call. - ...(p.metadata && typeof p.metadata === "object" - ? { metadata: p.metadata as Record } - : {}), - }); - contentPartsState.currentTextPartIndex = -1; - } else if (p.type === "data-thinking-steps") { - const stepsData = p.data as { steps: ThinkingStepData[] } | undefined; - contentParts.push({ - type: "data-thinking-steps", - data: { steps: stepsData?.steps ?? [] }, - }); - for (const step of stepsData?.steps ?? []) { - currentThinkingSteps.set(step.id, step); - } - } - } - } - } - - // Apply each decision to its own card by toolCallId so mixed - // bundles (approve/edit/reject) and multi-edit bundles do not - // collapse onto ``decisions[0]``. Cards outside the bundle are - // untouched. Mirrors the host ``hitl-decision`` handler. - const decisionByTcId = new Map(); - const tcIds = allBundleToolCallIds; - if (decisions.length === tcIds.length) { - for (let i = 0; i < tcIds.length; i++) decisionByTcId.set(tcIds[i], decisions[i]); - } - if (decisionByTcId.size > 0) { - for (const part of contentParts) { - if (part.type !== "tool-call") continue; - const tcId = part.toolCallId as string | undefined; - const d = tcId ? decisionByTcId.get(tcId) : undefined; - if (!d) continue; - if (typeof part.result !== "object" || part.result === null) continue; - if (!("__interrupt__" in (part.result as Record))) continue; - const decided = d.type; - if (decided === "edit" && d.edited_action) { - const mergedArgs = { ...part.args, ...d.edited_action.args }; - part.args = mergedArgs; - // Sync argsText so the rendered card shows the - // edited inputs (assistant-ui prefers it over - // JSON.stringify(args)). - part.argsText = JSON.stringify(mergedArgs, null, 2); - } - part.result = { - ...(part.result as Record), - __decided__: decided, - }; - } - } - - try { - const selection = await getAgentFilesystemSelection(workspaceId, { - localFilesystemEnabled, - }); - const response = await fetchWithTurnCancellingRetry(() => - authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - workspace_id: workspaceId, - decisions, - disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, - filesystem_mode: selection.filesystem_mode, - client_platform: selection.client_platform, - local_filesystem_mounts: selection.local_filesystem_mounts, - }), - signal: controller.signal, - }) - ); - - if (!response.ok) { - throw await toHttpResponseError(response); - } - resumeAccepted = true; - - const flushMessages = () => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - }; - const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); - streamBatcher = batcher; - - await consumeSseEvents(response, async (parsed) => { - if ( - processSharedStreamEvent(parsed, { - contentPartsState, - toolsWithUI, - currentThinkingSteps, - scheduleFlush, - forceFlush, - onTokenUsage: (data) => { - tokenUsageStore.set(assistantMsgId, data); - }, - onTurnStatus: (data) => { - if (data.status === "cancelling") { - recentCancelRequestedAtRef.current = Date.now(); - } - }, - }) - ) { - return; - } - switch (parsed.type) { - case "data-interrupt-request": { - const interruptData = parsed.data as Record; - const actionRequests = (interruptData.action_requests ?? []) as Array<{ - name: string; - args: Record; - }>; - const paired = pairBundleToolCallIds( - contentPartsState.toolCallIndices, - contentPartsState.contentParts, - actionRequests - ); - const bundleToolCallIds: string[] = []; - for (let i = 0; i < actionRequests.length; i++) { - const action = actionRequests[i]; - let targetTcId = paired[i]; - if (!targetTcId) { - targetTcId = freshSynthToolCallId( - contentPartsState.toolCallIndices, - action.name, - i - ); - addToolCall( - contentPartsState, - toolsWithUI, - targetTcId, - action.name, - action.args, - true - ); - } - updateToolCall(contentPartsState, targetTcId, { - result: { __interrupt__: true, ...interruptData }, - }); - bundleToolCallIds.push(targetTcId); - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - { - const interruptId = String(interruptData.tool_call_id ?? ""); - if (interruptId) { - const incoming: PendingInterruptState = { - interruptId, - threadId: resumeThreadId, - assistantMsgId, - interruptData, - bundleToolCallIds, - }; - setPendingInterrupts((prev) => { - const without = prev.filter((p) => p.interruptId !== interruptId); - return [...without, incoming]; - }); - } - } - break; - } - - case "data-action-log": { - applyActionLogSse(queryClient, resumeThreadId, workspaceId, parsed.data); - break; - } - - case "data-action-log-updated": { - applyActionLogUpdatedSse( - queryClient, - resumeThreadId, - parsed.data.id, - parsed.data.reversible - ); - break; - } - - case "data-turn-info": { - const turnId = readStreamedChatTurnId(parsed.data); - if (turnId) { - setMessages((prev) => - applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) - ); - } - break; - } - - case "data-assistant-message-id": { - // Resume always allocates a fresh ``new_chat_messages`` - // row anchored to a new ``turn_id`` (the original - // interrupted turn's row stays as-is), so this is a - // real id swap. Rename the optimistic placeholder to - // ``msg-{db_id}`` and reassign closure state. Resume - // does NOT emit ``data-user-message-id`` — the user - // row belongs to the original interrupted turn. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newAssistantMsgId = `msg-${parsedMsg.messageId}`; - const oldAssistantMsgId = assistantMsgId; - tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); - setMessages((prev) => - prev.map((m) => - m.id === oldAssistantMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) - : m - ) - ); - assistantMsgId = newAssistantMsgId; - break; - } - } - }); - - batcher.flush(); - - // Server-authoritative persistence: ``stream_resume_chat`` - // finalises the assistant row in - // ``finalize_assistant_turn`` from a shielded - // ``finally`` block (covers both happy-path and - // abort-mid-stream). FE has no remaining persistence - // work here. - } catch (error) { - streamBatcher?.dispose(); - await handleStreamTerminalError({ - error, - flow: "resume", - threadId: resumeThreadId, - assistantMsgId, - accepted: resumeAccepted, - }); - } finally { - setIsRunning(false); - abortControllerRef.current = null; - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.messages(resumeThreadId), - }); - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.detail(resumeThreadId), - }); - } + window.dispatchEvent(new CustomEvent("hitl-decision", { detail: { decisions: ordered } })); }, - [ - pendingInterrupts, - messages, - workspaceId, - localFilesystemEnabled, - disabledTools, - queryClient, - tokenUsageStore, - fetchWithTurnCancellingRetry, - handleStreamTerminalError, - ] + [pendingInterrupts] ); + const handleEditDialogChoice = useCallback( + async (choice: EditMessageDialogChoice) => { + const pending = editDialogState; + if (!pending) return; + setEditDialogState(null); + if (choice === "cancel") return; + await regenerateChat( + buildCtx(), + pending.userQuery, + { + userMessageContent: pending.userMessageContent, + userImages: pending.userImages, + sourceUserMessageId: `msg-${pending.fromMessageId}`, + }, + { + fromMessageId: pending.fromMessageId, + revertActions: choice === "revert", + } + ); + }, + [editDialogState, buildCtx] + ); + + // Handle reloading/refreshing the last AI response + const onReload = useCallback(async () => { + await regenerateChat(buildCtx(), null); + }, [buildCtx]); + + // HITL resume bridge. Submit always happens from this page's approval UI, so + // the currently-viewed thread owns the pending interrupts. Applies each + // decision to its card, then resumes the (durable) stream. useEffect(() => { const handler = (e: Event) => { const detail = (e as CustomEvent).detail as { @@ -1859,15 +679,9 @@ export default function NewChatPage() { if (!detail?.decisions || pendingInterrupts.length === 0) return; const incoming = detail.decisions; if (incoming.length === 0) return; - // Concatenated tool-call ids across every pending card, in the - // order ``handleApprovalSubmit`` produced ``incoming``. const tcIds = pendingInterrupts.flatMap((p) => p.bundleToolCallIds); const N = tcIds.length; - // Refuse rather than silently broadcast or drop. The orchestrator - // only fires ``hitl-decision`` once every pending card has - // submitted, so a count mismatch indicates a contract drift - // (and would later make the backend slicer raise). if (incoming.length !== N) { toast.error( `Cannot resume: ${incoming.length} decision(s) submitted for ${N} pending actions.` @@ -1890,631 +704,71 @@ export default function NewChatPage() { submittedDecisions.push(decision); } - // All pending cards belong to the same assistant message, so a - // single content-update pass suffices. const targetAssistantMsgId = pendingInterrupts[0].assistantMsgId; - setMessages((prev) => - prev.map((m) => { - if (m.id !== targetAssistantMsgId) return m; - const parts = m.content as unknown as Array>; - const newContent = parts.map((part) => { - const tcId = part.toolCallId as string | undefined; - const d = tcId ? byTcId.get(tcId) : undefined; - if (!d || part.type !== "tool-call") return part; - if (typeof part.result !== "object" || part.result === null) return part; - if (!("__interrupt__" in (part.result as Record))) return part; - const decided = d.type; - if (decided === "edit" && d.edited_action) { + if (activeThreadId != null) { + chatStreamStore.setMessages(activeThreadId, (prev) => + prev.map((m) => { + if (m.id !== targetAssistantMsgId) return m; + const parts = m.content as unknown as Array>; + const newContent = parts.map((part) => { + const tcId = part.toolCallId as string | undefined; + const d = tcId ? byTcId.get(tcId) : undefined; + if (!d || part.type !== "tool-call") return part; + if (typeof part.result !== "object" || part.result === null) return part; + if (!("__interrupt__" in (part.result as Record))) return part; + const decided = d.type; + if (decided === "edit" && d.edited_action) { + return { + ...part, + args: d.edited_action.args, + argsText: JSON.stringify(d.edited_action.args, null, 2), + result: { + ...(part.result as Record), + __decided__: decided, + }, + }; + } return { ...part, - args: d.edited_action.args, - // Sync argsText so the card renders the edited - // inputs (assistant-ui prefers it over JSON.stringify). - argsText: JSON.stringify(d.edited_action.args, null, 2), result: { ...(part.result as Record), __decided__: decided, }, }; - } - return { - ...part, - result: { - ...(part.result as Record), - __decided__: decided, - }, - }; - }); - return { ...m, content: newContent as unknown as ThreadMessageLike["content"] }; - }) - ); - handleResume(submittedDecisions); + }); + return { ...m, content: newContent as unknown as ThreadMessageLike["content"] }; + }) + ); + } + void resumeChat(buildCtx(), submittedDecisions); }; window.addEventListener("hitl-decision", handler); return () => window.removeEventListener("hitl-decision", handler); - }, [handleResume, pendingInterrupts]); - - // Convert message (pass through since already in correct format) - const convertMessage = useCallback( - (message: ThreadMessageLike): ThreadMessageLike => message, - [] - ); - - /** - * Handle regeneration (edit or reload) by calling the regenerate endpoint - * and streaming the response. This rewinds the LangGraph checkpointer state. - * - * @param newUserQuery - `null` = reload with same turn from the server. A string = edit - * (including an empty string when the edited turn is images-only); pass `editExtras` for images/content. - */ - const handleRegenerate = useCallback( - async ( - newUserQuery: string | null, - editExtras?: { - userMessageContent: ThreadMessageLike["content"]; - userImages: NewChatUserImagePayload[]; - sourceUserMessageId?: string; - }, - editFromPosition?: { - /** Message id (numeric, parsed from ``msg-``) to rewind to. */ - fromMessageId?: number | null; - /** When true, revert reversible downstream actions before stream. */ - revertActions?: boolean; - } - ) => { - if (!threadId) { - toast.error("Cannot regenerate: no active chat thread"); - return; - } - - const isEdit = newUserQuery !== null; - - // Abort any previous streaming request - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - - // Extract the original user query BEFORE removing messages (for reload mode) - let userQueryToDisplay: string | undefined; - let originalUserMessageContent: ThreadMessageLike["content"] | null = null; - let originalUserMessageMetadata: ThreadMessageLike["metadata"] | undefined; - let sourceUserMessageId: string | undefined = editExtras?.sourceUserMessageId; - - if (!isEdit) { - // Reload mode - find and preserve the last user message content - const lastUserMessage = [...messages].reverse().find((m) => m.role === "user"); - if (lastUserMessage) { - sourceUserMessageId = lastUserMessage.id; - originalUserMessageContent = lastUserMessage.content; - originalUserMessageMetadata = lastUserMessage.metadata; - // Extract text for the API request - for (const part of lastUserMessage.content) { - if (typeof part === "object" && part.type === "text" && "text" in part) { - userQueryToDisplay = part.text; - break; - } - } - } - } else { - userQueryToDisplay = newUserQuery; - } - - // Start streaming - setIsRunning(true); - const controller = new AbortController(); - abortControllerRef.current = controller; - - // Add placeholder user message if we have a new query (edit mode). - // Mutable for the same reason as in ``onNew`` — both ids are - // renamed mid-stream by the new ``data-user-message-id`` / - // ``data-assistant-message-id`` SSE handlers below. - let userMsgId = `msg-user-${Date.now()}`; - let assistantMsgId = `msg-assistant-${Date.now()}`; - const currentThinkingSteps = new Map(); - - const contentPartsState: ContentPartsState = { - contentParts: [], - currentTextPartIndex: -1, - currentReasoningPartIndex: -1, - toolCallIndices: new Map(), - }; - const { contentParts } = contentPartsState; - let regenerateAccepted = false; - let streamBatcher: FrameBatchedUpdater | null = null; - - // Add placeholder messages to UI - // Always add back the user message (with new query for edit, or original content for reload) - const userMessage: ThreadMessageLike = { - id: userMsgId, - role: "user", - content: isEdit - ? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }]) - : originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }], - createdAt: new Date(), - metadata: isEdit ? undefined : originalUserMessageMetadata, - }; - const sourceMentionedDocs = - sourceUserMessageId && messageDocumentsMap[sourceUserMessageId] - ? messageDocumentsMap[sourceUserMessageId] - : []; - try { - const selection = await getAgentFilesystemSelection(workspaceId, { - localFilesystemEnabled, - }); - // Partition the source mentions back into doc/folder id buckets - // so the regenerate route can pass them to ``stream_new_chat`` - // and the priority middleware sees the same ``[USER-MENTIONED]`` - // priority entries the original turn did. Without this partition - // the regenerate flow silently dropped the agent's mention - // awareness — same architectural bug we fixed on the new-chat path. - const regenerateDocIds = sourceMentionedDocs - .filter((d) => d.kind === "doc") - .map((d) => d.id); - const regenerateFolderIds = sourceMentionedDocs - .filter((d) => d.kind === "folder") - .map((d) => d.id); - const regenerateConnectors = sourceMentionedDocs.filter((d) => d.kind === "connector"); - const regenerateThreadIds = sourceMentionedDocs - .filter((d) => d.kind === "thread") - .map((d) => d.id); - - const requestBody: Record = { - workspace_id: workspaceId, - user_query: newUserQuery, - disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, - filesystem_mode: selection.filesystem_mode, - client_platform: selection.client_platform, - local_filesystem_mounts: selection.local_filesystem_mounts, - mentioned_document_ids: regenerateDocIds.length > 0 ? regenerateDocIds : undefined, - mentioned_folder_ids: regenerateFolderIds.length > 0 ? regenerateFolderIds : undefined, - mentioned_connector_ids: - regenerateConnectors.length > 0 ? regenerateConnectors.map((d) => d.id) : undefined, - mentioned_connectors: regenerateConnectors.length > 0 ? regenerateConnectors : undefined, - mentioned_thread_ids: regenerateThreadIds.length > 0 ? regenerateThreadIds : undefined, - // Full mention metadata for the regenerate-specific - // source list. Only meaningful for edit (the BE only - // re-persists a user row when ``user_query`` is set); - // reload reuses the original turn's mentioned_documents. - mentioned_documents: sourceMentionedDocs.length > 0 ? sourceMentionedDocs : undefined, - }; - if (isEdit) { - requestBody.user_images = editExtras?.userImages ?? []; - } - // Explicit edit-from-arbitrary-position. Only send - // ``from_message_id`` / ``revert_actions`` when the - // caller asked for them; otherwise the backend keeps the - // legacy "last 2 messages" behaviour for back-compat. - if (editFromPosition?.fromMessageId != null) { - requestBody.from_message_id = editFromPosition.fromMessageId; - if (editFromPosition.revertActions) { - requestBody.revert_actions = true; - } - } - const response = await fetchWithTurnCancellingRetry(() => - authenticatedFetch(getRegenerateUrl(threadId), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(requestBody), - signal: controller.signal, - }) - ); - - if (!response.ok) { - throw await toHttpResponseError(response); - } - regenerateAccepted = true; - - // Only switch UI to regenerated placeholder messages after the backend accepts - // regenerate. This avoids local message loss when regenerate fails early (e.g. 400). - // - // When an explicit ``editFromPosition.fromMessageId`` is passed, slice from - // that message forward so edit-from-arbitrary-position drops every downstream - // message; otherwise fall back to the legacy "drop the last 2" behaviour. - setMessages((prev) => { - let base = prev; - if (editFromPosition?.fromMessageId != null) { - const targetId = `msg-${editFromPosition.fromMessageId}`; - const sliceIndex = prev.findIndex((m) => m.id === targetId); - if (sliceIndex >= 0) { - base = prev.slice(0, sliceIndex); - } - } else if (prev.length >= 2) { - base = prev.slice(0, -2); - } - return [ - ...base, - userMessage, - { - id: assistantMsgId, - role: "assistant", - content: [{ type: "text", text: "" }], - createdAt: new Date(), - }, - ]; - }); - if (sourceMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => ({ - ...prev, - [userMsgId]: sourceMentionedDocs, - })); - } - - const flushMessages = () => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - }; - const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); - streamBatcher = batcher; - - await consumeSseEvents(response, async (parsed) => { - if ( - processSharedStreamEvent(parsed, { - contentPartsState, - toolsWithUI, - currentThinkingSteps, - scheduleFlush, - forceFlush, - onTokenUsage: (data) => { - tokenUsageStore.set(assistantMsgId, data); - }, - onTurnStatus: (data) => { - if (data.status === "cancelling") { - recentCancelRequestedAtRef.current = Date.now(); - } - }, - }) - ) { - return; - } - switch (parsed.type) { - case "data-action-log": { - if (threadId !== null) { - applyActionLogSse(queryClient, threadId, workspaceId, parsed.data); - } - break; - } - - case "data-action-log-updated": { - if (threadId !== null) { - applyActionLogUpdatedSse( - queryClient, - threadId, - parsed.data.id, - parsed.data.reversible - ); - } - break; - } - - case "data-turn-info": { - const turnId = readStreamedChatTurnId(parsed.data); - if (turnId) { - setMessages((prev) => - applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) - ); - } - break; - } - - case "data-user-message-id": { - // Same role as in ``onNew`` but the regenerate-specific - // mention metadata (``sourceMentionedDocs``) is the - // list to migrate onto the canonical id key. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newUserMsgId = `msg-${parsedMsg.messageId}`; - const oldUserMsgId = userMsgId; - setMessages((prev) => - prev.map((m) => - m.id === oldUserMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) - : m - ) - ); - if (sourceMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => { - if (!(oldUserMsgId in prev)) { - return { ...prev, [newUserMsgId]: sourceMentionedDocs }; - } - const { [oldUserMsgId]: _removed, ...rest } = prev; - return { ...rest, [newUserMsgId]: sourceMentionedDocs }; - }); - } - userMsgId = newUserMsgId; - break; - } - - case "data-assistant-message-id": { - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newAssistantMsgId = `msg-${parsedMsg.messageId}`; - const oldAssistantMsgId = assistantMsgId; - tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); - setMessages((prev) => - prev.map((m) => - m.id === oldAssistantMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) - : m - ) - ); - assistantMsgId = newAssistantMsgId; - break; - } - - case "data-revert-results": { - const summary = parsed.data; - // failureCount must include every "not undone" bucket - // (not_reversible, permission_denied, failed) so the - // toast's "X could not be rolled back" math matches - // the response invariant ``total === sum(counters)``. - // ``skipped`` rows are batch revert artefacts (revert - // rows themselves) and are not user-facing failures. - const failureCount = - summary.failed + summary.not_reversible + (summary.permission_denied ?? 0); - if (failureCount > 0) { - toast.warning( - `Pre-revert: ${summary.reverted}/${summary.total} undone, ${failureCount} could not be rolled back.` - ); - } else if (summary.reverted > 0) { - toast.success( - summary.reverted === 1 - ? "Reverted 1 downstream action before regenerating." - : `Reverted ${summary.reverted} downstream actions before regenerating.` - ); - } - if (threadId !== null) { - for (const r of summary.results) { - if (r.status === "reverted" || r.status === "already_reverted") { - markActionRevertedInCache( - queryClient, - threadId, - r.action_id, - r.new_action_id ?? null - ); - } - } - } - break; - } - } - }); - - batcher.flush(); - - // Server-authoritative persistence: ``stream_new_chat`` - // (regenerate flow) wrote the user row in - // ``persist_user_turn`` and finalises the assistant row - // in ``finalize_assistant_turn`` from a shielded - // ``finally`` block (covers both happy-path and - // abort-mid-stream). FE only needs to track the - // successful response here. - if (contentParts.length > 0) { - trackChatResponseReceived(workspaceId, threadId); - } - } catch (error) { - streamBatcher?.dispose(); - await handleStreamTerminalError({ - error, - flow: "regenerate", - threadId, - assistantMsgId, - accepted: regenerateAccepted, - }); - } finally { - setIsRunning(false); - abortControllerRef.current = null; - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.messages(threadId), - }); - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.detail(threadId), - }); - } - }, - [ - threadId, - workspaceId, - messages, - disabledTools, - localFilesystemEnabled, - messageDocumentsMap, - setMessageDocumentsMap, - queryClient, - tokenUsageStore, - fetchWithTurnCancellingRetry, - handleStreamTerminalError, - ] - ); - - // Handle editing a message - truncates history and regenerates with new query. - // - // When ``message.sourceId`` is set (the assistant-ui way to say - // "this edit replaces an older message"), we pin - // ``from_message_id`` so the backend rewinds to the right LangGraph - // checkpoint instead of relying on the legacy "last 2 messages" - // rewind. We also count downstream reversible actions and prompt the - // user to revert / continue / cancel before regenerating. - const onEdit = useCallback( - async (message: AppendMessage) => { - const { userQuery, userImages } = extractUserTurnForNewChatApi(message, []); - const queryForApi = userQuery.trim(); - if (!queryForApi && userImages.length === 0) { - toast.error("Cannot edit with empty message"); - return; - } - - const userMessageContent = message.content as unknown as ThreadMessageLike["content"]; - - // ``sourceId`` per @assistant-ui/core's ``AppendMessage`` is - // "the ID of the message that was edited". Parse the numeric - // suffix so we can map it back to a DB row. - const sourceId = (message as { sourceId?: string }).sourceId; - const fromMessageId = - sourceId && /^msg-\d+$/.test(sourceId) - ? Number.parseInt(sourceId.replace(/^msg-/, ""), 10) - : null; - - if (fromMessageId == null) { - // No source id (or non-DB id) — fall back to today's - // last-2 behaviour. The user gets the legacy edit flow. - await handleRegenerate(queryForApi, { - userMessageContent, - userImages, - sourceUserMessageId: sourceId, - }); - return; - } - - // Pre-flight: count reversible downstream actions so we can - // auto-skip the dialog for harmless edits. - // - // "Downstream" means messages AFTER the edited one. The - // previous slice ``messages.slice(editedIndex)`` included - // the edited message itself in both the total - // count and the reversibility scan (any actions on the - // edited turn would be double-counted). Slice from - // ``editedIndex + 1`` so the dialog text matches reality: - // "N downstream messages will be dropped". - const editedIndex = messages.findIndex((m) => m.id === `msg-${fromMessageId}`); - let downstreamReversibleCount = 0; - let downstreamTotalCount = 0; - if (editedIndex >= 0) { - const downstream = messages.slice(editedIndex + 1); - downstreamTotalCount = downstream.length; - const seenTurns = new Set(); - const downstreamTurnIds = new Set(); - for (const m of downstream) { - const meta = (m.metadata ?? {}) as { custom?: { chatTurnId?: string } }; - const tid = meta.custom?.chatTurnId; - if (!tid || seenTurns.has(tid)) continue; - seenTurns.add(tid); - downstreamTurnIds.add(tid); - } - // Source of truth: the unified react-query cache. Every - // action whose ``chat_turn_id`` belongs to the slice we're - // about to drop counts toward the prompt. - for (const a of agentActionItems) { - if (!a.chat_turn_id || !downstreamTurnIds.has(a.chat_turn_id)) continue; - if ( - a.reversible && - (a.reverted_by_action_id === null || a.reverted_by_action_id === undefined) && - !a.is_revert_action && - (a.error === null || a.error === undefined) - ) { - downstreamReversibleCount += 1; - } - } - } - - if (downstreamReversibleCount === 0) { - // Nothing to revert — submit silently. - await handleRegenerate( - queryForApi, - { userMessageContent, userImages, sourceUserMessageId: sourceId }, - { fromMessageId, revertActions: false } - ); - return; - } - - setEditDialogState({ - fromMessageId, - userQuery: queryForApi, - userMessageContent, - userImages, - downstreamReversibleCount, - downstreamTotalCount, - }); - }, - [handleRegenerate, messages, agentActionItems] - ); - - const handleApprovalSubmit = useCallback( - (interruptId: string, decisions: HitlDecision[]) => { - // Stage this card's decisions; only fire the resume once every - // pending card in the current turn has submitted, so the - // backend slicer sees a single concatenated decisions list - // whose total matches the parent state's pending action count. - stagedDecisionsByInterruptIdRef.current.set(interruptId, decisions); - if (stagedDecisionsByInterruptIdRef.current.size < pendingInterrupts.length) { - return; - } - const ordered: HitlDecision[] = []; - for (const pi of pendingInterrupts) { - const staged = stagedDecisionsByInterruptIdRef.current.get(pi.interruptId); - if (!staged) { - // Defensive: a missing entry means the staging map and - // the pending list disagreed for one cycle. Bail rather - // than dispatch a count-mismatched batch. - return; - } - ordered.push(...staged); - } - stagedDecisionsByInterruptIdRef.current.clear(); - window.dispatchEvent(new CustomEvent("hitl-decision", { detail: { decisions: ordered } })); - }, - [pendingInterrupts] - ); - - const handleEditDialogChoice = useCallback( - async (choice: EditMessageDialogChoice) => { - const pending = editDialogState; - if (!pending) return; - setEditDialogState(null); - if (choice === "cancel") return; - await handleRegenerate( - pending.userQuery, - { - userMessageContent: pending.userMessageContent, - userImages: pending.userImages, - sourceUserMessageId: `msg-${pending.fromMessageId}`, - }, - { - fromMessageId: pending.fromMessageId, - revertActions: choice === "revert", - } - ); - }, - [editDialogState, handleRegenerate] - ); - - // Handle reloading/refreshing the last AI response - const onReload = useCallback(async () => { - // parentId is the ID of the message to reload from (the user message) - // We call regenerate without a query to use the same query - await handleRegenerate(null); - }, [handleRegenerate]); + }, [buildCtx, pendingInterrupts, activeThreadId]); // Surface the thread's deliverables to the layout-level artifacts sidebar. - useSyncChatArtifacts(messages); + useSyncChatArtifacts(displayMessages); // Create external store runtime const runtime = useExternalStoreRuntime({ - messages, + messages: displayMessages, isRunning, onNew, onEdit, onReload, convertMessage, - onCancel: cancelRun, + onCancel, }); const threadLoadError = activeThreadId ? (threadDetailQuery.error ?? threadMessagesQuery.error) : null; const shouldShowThreadLoadError = - !!threadLoadError && !!activeThreadId && !currentThread && messages.length === 0; + !!threadLoadError && !!activeThreadId && !currentThread && displayMessages.length === 0; const isThreadMessagesLoading = !!activeThreadId && threadMessagesQuery.isPending && - messages.length === 0 && + displayMessages.length === 0 && !threadMessagesQuery.error; if (shouldShowThreadLoadError) { From 9879306ccc4eafb32ad8e02ac7396b912e2672c9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:39:28 +0200 Subject: [PATCH 145/160] docs(chat): tighten ActiveChatStreamRunner comment to intent --- .../chat/active-chat-stream-runner.tsx | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/surfsense_web/components/chat/active-chat-stream-runner.tsx b/surfsense_web/components/chat/active-chat-stream-runner.tsx index da86c0aa8..dd27922a0 100644 --- a/surfsense_web/components/chat/active-chat-stream-runner.tsx +++ b/surfsense_web/components/chat/active-chat-stream-runner.tsx @@ -4,19 +4,13 @@ import { useEffect } from "react"; import { chatStreamStore } from "@/lib/chat/stream-engine/store"; /** - * Persistent, render-null host for app-wide chat-stream lifecycle. + * Persistent, render-null host that scopes the in-flight chat turn's lifetime + * to the workspace shell, not the chat page. * - * Mounted in the workspace shell (``LayoutDataProvider``), it persists across - * in-app navigation between workspace routes (``/new-chat`` -> ``/chats`` -> - * ``/automations`` and doc-tab switches) and only unmounts on real teardown - * (workspace change / app teardown). On that teardown it aborts the single - * in-flight turn — this replaces the old chat-page unmount abort, which was - * what killed streams on ordinary navigation. - * - * NOTE: the ``hitl-decision`` bridge and ``useChatSessionStateSync`` stay in - * the chat page: both need the currently-viewed thread id (HITL resume can - * only be triggered from the page's approval UI), which the shell-level runner - * does not cleanly have. + * Mounted in ``LayoutDataProvider``, it survives in-app navigation between + * workspace routes and aborts the single active turn only on workspace/app + * teardown, so ordinary navigation disconnects the view without stopping the + * stream. */ export function ActiveChatStreamRunner() { useEffect(() => { From a44c67da978720cabd8ed873bf840edf408a8e70 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:39:29 +0200 Subject: [PATCH 146/160] docs(chat): tighten engine comments to intent --- surfsense_web/lib/chat/stream-engine/engine.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/surfsense_web/lib/chat/stream-engine/engine.ts b/surfsense_web/lib/chat/stream-engine/engine.ts index a4a0c02a7..57ff47f61 100644 --- a/surfsense_web/lib/chat/stream-engine/engine.ts +++ b/surfsense_web/lib/chat/stream-engine/engine.ts @@ -80,10 +80,9 @@ const tokenUsageStore = chatStreamStore.tokenUsage; const toolsWithUI = TOOLS_WITH_UI_ALL; /** - * Display-only setters the page provides while it is mounted. After the chat - * page unmounts (in-app navigation) these are stale but harmless — the stream - * keeps writing to the module {@link chatStreamStore}, and the page re-derives - * its thread from the URL when it remounts. + * Display-only setters, valid only while the page is mounted. The stream drives + * durable state through {@link chatStreamStore}; these just sync the mounted + * page's local view and are ignored once it unmounts. */ export interface EngineView { setThreadId: Dispatch>; @@ -101,7 +100,7 @@ export interface EngineContext { } // --------------------------------------------------------------------------- -// Error handling (relocated from the chat page) +// Error handling // --------------------------------------------------------------------------- async function persistAssistantErrorMessage({ From f0cd20bcbd3afda0cb024bb067cd630315e15cb7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 147/160] feat(chat): add showMessageTimestamps preference atom (localStorage, off by default) --- surfsense_web/atoms/chat/show-timestamps.atom.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 surfsense_web/atoms/chat/show-timestamps.atom.ts diff --git a/surfsense_web/atoms/chat/show-timestamps.atom.ts b/surfsense_web/atoms/chat/show-timestamps.atom.ts new file mode 100644 index 000000000..434f065ff --- /dev/null +++ b/surfsense_web/atoms/chat/show-timestamps.atom.ts @@ -0,0 +1,11 @@ +import { atomWithStorage } from "jotai/utils"; + +/** + * Per-device preference: show a timestamp under each chat message. + * + * Off by default to match streaming-AI chat convention (ChatGPT/Claude keep + * the message stream clean and put time in the conversation list). Persisted + * in localStorage, so it does not sync across devices — acceptable for a + * cosmetic display toggle. + */ +export const showMessageTimestampsAtom = atomWithStorage("chat-show-timestamps:v1", false); From f4870ad1c20b8cd138a2ef20bce137ef317d1fbd Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 148/160] feat(chat): add shared formatMessageTimestamp helper --- surfsense_web/lib/format-date.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/surfsense_web/lib/format-date.ts b/surfsense_web/lib/format-date.ts index c2f445537..a062b08fa 100644 --- a/surfsense_web/lib/format-date.ts +++ b/surfsense_web/lib/format-date.ts @@ -68,3 +68,17 @@ export function formatRelativeFutureDate(dateString: string): string { export function formatThreadTimestamp(dateString: string): string { return format(new Date(dateString), "MMM d, yyyy 'at' h:mm a"); } + +/** + * Format a chat message timestamp for inline display under a bubble. + * Locale-aware, 12-hour clock. Example: "Jul 13, 10:42 PM". + */ +export function formatMessageTimestamp(date: Date): string { + return date.toLocaleDateString(undefined, { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + hour12: true, + }); +} From 00fbc24e4bb91d217a0da1c616ac23a58366b501 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 149/160] feat(chat): add MessageTimestamp component gated on the preference --- .../assistant-ui/message-timestamp.tsx | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 surfsense_web/components/assistant-ui/message-timestamp.tsx diff --git a/surfsense_web/components/assistant-ui/message-timestamp.tsx b/surfsense_web/components/assistant-ui/message-timestamp.tsx new file mode 100644 index 000000000..65a1da9be --- /dev/null +++ b/surfsense_web/components/assistant-ui/message-timestamp.tsx @@ -0,0 +1,24 @@ +import { useAuiState } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import type { FC } from "react"; +import { showMessageTimestampsAtom } from "@/atoms/chat/show-timestamps.atom"; +import { formatMessageTimestamp } from "@/lib/format-date"; +import { cn } from "@/lib/utils"; + +/** + * Muted, always-visible timestamp under a chat message. Renders only when the + * user has opted in via {@link showMessageTimestampsAtom} and the message + * carries a ``createdAt`` (absent on optimistic pre-persist messages). + */ +export const MessageTimestamp: FC<{ className?: string }> = ({ className }) => { + const show = useAtomValue(showMessageTimestampsAtom); + const createdAt = useAuiState(({ message }) => message?.createdAt); + + if (!show || !createdAt) return null; + + return ( +
+ {formatMessageTimestamp(createdAt)} +
+ ); +}; From 0093664fd10c6e1e0b0d9d490c9011d20189307c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 150/160] feat(chat): render timestamp under user messages --- surfsense_web/components/assistant-ui/user-message.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_web/components/assistant-ui/user-message.tsx b/surfsense_web/components/assistant-ui/user-message.tsx index 592dc4224..09a700ff1 100644 --- a/surfsense_web/components/assistant-ui/user-message.tsx +++ b/surfsense_web/components/assistant-ui/user-message.tsx @@ -22,6 +22,7 @@ import { currentThreadAtom } from "@/atoms/chat/current-thread.atom"; import { messageDocumentsMapAtom } from "@/atoms/chat/mentioned-documents.atom"; import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { MentionChip } from "@/components/assistant-ui/mention-chip"; +import { MessageTimestamp } from "@/components/assistant-ui/message-timestamp"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; @@ -182,6 +183,7 @@ export const UserMessage: FC = () => {
)}
+
); From 855f82409712f3b54369b5f63e7b58690403f2c6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 151/160] feat(chat): render timestamp under assistant messages and reuse shared formatter --- .../assistant-ui/assistant-message.tsx | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index c4a0e86dc..a45c651b1 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -35,6 +35,7 @@ import { useAllCitationMetadata, } from "@/components/assistant-ui/citation-metadata-context"; import { MarkdownText } from "@/components/assistant-ui/markdown-text"; +import { MessageTimestamp } from "@/components/assistant-ui/message-timestamp"; import { ReasoningMessagePart } from "@/components/assistant-ui/reasoning-message-part"; import { RevertTurnButton } from "@/components/assistant-ui/revert-turn-button"; import { @@ -62,6 +63,7 @@ import { withArtifactAnchor } from "@/features/chat-artifacts"; import { useComments } from "@/hooks/use-comments"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; +import { formatMessageTimestamp } from "@/lib/format-date"; import { getProviderIcon } from "@/lib/provider-icons"; import { tryGetHostname } from "@/lib/url"; import { cn } from "@/lib/utils"; @@ -249,16 +251,6 @@ export const MessageError: FC = () => { ); }; -function formatMessageDate(date: Date): string { - return date.toLocaleDateString(undefined, { - month: "short", - day: "numeric", - hour: "numeric", - minute: "2-digit", - hour12: true, - }); -} - /** * Format provider USD cost (in micro-USD) for inline display next to a * token count. Falls back to ``"<$0.001"`` for sub-tenth-of-a-cent @@ -367,7 +359,7 @@ const MessageInfoDropdown: FC<{ chatTurnId: string | null | undefined }> = ({ ch > {createdAt && ( - {formatMessageDate(createdAt)} + {formatMessageTimestamp(createdAt)} )} {hasUsage && ( @@ -463,6 +455,8 @@ const AssistantMessageInner: FC = () => {
+ + {isMobile && (
From 4b2f009233b06cdfb190d0fc06379dab9a9e5560 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 152/160] feat(settings): add Appearance content with show-timestamps toggle --- .../components/AppearanceContent.tsx | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx b/surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx new file mode 100644 index 000000000..258a31dfe --- /dev/null +++ b/surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx @@ -0,0 +1,43 @@ +"use client"; + +import { useAtom } from "jotai"; +import { showMessageTimestampsAtom } from "@/atoms/chat/show-timestamps.atom"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; + +export function AppearanceContent() { + const [showTimestamps, setShowTimestamps] = useAtom(showMessageTimestampsAtom); + + return ( +
+
+
+

Chat

+

+ Control how messages are displayed in your conversations. +

+
+
+
+
+ +

+ Display the time under each message in a chat. Saved on this device. +

+
+ +
+
+
+
+ ); +} From b3999a754e06e4a385fc3e441b7411e7113e70c2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 153/160] feat(settings): add Appearance user-settings page --- .../[workspace_id]/user-settings/appearance/page.tsx | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx new file mode 100644 index 000000000..6ae0952d5 --- /dev/null +++ b/surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx @@ -0,0 +1,5 @@ +import { AppearanceContent } from "../components/AppearanceContent"; + +export default function Page() { + return ; +} From e30a7afaf167accbb6ce1fc3318af5246d33ee6f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 154/160] feat(settings): register Appearance tab in user-settings nav --- .../[workspace_id]/user-settings/layout-shell.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx b/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx index aa73917ef..b18db9e64 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx @@ -7,6 +7,7 @@ import { Library, MessageCircle, Monitor, + Palette, ReceiptText, ShieldCheck, WandSparkles, @@ -20,6 +21,7 @@ import { usePlatform } from "@/hooks/use-platform"; export type UserSettingsTab = | "profile" + | "appearance" | "api-key" | "prompts" | "community-prompts" @@ -49,6 +51,12 @@ export function UserSettingsLayoutShell({ workspaceId, children }: UserSettingsL href: `/dashboard/${workspaceId}/user-settings/profile`, icon: , }, + { + value: "appearance" as const, + label: "Appearance", + href: `/dashboard/${workspaceId}/user-settings/appearance`, + icon: , + }, { value: "api-key" as const, label: t("api_key_nav_label"), From f70b83582fa813a812aff3062d4117f0bb4f7682 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 14:18:15 -0700 Subject: [PATCH 155/160] Update README files in multiple languages to reflect the rebranding of SurfSense as NotebookLM for competitive intelligence research, emphasizing live scraping capabilities and clarifying the platform's features. --- README.es.md | 4 ++-- README.hi.md | 4 ++-- README.md | 4 ++-- README.pt-BR.md | 4 ++-- README.zh-CN.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.es.md b/README.es.md index 70ccd5eb1..ecdd93c41 100644 --- a/README.es.md +++ b/README.es.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift
-# SurfSense: Dale inteligencia competitiva a tus agentes de IA +# SurfSense: NotebookLM para investigación de inteligencia competitiva -SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. +SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. > [!NOTE] > **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM** diff --git a/README.hi.md b/README.hi.md index cbf9ae8fa..3cb769318 100644 --- a/README.hi.md +++ b/README.hi.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift
-# SurfSense: अपने AI एजेंट्स को दें कॉम्पिटिटिव इंटेलिजेंस +# SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM -SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। +SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। > [!NOTE] > **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना** diff --git a/README.md b/README.md index 8f2e65118..eac392ebd 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift
-# SurfSense: Give Your AI Agents Competitive Intelligence +# SurfSense: NotebookLM for Competitive Intelligence Research -SurfSense is the **open-source competitive intelligence platform for AI agents**. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. +SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. > [!NOTE] > **📢 A note for our NotebookLM-alternative users** diff --git a/README.pt-BR.md b/README.pt-BR.md index 076736bcf..e32882831 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift
-# SurfSense: Dê Inteligência Competitiva aos Seus Agentes de IA +# SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva -O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. +O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. > [!NOTE] > **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM** diff --git a/README.zh-CN.md b/README.zh-CN.md index 7d2ae35c3..11bace243 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift
-# SurfSense:为你的 AI 智能体注入竞争情报能力 +# SurfSense:面向竞争情报研究的 NotebookLM -SurfSense 是**面向 AI 智能体的开源竞争情报平台**。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。 +SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 NotebookLM,但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。 > [!NOTE] > **📢 致我们的 NotebookLM 替代品用户** From ab5970fab11939ec0e4d9b258a9e9b10ccbbe3c1 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 14:42:11 -0700 Subject: [PATCH 156/160] feat: updating messaging of instagram and tiktok pages --- surfsense_web/lib/auth-utils.ts | 1 + .../lib/connectors-marketing/instagram.tsx | 47 +++++++----- .../lib/connectors-marketing/tiktok.tsx | 76 +++++++++++-------- 3 files changed, 72 insertions(+), 52 deletions(-) diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 8c8d919c5..7d9320fb0 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -40,6 +40,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/external-mcp-connectors", "/reddit", "/instagram", + "/tiktok", "/youtube", "/google-maps", "/google-search", diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index d22400b7a..5422a0c90 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -6,28 +6,32 @@ export const instagram: ConnectorPageContent = { name: "Instagram", icon: IconBrandInstagram, - metaTitle: "Instagram Scraper API for Creator Research | SurfSense", + metaTitle: "Instagram Scraper API for Profiles and Reels | SurfSense", metaDescription: - "Scrape public Instagram posts, reels, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", + "Instagram scraper API for public profiles, posts, and reels. No login or Graph API review. Structured data for AI agents, plus a free tier. Start now.", keywords: [ "instagram scraper", "instagram scraper api", "instagram api", "instagram api alternative", "scrape instagram", + "instagram scraping", "instagram graph api alternative", "instagram profile scraper", "instagram post scraper", + "instagram posts scraper", "instagram reel scraper", + "instagram reels scraper", + "instagram data scraper", "instagram data api", "instagram mcp server", "creator research", "social listening", ], - h1: "Instagram Scraper API for Creator Research and Social Listening", + h1: "Instagram Scraper API for Profiles, Posts, and Reels", heroLede: - "The SurfSense Instagram API extracts public posts, reels, and profile details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.", + "The SurfSense Instagram scraper extracts public profiles, posts, and reels without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.", transcript: { prompt: "Pull recent reels from @competitor and summarize what they're posting", @@ -54,48 +58,48 @@ export const instagram: ConnectorPageContent = { }, extractIntro: - "Every call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.", + "Every Instagram scraper call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.", extractFields: [ { - label: "Posts & Reels", + label: "Posts and reels", description: - "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.", + "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp from any public post or reel.", }, { - label: "Profiles", + label: "Profile data", description: - "Follower, following, and post counts, bio, external URL, verified and business flags.", + "Follower, following, and post counts, bio, external URL, verified and business flags for public Instagram profiles.", }, { - label: "Owner & Media", + label: "Owner and media", description: "Owner username and id on every item, plus image and video URLs, alt text, and view counts.", }, ], - useCasesHeading: "What teams do with the Instagram API", + useCasesHeading: "What teams do with the Instagram scraper API", useCases: [ { title: "Creator and competitor monitoring", description: - "Track what your competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.", + "Scrape Instagram profiles and feeds to track what competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.", }, { title: "Content and format research", description: - "Study a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.", + "Pull a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.", }, { title: "Influencer vetting and outreach", description: - "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership.", + "Use the Instagram profile scraper to verify follower count, post cadence, and real engagement from public data before you pay for a partnership.", }, ], comparison: { - heading: "An Instagram API alternative built for agents", + heading: "An Instagram Graph API alternative built for agents", intro: - "The official Instagram Graph API requires a Business account, app review, and access tokens, and it can't read arbitrary public profiles. Here is how SurfSense compares.", + "The official Instagram Graph API requires a Business account, app review, and access tokens, and it cannot read arbitrary public profiles. SurfSense is an Instagram API alternative for public data. Here is how it compares.", columnLabel: "Instagram Graph API", rows: [ { @@ -259,7 +263,12 @@ export const instagram: ConnectorPageContent = { { question: "Do I need an Instagram account or the Graph API?", answer: - "No. This is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured data back.", + "No. This Instagram scraper API is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call SurfSense with one key, or add the MCP server to your agent, and get structured profile, post, and reel data back.", + }, + { + question: "What Instagram data can I scrape?", + answer: + "Public profiles, posts, and reels. Each item includes captions, hashtags, mentions, engagement counts, media URLs, and owner metadata. Point the Instagram profile scraper at a handle, or pass post and reel URLs directly. Discover creators with search queries when you do not have a URL yet.", }, { question: "What are the rate limits?", @@ -274,11 +283,11 @@ export const instagram: ConnectorPageContent = { ], related: [ - { label: "Reddit API", href: "/reddit" }, + { label: "TikTok API", href: "/tiktok" }, { label: "YouTube API", href: "/youtube" }, + { label: "Reddit API", href: "/reddit" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, - { label: "Read the docs", href: "/docs" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index d77865633..f245de92b 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -6,30 +6,33 @@ export const tiktok: ConnectorPageContent = { name: "TikTok", icon: IconBrandTiktok, - metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", + metaTitle: "TikTok Scraper API for Videos and Comments | SurfSense", metaDescription: - "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, profile, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + "TikTok scraper API for public videos, comments, hashtags, and profiles. No Research API approval. Structured data for AI agents, plus a free tier. Start now.", keywords: [ "tiktok scraper", "tiktok scraper api", "tiktok api", "tiktok api alternative", "scrape tiktok", + "tiktok scraping", + "tiktok research api alternative", "tiktok data api", - "tiktok hashtag scraper", "tiktok comments scraper", + "tiktok comment scraper", + "tiktok hashtag scraper", + "tiktok profile scraper", + "tiktok video scraper", "tiktok trending scraper", "tiktok user search", - "tiktok trend tracking", "tiktok mcp", "social listening", "influencer research tool", - "short-form video analytics", ], - h1: "TikTok Scraper API for Trend and Creator Research", + h1: "TikTok Scraper API for Videos, Comments, and Trend Research", heroLede: - "The SurfSense TikTok API extracts public videos by hashtag, creator profile, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", + "The SurfSense TikTok scraper extracts public videos by hashtag, creator profile, or URL, plus comment threads and trending feeds, without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", transcript: { prompt: "Find trending TikToks about meal prep this week", @@ -55,46 +58,47 @@ export const tiktok: ConnectorPageContent = { }, extractIntro: - "Every call returns structured items. Scrape videos from a hashtag, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", + "Every TikTok scraper call returns structured items. Scrape videos from a hashtag, creator profile, or video URL, or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", extractFields: [ { label: "Videos", - description: "Caption text, canonical web URL, duration, and cover image for each video.", + description: "Caption text, canonical web URL, duration, and cover image for each TikTok video.", }, { label: "Engagement", description: - "Play, like, comment, share, and save counts — the signal for what is breaking out.", + "Play, like, comment, share, and save counts, the signal for what is breaking out.", }, { - label: "Authors", + label: "Authors and profiles", description: "Creator handle, nickname, follower and heart counts, and verified status.", }, + { + label: "Comments", + description: + "Public comment threads on any video URL, so you can read audience reaction beyond vanity views.", + }, { label: "Music", - description: "Track name, artist, and whether the sound is original — the seed of a trend.", + description: "Track name, artist, and whether the sound is original, the seed of a trend.", }, { label: "Hashtags", description: "Every hashtag on a video, so you can map a topic cluster or campaign.", }, - { - label: "Timestamps", - description: "Created and scraped times so you can track a video's momentum over runs.", - }, ], - useCasesHeading: "What teams do with the TikTok API", + useCasesHeading: "What teams do with the TikTok scraper API", useCases: [ { title: "Trend and hashtag monitoring", description: - "Track a hashtag and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", + "Use the TikTok hashtag scraper to track a topic and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", }, { title: "Creator and influencer discovery", description: - "Surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.", + "Scrape TikTok profiles and search users by keyword to surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.", }, { title: "Competitor content analysis", @@ -102,16 +106,16 @@ export const tiktok: ConnectorPageContent = { "Watch what your category posts and what actually lands. Turn a competitor's best-performing formats and hooks into your own content brief.", }, { - title: "Campaign and sentiment tracking", + title: "Campaign and comment sentiment", description: - "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — then pull the comments on top videos to read how the audience actually reacts, not just a vanity view count.", + "Measure how a launch or branded hashtag spreads across TikTok, then use the TikTok comments scraper on top videos to read how the audience actually reacts, not just a vanity view count.", }, ], comparison: { - heading: "A TikTok API alternative built for agents", + heading: "A TikTok Research API alternative built for agents", intro: - "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need it for commercial research, here is how SurfSense compares.", + "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need commercial TikTok scraping, SurfSense is a TikTok API alternative. Here is how it compares.", columnLabel: "Official TikTok API", rows: [ { @@ -180,7 +184,7 @@ export const tiktok: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "Keyword search terms. Keyword video search is login-walled and returns no videos — use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", + "Keyword search terms. Keyword video search is login-walled and returns no videos; use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", }, { name: "results_per_page", @@ -253,9 +257,19 @@ export const tiktok: ConnectorPageContent = { "SurfSense reads only public TikTok data, the same videos any logged-out visitor can see. It never logs in and cannot access private or deleted content. As always, review TikTok's terms and your own compliance needs before you run at scale.", }, { - question: "Does this need the official TikTok API?", + question: "Does this need the official TikTok Research API?", answer: - "No. It is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured videos back.", + "No. This TikTok scraper API is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call SurfSense with one key, or add the MCP server to your agent, and get structured videos, comments, and profile data back.", + }, + { + question: "What TikTok data can I scrape?", + answer: + "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + }, + { + question: "Can I scrape TikTok comments and hashtags?", + answer: + "Yes. Pass a video URL to the comments endpoint for the public comment thread. Pass hashtag names or /tag/ URLs to the TikTok hashtag scraper to pull videos under that tag. Keyword video search is login-walled, so hashtags and direct URLs are the reliable discovery paths.", }, { question: "What are the rate limits?", @@ -265,18 +279,14 @@ export const tiktok: ConnectorPageContent = { { question: "Can I scrape a specific creator's videos?", answer: - "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag or by a direct video URL.", - }, - { - question: "What TikTok data can I scrape?", - answer: - "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + "Pass a profile or profile URL and you always get the account's metadata: name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account. For reliable video results, scrape by hashtag or by a direct video URL.", }, ], related: [ - { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "YouTube API", href: "/youtube" }, + { label: "Reddit API", href: "/reddit" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, From 1bc7d9f51c7015917a0a3af9e584a6eb03a6faaa Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 15:03:24 -0700 Subject: [PATCH 157/160] Update README files in multiple languages to include Instagram and TikTok as new data sources for live scraping, enhancing the platform's competitive intelligence capabilities. --- .github/workflows/desktop-release.yml | 15 +++++++++++++++ README.es.md | 6 ++++-- README.hi.md | 6 ++++-- README.md | 6 ++++-- README.pt-BR.md | 6 ++++-- README.zh-CN.md | 6 ++++-- 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 3ad529671..7955adcbe 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -75,6 +75,21 @@ jobs: echo "Windows signing: skipped" fi + - name: Verify Apple notarization credentials + if: matrix.os == 'macos-latest' + shell: bash + # Fails in seconds with the same 401 notarytool would throw after the + # full build, instead of wasting ~15 min of build time on bad secrets. + run: | + xcrun notarytool history \ + --apple-id "$APPLE_ID" \ + --password "$APPLE_APP_SPECIFIC_PASSWORD" \ + --team-id "$APPLE_TEAM_ID" > /dev/null + env: + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + - name: Setup pnpm uses: pnpm/action-setup@v5 diff --git a/README.es.md b/README.es.md index ecdd93c41..08231b5ab 100644 --- a/README.es.md +++ b/README.es.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para investigación de inteligencia competitiva -SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. +SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. > [!NOTE] > **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM** @@ -103,6 +103,8 @@ Las automatizaciones ejecutan turnos completos de agente según un horario o en |---|---|---| | **Reddit** | Publicaciones, comentarios y flujos de subreddits sin los límites de tasa de la API oficial | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | Videos, transcripciones e hilos de comentarios para escuchar a tu marca y productos | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Perfiles, publicaciones y reels públicos sin la Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Videos, comentarios, hashtags y perfiles sin aprobación de la Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Lugares, calificaciones y reseñas para investigar competidores locales y prospectos | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs en vivo para seguimiento de posiciones y monitoreo de mercado | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** (rastreo web) | Cualquier página de la web abierta como contenido limpio y estructurado | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | Característica | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Google Maps, Google Search y rastreo web vía API REST y MCP | +| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y rastreo web vía API REST y MCP | | **Servidor MCP** | No | Cada conector expuesto como herramienta nativa de agente, más servidores MCP propios con aplicaciones OAuth de un clic | | **Fuentes por Notebook** | 50 (gratis) a 600 (Ultra, $249.99/mes) | Ilimitadas | | **Número de Notebooks** | 100 (gratis) a 500 (niveles de pago) | Ilimitado | diff --git a/README.hi.md b/README.hi.md index 3cb769318..eb65ff489 100644 --- a/README.hi.md +++ b/README.hi.md @@ -22,7 +22,7 @@ # SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM -SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। +SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। > [!NOTE] > **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना** @@ -103,6 +103,8 @@ SurfSense **AI एजेंट्स के लिए ओपन सोर्स |---|---|---| | **Reddit** | आधिकारिक API की रेट लिमिट के बिना पोस्ट, कमेंट और सबरेडिट स्ट्रीम | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | ब्रांड और प्रोडक्ट लिसनिंग के लिए वीडियो, ट्रांसक्रिप्ट और कमेंट थ्रेड | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Graph API के बिना सार्वजनिक प्रोफ़ाइल, पोस्ट और रील्स | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Research API अप्रूवल के बिना वीडियो, कमेंट, हैशटैग और प्रोफ़ाइल | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | स्थानीय प्रतिस्पर्धी और लीड रिसर्च के लिए स्थान, रेटिंग और रिव्यू | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | रैंक ट्रैकिंग और मार्केट मॉनिटरिंग के लिए लाइव SERP | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** | ओपन वेब का कोई भी पेज साफ़-सुथरे, स्ट्रक्चर्ड कंटेंट के रूप में | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | फ़ीचर | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Google Maps, Google Search और वेब क्रॉल कनेक्टर | +| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और वेब क्रॉल कनेक्टर | | **MCP सर्वर** | नहीं | हर कनेक्टर नेटिव एजेंट टूल के रूप में उपलब्ध, साथ ही वन-क्लिक OAuth ऐप्स के साथ अपने MCP सर्वर लाने की सुविधा | | **प्रति नोटबुक स्रोत** | 50 (Free) से 600 (Ultra, $249.99/माह) | असीमित | | **नोटबुक की संख्या** | 100 (Free) से 500 (सशुल्क टियर) | असीमित | diff --git a/README.md b/README.md index eac392ebd..ba289ada4 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM for Competitive Intelligence Research -SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. +SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. > [!NOTE] > **📢 A note for our NotebookLM-alternative users** @@ -103,6 +103,8 @@ Automations run full agent turns on a schedule or in response to events, then wr |---|---|---| | **Reddit** | Posts, comments, and subreddit streams without the official API's rate limits | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | Videos, transcripts, and comment threads for brand and product listening | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Public profiles, posts, and reels without the Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Videos, comments, hashtags, and profiles without Research API approval | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Places, ratings, and reviews for local competitor and lead research | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | Live SERPs for rank tracking and market monitoring | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** | Any page on the open web as clean, structured content | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -243,7 +245,7 @@ Still comparing us as a NotebookLM alternative? Here is the honest breakdown. | Feature | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Live market data for agents** | No | Reddit, YouTube, Google Maps, Google Search, and web crawl connectors via REST API and MCP | +| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and web crawl connectors via REST API and MCP | | **MCP server** | No | Every connector exposed as a native agent tool, plus bring-your-own MCP servers with one-click OAuth apps | | **Sources per Notebook** | 50 (Free) to 600 (Ultra, $249.99/mo) | Unlimited | | **Number of Notebooks** | 100 (Free) to 500 (paid tiers) | Unlimited | diff --git a/README.pt-BR.md b/README.pt-BR.md index e32882831..327e3373c 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva -O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. +O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. > [!NOTE] > **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM** @@ -103,6 +103,8 @@ As automações executam turnos completos de agente de forma agendada ou em resp |---|---|---| | **Reddit** | Posts, comentários e fluxos de subreddits sem os limites de requisição da API oficial | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | Vídeos, transcrições e threads de comentários para monitoramento de marca e produto | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Perfis, posts e reels públicos sem a Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Vídeos, comentários, hashtags e perfis sem aprovação da Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Estabelecimentos, avaliações e reviews para pesquisa local de concorrentes e leads | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs ao vivo para acompanhamento de rankings e monitoramento de mercado | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** (rastreamento web) | Qualquer página da web aberta como conteúdo limpo e estruturado | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ Ainda nos comparando como alternativa ao NotebookLM? Aqui está o comparativo ho | Recurso | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Google Maps, Google Search e rastreamento web via API REST e MCP | +| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e rastreamento web via API REST e MCP | | **Servidor MCP** | Não | Cada conector exposto como ferramenta nativa de agente, além de servidores MCP próprios com apps OAuth em um clique | | **Fontes por Notebook** | 50 (gratuito) a 600 (Ultra, US$ 249,99/mês) | Ilimitadas | | **Número de Notebooks** | 100 (gratuito) a 500 (planos pagos) | Ilimitado | diff --git a/README.zh-CN.md b/README.zh-CN.md index 11bace243..53a95968b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -22,7 +22,7 @@ # SurfSense:面向竞争情报研究的 NotebookLM -SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 NotebookLM,但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。 +SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 NotebookLM,但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**,利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和开放网络**的实时数据,监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警,内置的知识库则让每一条发现都可搜索、可引用。 > [!NOTE] > **📢 致我们的 NotebookLM 替代品用户** @@ -103,6 +103,8 @@ SurfSense 是**面向 AI 智能体的开源竞争情报平台**,就像 Noteboo |---|---|---| | **Reddit** | 帖子、评论和子版块信息流,不受官方 API 速率限制 | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | 视频、字幕转录和评论串,用于品牌和产品舆情监听 | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | 公开主页、帖子和 Reels,无需 Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | 视频、评论、话题标签和主页,无需 Research API 审批 | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | 地点、评分和评论,用于本地竞争对手和潜在客户调研 | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | 实时搜索结果页,用于排名追踪和市场监控 | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** | 把开放网络上的任意页面转为干净、结构化的内容 | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | 功能 | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Google Maps、Google Search 和网页爬取连接器 | +| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和网页爬取连接器 | | **MCP 服务器** | 无 | 每个连接器都作为原生智能体工具暴露,还可自带 MCP 服务器并使用一键 OAuth 应用 | | **每个笔记本的来源数** | 50 个(免费版)至 600 个(Ultra 版,249.99 美元/月) | 无限制 | | **笔记本数量** | 100 个(免费版)至 500 个(付费档位) | 无限制 | From 1131da5ed71bbffe3f729311da38fe1c3c1eaa0f Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 16:29:39 -0700 Subject: [PATCH 158/160] feat: bumped version to 0.0.32 --- VERSION | 2 +- .../builtins/instagram/system_prompt.md | 2 +- .../builtins/tiktok/system_prompt.md | 4 +- .../capabilities/tiktok/scrape/definition.py | 3 +- .../app/capabilities/tiktok/scrape/schemas.py | 7 +- .../platforms/instagram/scraper.py | 4 +- .../platforms/tiktok/orchestrator.py | 86 +++++++++- surfsense_backend/pyproject.toml | 2 +- .../unit/platforms/tiktok/test_discovery.py | 149 ++++++++++++++++++ surfsense_backend/uv.lock | 134 +++++++++++++++- surfsense_browser_extension/package.json | 2 +- surfsense_desktop/package.json | 2 +- surfsense_evals/scripts/analyze_failures.py | 2 - .../scripts/check_uploaded_status.py | 1 - .../scripts/compute_post_retry_accuracy.py | 4 +- .../patch_manifest_for_parallel_ingest.py | 1 - surfsense_evals/scripts/peek_crag_run.py | 7 +- surfsense_evals/scripts/peek_disagreements.py | 3 +- .../scripts/retry_failed_questions.py | 6 +- surfsense_evals/scripts/summarise_crag_run.py | 3 +- .../scripts/summarise_parser_compare_run.py | 3 +- .../src/surfsense_evals/core/cli.py | 7 +- .../surfsense_evals/core/clients/documents.py | 5 +- .../core/metrics/comparison.py | 4 +- .../surfsense_evals/core/parse/__init__.py | 2 +- .../surfsense_evals/core/parse/citations.py | 4 +- .../surfsense_evals/core/parsers/azure_di.py | 15 +- .../core/parsers/llamacloud.py | 2 +- .../core/providers/openrouter_chat.py | 2 +- .../core/providers/openrouter_pdf.py | 4 +- .../src/surfsense_evals/suites/__init__.py | 2 +- .../suites/_demo/hello/__init__.py | 1 - .../suites/medical/cure/__init__.py | 2 +- .../suites/medical/cure/ingest.py | 6 +- .../suites/medical/cure/runner.py | 5 +- .../suites/medical/mirage/__init__.py | 2 +- .../suites/medical/mirage/ingest.py | 96 +++++------ .../suites/medical/mirage/prompt.py | 1 - .../suites/medical/mirage/runner.py | 5 +- .../multimodal_doc/parser_compare/ingest.py | 4 +- .../multimodal_doc/parser_compare/runner.py | 4 +- .../suites/research/crag/html_extract.py | 5 +- .../suites/research/crag/prompt.py | 1 - .../suites/research/crag/runner.py | 4 +- .../suites/research/frames/dataset.py | 2 +- .../suites/research/frames/ingest.py | 1 - .../suites/research/frames/prompt.py | 1 - .../tests/suites/test_crag_dataset.py | 2 - .../tests/suites/test_crag_grader.py | 2 - .../tests/suites/test_crag_html_extract.py | 3 - .../tests/suites/test_frames_dataset.py | 3 - .../tests/suites/test_frames_grader.py | 3 - .../features/scrapers/platforms/instagram.py | 8 +- .../features/scrapers/platforms/tiktok.py | 18 ++- surfsense_web/package.json | 2 +- 55 files changed, 496 insertions(+), 159 deletions(-) create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py diff --git a/VERSION b/VERSION index d788d4335..78bae5bb6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.31 +0.0.32 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md index 42713cc38..ce9182771 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -13,7 +13,7 @@ Answer the delegated question from live Instagram data gathered with your verbs, - Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled). -- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). +- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). Google-backed discovery is slow (~30-60s per query), so start with **at most 3** distinct queries per task and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent. - Profile metadata (follower counts, bio, post count): call `instagram_details`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md index 03114cd12..4300e9313 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. +- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent. - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. - Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos. - Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. -- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL. +- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts. - "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. - Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). - Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py index 9f8632f3b..e2706a2ef 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py @@ -11,7 +11,8 @@ TIKTOK_SCRAPE = Capability( name="tiktok.scrape", description=( "Scrape public TikTok videos. Use urls, profiles, hashtags, or " - "search_queries." + "search_queries (search_queries are resolved via Google to public " + "videos; for accounts by keyword use tiktok.user_search)." ), input_schema=ScrapeInput, output_schema=ScrapeOutput, diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index 2f5005353..853b70bd9 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -43,7 +43,12 @@ class ScrapeInput(BaseModel): search_queries: list[str] = Field( default_factory=list, max_length=MAX_TIKTOK_SOURCES, - description="Search terms to run on TikTok.", + description=( + "Search terms resolved via Google (site:tiktok.com) to public TikTok " + "videos, since TikTok's own keyword search is login-walled. Slower " + "than hashtags/urls. To find accounts by keyword, use " + "tiktok.user_search instead." + ), ) results_per_page: int = Field( default=10, diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index a4c22bde7..9f4f6e2eb 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -306,9 +306,9 @@ async def _discover_via_google( """ serps = await scrape_serps( GoogleSearchScrapeInput( - queries=query, site="instagram.com", maxPagesPerQuery=2 + queries=query, site="instagram.com", maxPagesPerQuery=1 ), - limit=2, + limit=1, ) resolved: list[ResolvedUrl] = [] seen: set[tuple[str, str]] = set() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 5d6a24545..01fae6caf 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,6 +12,9 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote +from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput +from app.proprietary.platforms.google_search.scraper import scrape_serps + from .extraction.timestamps import now_iso from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn from .flows.comments import iter_comments @@ -35,9 +38,24 @@ _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" _SEARCH_URL = "https://www.tiktok.com/search?q={query}" _EXPLORE_URL = "https://www.tiktok.com/explore" +# A ``searchQueries`` term whose Google discovery surfaced no scrapable video +# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract: +# never vanish silently). +_EMPTY_DISCOVERY_MESSAGE = ( + "No public TikTok videos found for this query via Google discovery. Try a " + "narrower phrasing, a hashtag, or a direct video URL." +) + def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: - """Build the target list from every input source, dropping unresolved URLs.""" + """Build the target list from the URL/profile/hashtag sources. + + ``searchQueries`` is deliberately excluded: TikTok's own keyword search is + login-walled for anonymous sessions, so it is routed through Google video + discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...`` + URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and + keeps its native listing routing. + """ targets: list[TikTokTarget] = [] for entry in input_model.startUrls: resolved = resolve_target(entry.url) @@ -52,13 +70,42 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name))) for tag in input_model.hashtags: targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag))) - for query in input_model.searchQueries: - targets.append( - TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query))) - ) return targets +async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]: + """Discover public TikTok video targets via Google ``site:tiktok.com``. + + TikTok's anonymous keyword search is login-walled, so we reuse the existing + ``google_search`` platform, classify each organic URL with ``resolve_target``, + and keep only video hits (``/@user/video/``) — the one kind that scrapes + reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are + dropped (accounts belong to the ``user_search`` verb). De-duped, capped at + ``limit``. + """ + serps = await scrape_serps( + GoogleSearchScrapeInput( + queries=query, site="tiktok.com", maxPagesPerQuery=1 + ), + limit=1, + ) + resolved: list[TikTokTarget] = [] + seen: set[str] = set() + for serp in serps: + for org in serp.get("organicResults") or []: + url = org.get("url", "") if isinstance(org, dict) else "" + target = resolve_target(url) + if target is None or target.kind != "video": + continue + if target.value in seen: + continue + seen.add(target.value) + resolved.append(target) + if len(resolved) >= limit: + return resolved + return resolved + + def _dispatch( target: TikTokTarget, *, @@ -81,9 +128,11 @@ async def iter_tiktok( ) -> AsyncIterator[dict[str, Any]]: """Yield normalized items for every resolved target, in order. - The video flow's ``fetch_html`` opens its own warmed proxy session per call - when none is bound; the listing flow drives its own browser. Neither binds a - ContextVar across these ``yield``s, so the generator stays context-safe. + Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries`` + then run through Google video discovery. The video flow's ``fetch_html`` + opens its own warmed proxy session per call when none is bound; the listing + flow drives its own browser. Neither binds a ContextVar across these + ``yield``s, so the generator stays context-safe. """ cap = input_model.resultsPerPage for target in _resolve_targets(input_model): @@ -92,6 +141,27 @@ async def iter_tiktok( ): yield item + # searchQueries -> Google-discovered public video URLs, de-duped across + # queries so the same video surfacing under two terms is scraped once. + seen_videos: set[str] = set() + for query in input_model.searchQueries: + discovered = await _discover_via_google(query, limit=cap) + if not discovered: + yield ErrorItem( + url=_SEARCH_URL.format(query=quote(query)), + input=query, + error=_EMPTY_DISCOVERY_MESSAGE, + errorCode="no_items", + scrapedAt=now_iso(), + ).to_output() + continue + for target in discovered: + if target.value in seen_videos: + continue + seen_videos.add(target.value) + async for item in iter_video(target, fetch=fetch): + yield item + async def scrape_tiktok( input_model: TikTokScrapeInput, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 2faa2be17..30bb7ea5c 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "surf-new-backend" -version = "0.0.31" +version = "0.0.32" description = "SurfSense Backend" requires-python = ">=3.12" dependencies = [ diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py new file mode 100644 index 000000000..4664f85e0 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py @@ -0,0 +1,149 @@ +"""Offline tests for Google-backed TikTok video discovery. + +``searchQueries`` are login-walled on TikTok's native search, so they route +through the ``google_search`` platform (``site:tiktok.com``): each organic URL +is classified with ``resolve_target`` and only video hits (``/@user/video/``) +are kept — profiles/hashtags/search/photo/non-tiktok are dropped (accounts +belong to the user-search verb). These tests inject a fake ``scrape_serps`` so +there is no network: they pin the classification, cross-query de-dup, the limit +cap, the barren-query ErrorItem, and that no ``/search?q=`` listing target is +ever built. +""" + +from __future__ import annotations + +import json + +from app.proprietary.platforms.tiktok import ( + TikTokScrapeInput, + orchestrator, + scrape_tiktok, +) + + +def _fake_serps(*organic_urls: str): + async def _scrape_serps(input_model, *, limit=None): + assert input_model.site == "tiktok.com" + assert input_model.maxPagesPerQuery == 1 + return [{"organicResults": [{"url": u} for u in organic_urls]}] + + return _scrape_serps + + +def _video_page(url: str) -> str: + """Render a rehydration blob for a ``/@user/video/`` URL.""" + video_id = url.rsplit("/", 1)[1] + username = url.split("@")[1].split("/")[0] + blob = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": { + "itemInfo": { + "itemStruct": { + "id": video_id, + "desc": "hi", + "author": {"uniqueId": username}, + "stats": {"diggCount": 1}, + } + } + } + } + } + return ( + '' + ) + + +async def _fetch_video(url: str) -> str: + return _video_page(url) + + +async def test_search_discovery_keeps_only_videos(monkeypatch): + # Only the video URL survives; profile / hashtag / search / photo / + # non-tiktok organic results are dropped. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps( + "https://www.tiktok.com/@nasa/video/123", + "https://www.tiktok.com/@nasa", + "https://www.tiktok.com/tag/space", + "https://www.tiktok.com/search?q=space", + "https://www.tiktok.com/@nasa/photo/999", + "https://example.com/not-tiktok", + ), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10), + fetch=_fetch_video, + ) + assert [i["id"] for i in items] == ["123"] + + +async def test_search_discovery_dedupes_across_queries(monkeypatch): + # The same video surfacing under two queries is scraped once. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps("https://www.tiktok.com/@nasa/video/123"), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space", "rockets"], resultsPerPage=10), + fetch=_fetch_video, + ) + assert [i["id"] for i in items] == ["123"] + + +async def test_search_discovery_respects_per_target_limit(monkeypatch): + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps( + "https://www.tiktok.com/@a/video/1", + "https://www.tiktok.com/@b/video/2", + "https://www.tiktok.com/@c/video/3", + ), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["x"], resultsPerPage=2), + fetch=_fetch_video, + ) + assert [i["id"] for i in items] == ["1", "2"] + + +async def test_search_barren_query_emits_error_item(monkeypatch): + # A query whose discovery finds no video URLs degrades to one ErrorItem. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps( + "https://www.tiktok.com/@nasa", + "https://example.com/x", + ), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10), + fetch=_fetch_video, + ) + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" + assert items[0]["input"] == "space" + + +async def test_search_never_builds_listing_target(monkeypatch): + # searchQueries must never hit the (login-walled) native search listing flow. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps("https://www.tiktok.com/@nasa/video/123"), + ) + + async def _boom_listing(_url: str, _count: int) -> list[dict]: + raise AssertionError("searchQueries must not build a listing target") + + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10), + fetch=_fetch_video, + fetch_listing=_boom_listing, + ) + assert [i["id"] for i in items] == ["123"] diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index a5b492fc8..2b1272363 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -36,6 +36,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -70,6 +73,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -104,6 +110,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -138,6 +147,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -172,6 +184,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -206,6 +221,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -240,6 +258,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -274,6 +295,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -308,6 +332,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -342,6 +369,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -387,6 +417,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -421,6 +455,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -455,6 +492,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -489,6 +529,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -523,6 +566,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -557,6 +603,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -591,6 +640,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -625,6 +677,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -659,6 +714,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -693,6 +751,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -727,6 +788,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -772,6 +836,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -806,6 +874,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -840,6 +911,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -874,6 +948,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -908,6 +985,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -942,6 +1022,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -976,6 +1059,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1010,6 +1096,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1044,6 +1133,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1078,6 +1170,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1112,6 +1207,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1157,6 +1255,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1191,6 +1293,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1258,6 +1363,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1292,6 +1403,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1326,6 +1440,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1393,6 +1510,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1460,6 +1583,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1494,6 +1623,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -10353,7 +10485,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.31" +version = "0.0.32" source = { editable = "." } dependencies = [ { name = "alembic" }, diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index 8dda0d82f..a2c25a602 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.31", + "version": "0.0.32", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index f12b9722c..629fe801d 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -1,7 +1,7 @@ { "name": "surfsense-desktop", "productName": "SurfSense", - "version": "0.0.31", + "version": "0.0.32", "description": "SurfSense Desktop App", "main": "dist/main.js", "scripts": { diff --git a/surfsense_evals/scripts/analyze_failures.py b/surfsense_evals/scripts/analyze_failures.py index e7ace1e1b..ff5ec23f4 100644 --- a/surfsense_evals/scripts/analyze_failures.py +++ b/surfsense_evals/scripts/analyze_failures.py @@ -12,12 +12,10 @@ Outputs (printed to stdout + written to `failures_n171.json`): from __future__ import annotations import json -import re from collections import Counter, defaultdict from pathlib import Path from typing import Any - REPO = Path(__file__).resolve().parents[1] RUN = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare" RAW = RUN / "raw.jsonl" diff --git a/surfsense_evals/scripts/check_uploaded_status.py b/surfsense_evals/scripts/check_uploaded_status.py index 7021ba83d..1903502ed 100644 --- a/surfsense_evals/scripts/check_uploaded_status.py +++ b/surfsense_evals/scripts/check_uploaded_status.py @@ -15,7 +15,6 @@ from pathlib import Path import httpx from dotenv import load_dotenv - REPO = Path(__file__).resolve().parents[1] PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs" diff --git a/surfsense_evals/scripts/compute_post_retry_accuracy.py b/surfsense_evals/scripts/compute_post_retry_accuracy.py index 4c8c47672..f3a716219 100644 --- a/surfsense_evals/scripts/compute_post_retry_accuracy.py +++ b/surfsense_evals/scripts/compute_post_retry_accuracy.py @@ -47,9 +47,7 @@ def _row_key(row: dict) -> tuple[str, str]: def _is_failure(row: dict) -> bool: if row.get("error"): return True - if not (row.get("raw_text") or "").strip(): - return True - return False + return bool(not (row.get("raw_text") or "").strip()) def _summarise(rows_by_arm: dict[str, list[dict]]) -> dict[str, dict]: diff --git a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py index e1a2edc65..8cfb13eb7 100644 --- a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py +++ b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py @@ -27,7 +27,6 @@ from __future__ import annotations import json from pathlib import Path - REPO = Path(__file__).resolve().parents[1] MAP_PATH = REPO / "data" / "multimodal_doc" / "maps" / "mmlongbench_doc_map.jsonl" PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs" diff --git a/surfsense_evals/scripts/peek_crag_run.py b/surfsense_evals/scripts/peek_crag_run.py index 225e5ec98..3a79d76c7 100644 --- a/surfsense_evals/scripts/peek_crag_run.py +++ b/surfsense_evals/scripts/peek_crag_run.py @@ -10,19 +10,20 @@ from collections import defaultdict def main() -> None: raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1] print(f"Reading: {raw_path}") - rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()] + with open(raw_path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] by_q: dict[str, dict[str, dict]] = defaultdict(dict) for r in rows: by_q[r["qid"]][r["arm"]] = r for qid, arms in list(by_q.items()): b = arms.get("bare_llm", {}) - l = arms.get("long_context", {}) + lc = arms.get("long_context", {}) s = arms.get("surfsense", {}) print(f"\n=== {qid} ({b.get('domain')}/{b.get('question_type')}) ===") print(f" question: {b.get('extra', {}).get('question', '?')!r}") print(f" gold: {b.get('gold')!r}") - for arm_name, a in (("bare_llm", b), ("long_context", l), ("surfsense", s)): + for arm_name, a in (("bare_llm", b), ("long_context", lc), ("surfsense", s)): grade = a.get("graded", {}) text = (a.get("raw_text") or "").strip() tail = text[-200:] if text else "" diff --git a/surfsense_evals/scripts/peek_disagreements.py b/surfsense_evals/scripts/peek_disagreements.py index c0fe0acd9..b6497570e 100644 --- a/surfsense_evals/scripts/peek_disagreements.py +++ b/surfsense_evals/scripts/peek_disagreements.py @@ -10,7 +10,8 @@ from collections import defaultdict def main() -> None: raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1] print(f"Reading: {raw_path}") - rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()] + with open(raw_path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] by_q: dict[str, dict[str, dict]] = defaultdict(dict) for r in rows: by_q[r["qid"]][r["arm"]] = r diff --git a/surfsense_evals/scripts/retry_failed_questions.py b/surfsense_evals/scripts/retry_failed_questions.py index 7cc9478e0..25facfe60 100644 --- a/surfsense_evals/scripts/retry_failed_questions.py +++ b/surfsense_evals/scripts/retry_failed_questions.py @@ -106,9 +106,7 @@ def _is_failure_row(row: dict[str, Any]) -> bool: if row.get("error"): return True - if not (row.get("raw_text") or "").strip(): - return True - return False + return bool(not (row.get("raw_text") or "").strip()) @dataclass @@ -428,7 +426,7 @@ async def _run(args: argparse.Namespace) -> int: if f.arm == "native_pdf": pdf_path = Path(map_row["pdf_path"]) - if not pdf_path.exists(): + if not await asyncio.to_thread(pdf_path.exists): logger.error("PDF missing on disk: %s — skipping", pdf_path) continue request = _build_native_request( diff --git a/surfsense_evals/scripts/summarise_crag_run.py b/surfsense_evals/scripts/summarise_crag_run.py index 646fb6a97..9722cd391 100644 --- a/surfsense_evals/scripts/summarise_crag_run.py +++ b/surfsense_evals/scripts/summarise_crag_run.py @@ -11,7 +11,8 @@ def main() -> None: if not runs: print("(no CRAG runs found)") return - m = json.load(open(runs[-1], encoding="utf-8")) + with open(runs[-1], encoding="utf-8") as fh: + m = json.load(fh) metrics = m["metrics"] print(f"Reading: {runs[-1]}") diff --git a/surfsense_evals/scripts/summarise_parser_compare_run.py b/surfsense_evals/scripts/summarise_parser_compare_run.py index c54d82784..c091043aa 100644 --- a/surfsense_evals/scripts/summarise_parser_compare_run.py +++ b/surfsense_evals/scripts/summarise_parser_compare_run.py @@ -16,7 +16,6 @@ import statistics from collections import defaultdict from pathlib import Path - REPO = Path(__file__).resolve().parents[1] RUN_DIR = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare" RAW = RUN_DIR / "raw.jsonl" @@ -91,7 +90,7 @@ def main() -> None: print() print("by answer_format (accuracy):") - formats = sorted({f for m in arm_metrics.values() for f in m["by_format"].keys()}) + formats = sorted({f for m in arm_metrics.values() for f in m["by_format"]}) header = f"{'arm':<25} " + " ".join(f"{f:>10}" for f in formats) print(header) print("-" * len(header)) diff --git a/surfsense_evals/src/surfsense_evals/core/cli.py b/surfsense_evals/src/surfsense_evals/core/cli.py index 17979fba0..2dcbe7327 100644 --- a/surfsense_evals/src/surfsense_evals/core/cli.py +++ b/surfsense_evals/src/surfsense_evals/core/cli.py @@ -32,14 +32,13 @@ from __future__ import annotations import argparse import asyncio +import contextlib import json import logging import sys from dataclasses import dataclass from typing import Any -import sys - import httpx from rich.console import Console from rich.table import Table @@ -51,10 +50,8 @@ from rich.table import Table # Terminal, PowerShell, cmd) all interpret ANSI escapes natively. if sys.platform == "win32": for _stream in (sys.stdout, sys.stderr): - try: + with contextlib.suppress(AttributeError, ValueError): _stream.reconfigure(encoding="utf-8", errors="replace") - except (AttributeError, ValueError): - pass from . import registry from .auth import CredentialError, acquire_token, client_with_auth diff --git a/surfsense_evals/src/surfsense_evals/core/clients/documents.py b/surfsense_evals/src/surfsense_evals/core/clients/documents.py index 362aae53b..b96bef6d8 100644 --- a/surfsense_evals/src/surfsense_evals/core/clients/documents.py +++ b/surfsense_evals/src/surfsense_evals/core/clients/documents.py @@ -18,6 +18,7 @@ Document processing is asynchronous: from __future__ import annotations import asyncio +import contextlib import logging import mimetypes from collections.abc import Iterable, Sequence @@ -157,10 +158,8 @@ class DocumentsClient: ) finally: for _, (_, file_obj, _) in opened: - try: + with contextlib.suppress(Exception): file_obj.close() - except Exception: # noqa: BLE001 - pass response.raise_for_status() return FileUploadResult.from_payload(response.json()) diff --git a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py index 579576f4f..9e40dc5ca 100644 --- a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py +++ b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py @@ -71,8 +71,8 @@ def mcnemar_test( f"Length mismatch: arm_a={len(arm_a_correct)}, arm_b={len(arm_b_correct)}" ) n = len(arm_a_correct) - b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct) if a and not c) - c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct) if (not a) and cc) + b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct, strict=False) if a and not c) + c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct, strict=False) if (not a) and cc) discordant = b + c if discordant == 0: return McnemarResult( diff --git a/surfsense_evals/src/surfsense_evals/core/parse/__init__.py b/surfsense_evals/src/surfsense_evals/core/parse/__init__.py index 208c2d374..8b90de78b 100644 --- a/surfsense_evals/src/surfsense_evals/core/parse/__init__.py +++ b/surfsense_evals/src/surfsense_evals/core/parse/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations from .answer_letter import AnswerLetterResult, extract_answer_letter -from .citations import CITATION_REGEX, CitationToken, ChunkCitation, UrlCitation, parse_citations +from .citations import CITATION_REGEX, ChunkCitation, CitationToken, UrlCitation, parse_citations from .freeform_answer import extract_freeform_answer from .sse import SseEvent, iter_sse_events diff --git a/surfsense_evals/src/surfsense_evals/core/parse/citations.py b/surfsense_evals/src/surfsense_evals/core/parse/citations.py index 1fcd35434..c57ffeb0b 100644 --- a/surfsense_evals/src/surfsense_evals/core/parse/citations.py +++ b/surfsense_evals/src/surfsense_evals/core/parse/citations.py @@ -15,7 +15,7 @@ from __future__ import annotations import re from dataclasses import dataclass -from typing import Any, Union +from typing import Any # Pattern preserves the TS source verbatim: # /[\[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g @@ -64,7 +64,7 @@ class UrlCitation: return {"kind": "url", "url": self.url} -CitationToken = Union[ChunkCitation, UrlCitation] +CitationToken = ChunkCitation | UrlCitation def parse_citations(text: str, *, url_map: dict[str, str] | None = None) -> list[CitationToken]: diff --git a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py index eebad906a..7d796ee99 100644 --- a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py +++ b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py @@ -25,6 +25,7 @@ import asyncio import logging import os import random +from pathlib import Path logger = logging.getLogger(__name__) @@ -82,7 +83,7 @@ async def parse_with_azure_di( ServiceResponseError, ) - file_size_mb = os.path.getsize(file_path) / (1024 * 1024) + file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024) logger.info( "Azure DI parsing %s (mode=%s, model=%s, size=%.1fMB)", file_path, processing_mode, model_id, file_size_mb, @@ -96,12 +97,12 @@ async def parse_with_azure_di( credential=AzureKeyCredential(api_key), ) async with client: - with open(file_path, "rb") as fh: - poller = await client.begin_analyze_document( - model_id, - body=fh, - output_content_format=DocumentContentFormat.MARKDOWN, - ) + body = await asyncio.to_thread(Path(file_path).read_bytes) + poller = await client.begin_analyze_document( + model_id, + body=body, + output_content_format=DocumentContentFormat.MARKDOWN, + ) result = await poller.result() content = (result.content or "").strip() if not content: diff --git a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py index ba3d787ef..300b4ec87 100644 --- a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py +++ b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py @@ -98,7 +98,7 @@ async def parse_with_llamacloud( from llama_cloud_services.parse.base import JobFailedException from llama_cloud_services.parse.utils import ResultType - file_size_mb = os.path.getsize(file_path) / (1024 * 1024) + file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024) # Match backend's per-page timeout heuristic so big PDFs don't drop # mid-job: 60s baseline + 30s/page (premium agent runs longer than # basic; both fit comfortably here). diff --git a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py index 2494434be..208fbb865 100644 --- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py +++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py @@ -29,8 +29,8 @@ from typing import Any import httpx from .openrouter_pdf import ( - OpenRouterResponse, _DEFAULT_HEADERS, + OpenRouterResponse, _parse_chat_completion, ) diff --git a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py index e98590cbf..985d88a68 100644 --- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py +++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py @@ -34,7 +34,7 @@ import base64 import logging import time from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from pathlib import Path from typing import Any @@ -43,7 +43,7 @@ import httpx logger = logging.getLogger(__name__) -class PdfEngine(str, Enum): +class PdfEngine(StrEnum): NATIVE = "native" MISTRAL_OCR = "mistral-ocr" CLOUDFLARE_AI = "cloudflare-ai" diff --git a/surfsense_evals/src/surfsense_evals/suites/__init__.py b/surfsense_evals/src/surfsense_evals/suites/__init__.py index 95ed958ca..a0a01223c 100644 --- a/surfsense_evals/src/surfsense_evals/suites/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/__init__.py @@ -20,7 +20,7 @@ from __future__ import annotations import importlib import logging import pkgutil -from typing import Iterable +from collections.abc import Iterable logger = logging.getLogger(__name__) diff --git a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py index 1da33926c..43dc51ac5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py @@ -6,7 +6,6 @@ import argparse from typing import Any from ....core.registry import ( - Benchmark, ReportSection, RunArtifact, RunContext, diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py index e13224be7..7e9d9a07b 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py @@ -12,7 +12,7 @@ Recall@k / MRR / nDCG@10 against qrels. from __future__ import annotations -from .runner import CureBenchmark from ....core import registry as _registry +from .runner import CureBenchmark _registry.register(CureBenchmark()) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py index 275e28ce5..0c32a38a1 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py @@ -227,12 +227,10 @@ async def run_ingest( def _take(it: Iterable, n: int) -> Iterable: - yielded = 0 - for x in it: - if yielded >= n: + for i, x in enumerate(it): + if i >= n: return yield x - yielded += 1 __all__ = ["DISCIPLINES", "CorpusPassage", "PassageBatch", "run_ingest"] diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py index 041e0e8b5..4a85b6ba5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py @@ -34,7 +34,6 @@ from ....core.ingest_settings import ( ) from ....core.metrics.retrieval import score_run from ....core.registry import ( - Benchmark, ReportSection, RunArtifact, RunContext, @@ -276,7 +275,7 @@ class CureBenchmark: ) per_query_retrieved: dict[str, list[str]] = {} - for q, res in zip(queries, results): + for q, res in zip(queries, results, strict=False): chunk_ids: list[int] = [] seen: set[int] = set() for citation in res.citations: @@ -311,7 +310,7 @@ class CureBenchmark: run_dir = ctx.runs_dir(run_timestamp=run_timestamp) raw_path = run_dir / "raw.jsonl" with raw_path.open("w", encoding="utf-8") as fh: - for q, res in zip(queries, results): + for q, res in zip(queries, results, strict=False): fh.write( json.dumps( { diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py index e527b37f4..265dd62f7 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py @@ -11,7 +11,7 @@ document — the corpus is millions of biomedical snippets. from __future__ import annotations -from .runner import MirageBenchmark from ....core import registry as _registry +from .runner import MirageBenchmark _registry.register(MirageBenchmark()) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py index 59006b6c0..8e891aabf 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py @@ -93,6 +93,25 @@ class SnippetRow: # --------------------------------------------------------------------------- +def _reuse_cached_dest(dest: Path, *, expect_zip: bool, label: str) -> Path | None: + """Return ``dest`` if a usable cache hit, else ``None`` (and delete corrupt zips).""" + + if not dest.exists(): + return None + if expect_zip and not _is_valid_zip(dest): + logger.warning( + "Cached %s at %s failed ZIP validation (size=%d B); deleting " + "and re-downloading.", + label, + dest, + dest.stat().st_size, + ) + dest.unlink(missing_ok=True) + return None + logger.info("Using cached %s at %s", label, dest) + return dest + + async def _fetch_to_path( url: str, *, @@ -127,19 +146,9 @@ async def _fetch_to_path( surprise-grabbing 16 GB on a slow link. """ - if dest.exists(): - if expect_zip and not _is_valid_zip(dest): - logger.warning( - "Cached %s at %s failed ZIP validation (size=%d B); deleting " - "and re-downloading.", - label, - dest, - dest.stat().st_size, - ) - dest.unlink(missing_ok=True) - else: - logger.info("Using cached %s at %s", label, dest) - return dest + cached = _reuse_cached_dest(dest, expect_zip=expect_zip, label=label) + if cached is not None: + return cached dest.parent.mkdir(parents=True, exist_ok=True) partial = dest.with_suffix(dest.suffix + ".partial") @@ -170,39 +179,38 @@ async def _fetch_to_path( async with httpx.AsyncClient( timeout=httpx.Timeout(timeout_s, connect=20.0), follow_redirects=True, - ) as client: - async with client.stream("GET", url, headers=headers) as response: - if existing_bytes and response.status_code == 200: - logger.warning( - "Server ignored Range header for %s; restarting from 0.", - label, - ) - partial.unlink(missing_ok=True) - existing_bytes = 0 - elif response.status_code == 416: - # Range not satisfiable — the .partial is at or - # past the end. Treat as "already downloaded"; - # validate by closing and re-opening for atomic - # rename below. - logger.info( - "Server reports %s already complete (HTTP 416).", - label, - ) - elif response.status_code not in (200, 206): - response.raise_for_status() + ) as client, client.stream("GET", url, headers=headers) as response: + if existing_bytes and response.status_code == 200: + logger.warning( + "Server ignored Range header for %s; restarting from 0.", + label, + ) + partial.unlink(missing_ok=True) + existing_bytes = 0 + elif response.status_code == 416: + # Range not satisfiable — the .partial is at or + # past the end. Treat as "already downloaded"; + # validate by closing and re-opening for atomic + # rename below. + logger.info( + "Server reports %s already complete (HTTP 416).", + label, + ) + elif response.status_code not in (200, 206): + response.raise_for_status() - total_size = _planned_total_size(response, existing_bytes) - if ( - total_size is not None - and total_size > _LARGE_DOWNLOAD_BYTES - and not allow_large_download - ): - raise _LargeDownloadAbort(label, total_size) + total_size = _planned_total_size(response, existing_bytes) + if ( + total_size is not None + and total_size > _LARGE_DOWNLOAD_BYTES + and not allow_large_download + ): + raise _LargeDownloadAbort(label, total_size) - mode = "ab" if existing_bytes else "wb" - with partial.open(mode) as fh: - async for chunk in response.aiter_bytes(chunk_size=1 << 18): - fh.write(chunk) + mode = "ab" if existing_bytes else "wb" + with partial.open(mode) as fh: + async for chunk in response.aiter_bytes(chunk_size=1 << 18): + fh.write(chunk) # Optional content sanity check before promoting to dest. if expect_zip and not _is_valid_zip(partial): raise zipfile.BadZipFile( diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py index 9e5b1c618..3e5192aaa 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py @@ -8,7 +8,6 @@ from __future__ import annotations from collections.abc import Mapping - _PROMPT_TEMPLATE = """\ You are a helpful medical expert. Answer the following multiple-choice question using the relevant medical knowledge available to you (and any diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py index b01b645a9..8a6b04ae0 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py @@ -29,7 +29,6 @@ from ....core.ingest_settings import ( ) from ....core.metrics.mc_accuracy import accuracy_with_wilson_ci, macro_accuracy from ....core.registry import ( - Benchmark, ReportSection, RunArtifact, RunContext, @@ -229,7 +228,7 @@ class MirageBenchmark: run_dir = ctx.runs_dir(run_timestamp=run_timestamp) raw_path = run_dir / "raw.jsonl" with raw_path.open("w", encoding="utf-8") as fh: - for q, res in zip(questions, results): + for q, res in zip(questions, results, strict=False): fh.write( json.dumps( { @@ -246,7 +245,7 @@ class MirageBenchmark: for task in tasks: n_correct = 0 n_total = 0 - for q, res in zip(questions, results): + for q, res in zip(questions, results, strict=False): if q.task != task: continue n_total += 1 diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py index 93c8db4ab..1360ec89e 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py @@ -41,8 +41,6 @@ from typing import Any from ....core.config import set_suite_state from ....core.parsers import ( - AzureDIError, - LlamaCloudError, count_pdf_pages, parse_with_azure_di, parse_with_llamacloud, @@ -131,7 +129,7 @@ async def _run_one_extraction( else: raise ValueError(f"Unknown parser {parser!r}") out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(markdown, encoding="utf-8") + await asyncio.to_thread(out_path.write_text, markdown, encoding="utf-8") return markdown, time.monotonic() - started diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py index 2c4a0ffe4..297bd2fa0 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py @@ -603,8 +603,8 @@ class ParserCompareBenchmark: f"engine: `{extra.get('pdf_engine', 'native')}`)." ) body.append( - f"- Preprocess tariff: basic = $1 / 1k pages, " - f"premium = $10 / 1k pages." + "- Preprocess tariff: basic = $1 / 1k pages, " + "premium = $10 / 1k pages." ) body.append("") body.append("### Per-arm summary") diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py index 1b00aedc2..dd618d7e3 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py @@ -177,10 +177,7 @@ def extract_main_content( # Prefer trafilatura output even if short, but only if it # contained any prose at all — empty trafilatura fall-through # to the stripped form. - if body and stripped and len(stripped) > len(body) * 1.5: - body = stripped - method = "fallback_strip" - elif not body and stripped: + if body and stripped and len(stripped) > len(body) * 1.5 or not body and stripped: body = stripped method = "fallback_strip" elif body: diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py index 626834505..0d4327774 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py @@ -28,7 +28,6 @@ in the runner, so the format is mandatory. from __future__ import annotations - _BASE_INSTRUCTIONS = ( "You are a careful question-answering assistant. The question is a " "real-world factual question that may be about finance, music, " diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py index 654c261a2..efb7b4474 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py @@ -830,11 +830,11 @@ def _per_facet_truthfulness( """Bucket truthfulness scores by ``key_fn(q)``.""" buckets: dict[str, dict[str, list[CragGradeResult]]] = {} - for q, b, l, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False): + for q, b, lc, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False): key = key_fn(q) bucket = buckets.setdefault(key, {"bare_llm": [], "long_context": [], "surfsense": []}) bucket["bare_llm"].append(b) - bucket["long_context"].append(l) + bucket["long_context"].append(lc) bucket["surfsense"].append(s) out: dict[str, Any] = {} for key, arms in buckets.items(): diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py index c3b6b878e..629874102 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py @@ -102,7 +102,7 @@ def _parse_wiki_links(raw: Any) -> list[str]: except (SyntaxError, ValueError): # Fall back: maybe it's a comma-separated string with no quotes. return [tok.strip() for tok in text.strip("[]").split(",") if tok.strip()] - if isinstance(parsed, (list, tuple)): + if isinstance(parsed, list | tuple): return [str(x).strip() for x in parsed if str(x).strip()] return [str(parsed).strip()] diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py index 98e035f28..0288e192e 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py @@ -34,7 +34,6 @@ filename (without extension), so we round-trip via from __future__ import annotations -import asyncio import json import logging from dataclasses import dataclass diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py index 16bb06da4..8f8b6e3b6 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py @@ -17,7 +17,6 @@ Format expectations (mirrors the FRAMES paper, section 4): from __future__ import annotations - _BASE_INSTRUCTIONS = ( "You are a careful question-answering assistant. The question may " "require combining facts from multiple sources, doing arithmetic, " diff --git a/surfsense_evals/tests/suites/test_crag_dataset.py b/surfsense_evals/tests/suites/test_crag_dataset.py index 36114b52e..6221467fd 100644 --- a/surfsense_evals/tests/suites/test_crag_dataset.py +++ b/surfsense_evals/tests/suites/test_crag_dataset.py @@ -11,8 +11,6 @@ import bz2 import json from pathlib import Path -import pytest - from surfsense_evals.suites.research.crag.dataset import ( CragPage, CragQuestion, diff --git a/surfsense_evals/tests/suites/test_crag_grader.py b/surfsense_evals/tests/suites/test_crag_grader.py index 93bf6f478..74960afa6 100644 --- a/surfsense_evals/tests/suites/test_crag_grader.py +++ b/surfsense_evals/tests/suites/test_crag_grader.py @@ -7,8 +7,6 @@ exercise the deterministic shortcut + the special-case routing for from __future__ import annotations -import pytest - from surfsense_evals.suites.research.crag.grader import ( CragGradeResult, _flags_false_premise, diff --git a/surfsense_evals/tests/suites/test_crag_html_extract.py b/surfsense_evals/tests/suites/test_crag_html_extract.py index a2b47aa45..3bf757dbd 100644 --- a/surfsense_evals/tests/suites/test_crag_html_extract.py +++ b/surfsense_evals/tests/suites/test_crag_html_extract.py @@ -11,13 +11,10 @@ We don't network-fetch trafilatura; we just verify the wrapper: from __future__ import annotations -import pytest - from surfsense_evals.suites.research.crag.html_extract import ( extract_main_content, ) - _RICH_HTML = """\ diff --git a/surfsense_evals/tests/suites/test_frames_dataset.py b/surfsense_evals/tests/suites/test_frames_dataset.py index e79e7db89..f76dc0b6f 100644 --- a/surfsense_evals/tests/suites/test_frames_dataset.py +++ b/surfsense_evals/tests/suites/test_frames_dataset.py @@ -16,8 +16,6 @@ from __future__ import annotations import textwrap from pathlib import Path -import pytest - from surfsense_evals.suites.research.frames.dataset import ( FramesQuestion, _parse_reasoning_types, @@ -25,7 +23,6 @@ from surfsense_evals.suites.research.frames.dataset import ( load_questions, ) - # --------------------------------------------------------------------------- # Pure-function tests # --------------------------------------------------------------------------- diff --git a/surfsense_evals/tests/suites/test_frames_grader.py b/surfsense_evals/tests/suites/test_frames_grader.py index e6e38ff8a..d4bbad79a 100644 --- a/surfsense_evals/tests/suites/test_frames_grader.py +++ b/surfsense_evals/tests/suites/test_frames_grader.py @@ -8,10 +8,7 @@ runner knows to consult the judge. from __future__ import annotations -import pytest - from surfsense_evals.suites.research.frames.grader import ( - GradeResult, _maybe_number, _normalise, _whole_word_substring, diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py index 28ad34e66..1792bf7dd 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -41,7 +41,9 @@ def register( list[str] | None, Field( description="Terms to discover public profiles for (resolved via " - "Google). Provide search_queries OR urls." + "Google). Google-backed discovery is slow (~30-60s per query), so " + "start with at most 3 queries and expand only if nothing " + "significant is found. Provide search_queries OR urls." ), ] = None, search_type: Annotated[ @@ -100,7 +102,9 @@ def register( search_queries: Annotated[ list[str] | None, Field( - description="Terms to discover public profiles for. Provide " + description="Terms to discover public profiles for. Google-backed " + "discovery is slow (~30-60s per query), so start with at most 3 " + "queries and expand only if nothing significant is found. Provide " "search_queries OR urls." ), ] = None, diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py index c295ec3b6..7e2bff509 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -52,9 +52,11 @@ def register( search_queries: Annotated[ list[str] | None, Field( - description="Keyword search terms. Returns no videos — use " - "hashtags/profiles/urls for videos, or the user-search tool for " - "accounts." + description="Keyword search terms, resolved via Google to public " + "TikTok videos (TikTok's own keyword search is login-walled). " + "Slower than hashtags/urls, so start with at most 3 queries and " + "expand only if nothing significant is found. For accounts by " + "keyword use surfsense_tiktok_user_search." ), ] = None, results_per_page: Annotated[ @@ -67,13 +69,15 @@ def register( workspace: WorkspaceParam = None, response_format: ResponseFormatParam = "markdown", ) -> str: - """Scrape public TikTok videos by hashtag, profile, or URL. + """Scrape public TikTok videos by hashtag, profile, URL, or keyword. Use for TikTok video research — a creator's videos, a hashtag feed, or a specific video/profile/hashtag URL — instead of a generic web search. - Returns videos with text, author, stats, music, and the web URL. For - accounts by keyword use the user-search tool; keyword search returns no - videos. Example: hashtags=['food'], max_items=20. + search_queries also finds videos on a topic (resolved via Google), but is + slower: start with at most 3 queries and expand only if nothing + significant is found. Returns videos with text, author, stats, music, and + the web URL. For accounts by keyword use surfsense_tiktok_user_search. + Example: hashtags=['food'], max_items=20. """ return await run_scraper( client, diff --git a/surfsense_web/package.json b/surfsense_web/package.json index efa04b4f6..989dde480 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,6 +1,6 @@ { "name": "surfsense_web", - "version": "0.0.31", + "version": "0.0.32", "private": true, "packageManager": "pnpm@10.26.0", "description": "SurfSense Frontend", From e8b3692b546c234964b0fd3a32b39b535a3c796d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 16:30:37 -0700 Subject: [PATCH 159/160] chore: linting --- .../playground/components/output-viewer.tsx | 10 ++------- .../components/playground-index.tsx | 3 ++- .../playground/components/run-detail.tsx | 8 +------ .../components/run-progress-panel.tsx | 4 +++- .../components/run-status-badge.tsx | 5 ++++- .../playground/components/runs-table.tsx | 3 ++- .../playground/components/schema-form.tsx | 19 +++-------------- .../components/assistant-ui/thread.tsx | 5 +---- .../components/homepage/hero-chat-demo.tsx | 3 ++- .../components/homepage/hero-section.tsx | 8 +++---- .../ui/sidebar/NotificationsDropdown.tsx | 4 +--- .../settings/auto-reload-settings.tsx | 10 ++++++--- .../settings/workspace-api-access-control.tsx | 21 ++++++++++++++++--- .../content/docs/connectors/native/meta.json | 10 ++++++++- surfsense_web/hooks/use-run-stream.ts | 16 +++----------- .../lib/apis/scrapers-api.service.ts | 6 +++--- .../lib/connectors-marketing/tiktok.tsx | 3 ++- .../lib/playground/code-snippets.selfcheck.ts | 6 +++++- surfsense_web/lib/playground/format.ts | 4 +--- surfsense_web/lib/playground/json-schema.ts | 8 +------ 20 files changed, 74 insertions(+), 82 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx index e692734d3..72262d440 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx @@ -3,7 +3,6 @@ import { Check, Copy, Download } from "lucide-react"; import { useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Table, TableBody, @@ -12,6 +11,7 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { downloadCsv, rowsToCsv } from "@/lib/playground/csv"; const MAX_TABLE_ROWS = 200; @@ -117,13 +117,7 @@ export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBa
{items && items.length > 0 && ( - diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx index 58d56e146..732e1105c 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx @@ -25,7 +25,8 @@ export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) {

- Manually run SurfSense's platform-native APIs and inspect their output. To use these APIs outside SurfSense,{" "} + Manually run SurfSense's platform-native APIs and inspect their output. To use these + APIs outside SurfSense,{" "} parseJsonl(run?.output_text ?? null), [run?.output_text]); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx index 33a88ae76..a4e9f0023 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx @@ -9,7 +9,9 @@ import { formatDuration } from "@/lib/playground/format"; function eventLabel(event: ScraperRunEvent): string { const base = event.message || - (event.phase ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase()) : "Working"); + (event.phase + ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase()) + : "Working"); if (event.current !== undefined && event.current !== null) { const counter = event.total !== undefined && event.total !== null diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx index cfc75ca8a..9627fb287 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx @@ -14,7 +14,10 @@ export function RunStatusBadge({ status }: { status: string }) { } if (normalized === "success") { return ( - + Success ); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx index 6368a1f76..57b7a547c 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx @@ -71,7 +71,8 @@ export function RunsTable({ workspaceId }: { workspaceId: number }) { - View all API runs for this workspace, including runs from the playground, API keys, and agents. + View all API runs for this workspace, including runs from the playground, API keys, and + agents. diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx index 36aee26bb..1d7eec8cf 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx @@ -43,12 +43,7 @@ function FieldControl({ if (field.kind === "boolean") { return ( - + ); } @@ -141,9 +136,7 @@ function FieldRow({ {field.required ? "required" : "optional"}

- {field.description && ( -

{field.description}

- )} + {field.description &&

{field.description}

} { diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 0cf6ef296..4837c0b99 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -1018,10 +1018,7 @@ const ConnectedScraperIcons: FC<{ workspaceId: number }> = ({ workspaceId }) => return ( - + diff --git a/surfsense_web/components/homepage/hero-chat-demo.tsx b/surfsense_web/components/homepage/hero-chat-demo.tsx index e617cd51e..a4a5ebc2a 100644 --- a/surfsense_web/components/homepage/hero-chat-demo.tsx +++ b/surfsense_web/components/homepage/hero-chat-demo.tsx @@ -27,7 +27,8 @@ export type HeroChatDemoScript = { type Stage = "typing" | "steps" | "answer" | "done"; -const PLACEHOLDER = "Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs"; +const PLACEHOLDER = + "Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs"; /** Blinking caret for the typewriter (overlay only, never inside the real input). */ function Caret() { diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 05db2781f..6e31891bd 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -717,10 +717,10 @@ export function HeroSection() { "relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400" )} > - SurfSense is an open-source competitive intelligence platform, like NotebookLM but with - live scraping connectors. Your AI agents monitor competitors, track rankings, and listen - to your market with live data from platforms like Reddit, YouTube, Instagram, TikTok, - Google Maps, Google Search, and the open web. + SurfSense is an open-source competitive intelligence platform, like NotebookLM but + with live scraping connectors. Your AI agents monitor competitors, track rankings, and + listen to your market with live data from platforms like Reddit, YouTube, Instagram, + TikTok, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx b/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx index 4fe035ec2..1e9b1a33e 100644 --- a/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx +++ b/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx @@ -324,9 +324,7 @@ export function NotificationsDropdown({ )} > {tab.label} - + {formatNotificationCount(tab.count)} diff --git a/surfsense_web/components/settings/auto-reload-settings.tsx b/surfsense_web/components/settings/auto-reload-settings.tsx index ede826dd0..316c5975d 100644 --- a/surfsense_web/components/settings/auto-reload-settings.tsx +++ b/surfsense_web/components/settings/auto-reload-settings.tsx @@ -198,8 +198,8 @@ export function AutoReloadSettings() { Last top-up failed - Your saved card was declined and top-ups were turned off. Update your card and - re-enable top-ups below. + Your saved card was declined and top-ups were turned off. Update your card and re-enable + top-ups below. )} @@ -290,7 +290,11 @@ export function AutoReloadSettings() {
-