From 6e0b8516dcf7a085ad6294e11662b81d78d9c023 Mon Sep 17 00:00:00 2001 From: Benebo7 Date: Sat, 11 Jul 2026 14:38:16 -0300 Subject: [PATCH 01/28] fix: correct invalid placeholder values in backed .env.example --- surfsense_backend/.env.example | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 2c0cf38ff..3d2355460 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -120,8 +120,10 @@ STRIPE_RECONCILIATION_BATCH_SIZE=100 # BACKEND_URL=https://api.yourdomain.com # Auth -AUTH_TYPE=GOOGLE or LOCAL -REGISTRATION_ENABLED=TRUE or FALSE +# AUTH_TYPE: GOOGLE or LOCAL +AUTH_TYPE=LOCAL +# REGISTRATION_ENABLED: TRUE or FALSE +REGISTRATION_ENABLED=TRUE # For Google Auth Only GOOGLE_OAUTH_CLIENT_ID=924507538m GOOGLE_OAUTH_CLIENT_SECRET=GOCSV From a6c6b4b3d834e7c112712d87c8f5580f0341634d Mon Sep 17 00:00:00 2001 From: Benebo7 Date: Sat, 11 Jul 2026 14:40:04 -0300 Subject: [PATCH 02/28] docs: Improve contributor Docker setup guide and fix broken link --- CONTRIBUTING.md | 2 +- .../docs/docker-installation/index.mdx | 40 +++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 493f24c02..1998bd74e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ We follow a **branch protection model** to keep `main` stable: ``` 3. **Choose your setup method**: - - **Docker Setup**: Follow the [Docker Setup Guide](./DOCKER_SETUP.md) + - **Docker Setup**: Follow the [Building from Source (Contributors)](https://www.surfsense.com/docs/docker-installation#building-from-source-contributors) section of the Docker Installation guide - **Manual Setup**: Follow the [Installation Guide](https://www.surfsense.com/docs/) 4. **Configure services**: diff --git a/surfsense_web/content/docs/docker-installation/index.mdx b/surfsense_web/content/docs/docker-installation/index.mdx index a2db4e8f5..de65e3699 100644 --- a/surfsense_web/content/docs/docker-installation/index.mdx +++ b/surfsense_web/content/docs/docker-installation/index.mdx @@ -143,14 +143,48 @@ To update SurfSense, see [Updating](/docs/docker-installation/updating). ## Building from Source (Contributors) -If you're contributing to SurfSense and want to build the images from your local checkout, use the dev compose file: +If you're contributing to SurfSense and want to build the images from your local checkout, use the dev compose file. Unlike the production setup above, there's no bundled Caddy proxy — each service publishes its own port directly, and you need **three** separate `.env` files instead of one. + +**Requirements:** Docker with the `buildx` plugin (needed for the multi-stage `Dockerfile`; a plain `docker build` without BuildKit fails). Check with `docker buildx version` — if missing, install `docker-buildx` (or `docker-buildx-plugin` on some distros). ```bash -cd SurfSense/docker +git clone https://github.com//SurfSense.git +cd SurfSense +cp docker/.env.example docker/.env +cp surfsense_backend/.env.example surfsense_backend/.env +cp surfsense_web/.env.example surfsense_web/.env +``` + +`docker/.env` only configures Docker Compose variable substitution — it is **not** passed into the containers. The backend and frontend read their own `.env` files instead (wired via `env_file:` in `docker-compose.dev.yml`). + +Edit before starting: + +- **`SECRET_KEY`** in `surfsense_backend/.env` — required, generate with `openssl rand -base64 32`. Note this is separate from `SECRET_KEY` in `docker/.env`; the dev compose file does not forward one to the other. + +Everything else in the three example files (auth type, Stripe, connector OAuth credentials, messaging bots, proxy settings, etc.) is optional — only needed if you're testing that specific integration. + +```bash +cd docker docker compose -f docker-compose.dev.yml up --build ``` -It builds the backend and frontend from source, publishes raw service ports for debugging, and includes pgAdmin at [http://localhost:5050](http://localhost:5050). There's also a `docker-compose.deps-only.yml` that runs just the dependencies (Postgres, Redis, zero-cache) in Docker while you run the backend and frontend natively — see [Manual Installation](/docs/manual-installation#alternative-let-docker-manage-all-dependencies). +| Service | Port | Notes | +|---------|------|-------| +| `frontend` | `localhost:3000` | Next.js dev server | +| `backend` | `localhost:8000` | FastAPI, `/ready` for healthcheck | +| `pgadmin` | `localhost:5050` | Postgres GUI | +| `zero-cache` | `localhost:4848` (ws) | Real-time sync | +| `celery_worker` | — | Background task processing, no exposed port | +| `celery_beat` | — | Periodic task scheduler, no exposed port | +| `otel-lgtm` | `localhost:3001` | Grafana + Loki + Tempo + Mimir bundle, for tracing/metrics | + +Observability (`otel-lgtm`) isn't needed for regular dev work — it's the heaviest non-essential service, so skip it if you're not debugging traces/metrics: + +```bash +docker compose -f docker-compose.dev.yml up --build db redis zero-cache backend celery_worker celery_beat frontend +``` + +There's also a `docker-compose.deps-only.yml` that runs just the dependencies (Postgres, Redis, zero-cache) in Docker while you run the backend and frontend natively — see [Manual Installation](/docs/manual-installation#alternative-let-docker-manage-all-dependencies). ## Troubleshooting From a172e9c1624ce5ebf095333d8bf2410eb9758c8c Mon Sep 17 00:00:00 2001 From: zzh Date: Sun, 12 Jul 2026 23:25:02 +0800 Subject: [PATCH 03/28] Add raw OpenAI-compatible provider without /v1 normalization --- .../app/services/provider_registry.py | 9 +++++++++ .../unit/services/test_model_connections.py | 16 ++++++++++++++++ .../model-connections/default-connect-form.tsx | 5 ++++- .../model-connections/provider-metadata.tsx | 6 ++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/services/provider_registry.py b/surfsense_backend/app/services/provider_registry.py index 67d1c4db4..6392e38c3 100644 --- a/surfsense_backend/app/services/provider_registry.py +++ b/surfsense_backend/app/services/provider_registry.py @@ -87,6 +87,15 @@ REGISTRY: dict[str, ProviderSpec] = { "bearer", "OpenAI-compatible provider", ), + "openai_compatible_raw": ProviderSpec( + Transport.NATIVE, + "openai", + "none", + None, + True, + "bearer", + "OpenAI-compatible raw endpoint", + ), "lm_studio": ProviderSpec( Transport.OPENAI_COMPATIBLE, "openai", diff --git a/surfsense_backend/tests/unit/services/test_model_connections.py b/surfsense_backend/tests/unit/services/test_model_connections.py index b4e7c18d7..bb5ca318e 100644 --- a/surfsense_backend/tests/unit/services/test_model_connections.py +++ b/surfsense_backend/tests/unit/services/test_model_connections.py @@ -64,6 +64,22 @@ def test_openai_compatible_resolver_uses_explicit_api_base() -> None: assert ensure_v1("http://example.com/v1") == "http://example.com/v1" +def test_openai_compatible_raw_resolver_does_not_append_v1() -> None: + model, kwargs = to_litellm( + { + "provider": "openai_compatible_raw", + "base_url": "https://ark.cn-beijing.volces.com/api/v3", + "api_key": "ark-key", + "extra": {}, + }, + "ep-20260101000000-test", + ) + + assert model == "openai/ep-20260101000000-test" + assert kwargs["api_base"] == "https://ark.cn-beijing.volces.com/api/v3" + assert kwargs["api_key"] == "ark-key" + + def test_ollama_resolver_uses_native_api_base() -> None: model, kwargs = to_litellm( { diff --git a/surfsense_web/components/settings/model-connections/default-connect-form.tsx b/surfsense_web/components/settings/model-connections/default-connect-form.tsx index e3111202d..6ca96ced2 100644 --- a/surfsense_web/components/settings/model-connections/default-connect-form.tsx +++ b/surfsense_web/components/settings/model-connections/default-connect-form.tsx @@ -9,7 +9,10 @@ function baseUrlHint(provider: string) { return "For local servers, use host.docker.internal instead of localhost."; } if (provider === "openai_compatible") { - return "Enter the full endpoint URL."; + return "Enter the full endpoint URL. This provider expects a /v1-compatible endpoint."; + } + if (provider === "openai_compatible_raw") { + return "Enter the exact chat-completions API base URL. SurfSense will not append /v1."; } if (provider === "openai" || provider === "anthropic" || provider === "openrouter") { return "Override only if you route through a proxy or gateway."; diff --git a/surfsense_web/components/settings/model-connections/provider-metadata.tsx b/surfsense_web/components/settings/model-connections/provider-metadata.tsx index 8b8a877b9..0a5cc53c8 100644 --- a/surfsense_web/components/settings/model-connections/provider-metadata.tsx +++ b/surfsense_web/components/settings/model-connections/provider-metadata.tsx @@ -10,6 +10,7 @@ export const PROVIDER_ORDER = [ "ollama_chat", "lm_studio", "openai_compatible", + "openai_compatible_raw", ]; export const PROVIDER_DISPLAY: Record< @@ -37,6 +38,11 @@ export const PROVIDER_DISPLAY: Record< subtitle: "OpenAI-compatible endpoint", iconKey: "custom", }, + openai_compatible_raw: { + name: "OpenAI-Compatible Raw", + subtitle: "Use the exact base URL, no /v1 is appended", + iconKey: "custom", + }, openrouter: { name: "OpenRouter", subtitle: "OpenRouter", From 2ff7ea4cb6b1e11857e2404e8850d2ff9641fbdd Mon Sep 17 00:00:00 2001 From: Thibault Jaigu Date: Mon, 13 Jul 2026 09:42:30 +0100 Subject: [PATCH 04/28] feat: add Requesty as a model provider Add Requesty (https://requesty.ai), an OpenAI-compatible LLM router, as a model provider by mirroring the existing OpenRouter integration. Backend: - app/services/requesty_model_normalizer.py: normalizes Requesty's /v1/models catalogue, mapping its flat capability booleans (supports_tool_calling/ supports_vision/supports_image_generation) and context_window field onto the shared normalized shape (Requesty differs from OpenRouter's architecture + supported_parameters + context_length layout) - provider_registry.py: Requesty ProviderSpec (OpenAI-compatible, base URL https://router.requesty.ai/v1, REQUESTY_API_KEY bearer auth) - model_connection_service.py: key verification + live model discovery - quality_score.py: Requesty score entry - unit tests mirroring the OpenRouter normalizer coverage Frontend: - Requesty provider icon + registration, metadata entry, and base-url hint Signed-off-by: Thibault Jaigu --- .../app/services/model_connection_service.py | 15 ++- .../app/services/provider_registry.py | 10 ++ .../app/services/quality_score.py | 1 + .../app/services/requesty_model_normalizer.py | 123 ++++++++++++++++++ .../test_requesty_model_normalizer.py | 107 +++++++++++++++ .../components/icons/providers/index.ts | 1 + .../components/icons/providers/requesty.svg | 1 + .../default-connect-form.tsx | 7 +- .../model-connections/provider-metadata.tsx | 7 + surfsense_web/lib/provider-icons.tsx | 3 + 10 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/services/requesty_model_normalizer.py create mode 100644 surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py create mode 100644 surfsense_web/components/icons/providers/requesty.svg diff --git a/surfsense_backend/app/services/model_connection_service.py b/surfsense_backend/app/services/model_connection_service.py index cdfd1d725..54d0a3c3f 100644 --- a/surfsense_backend/app/services/model_connection_service.py +++ b/surfsense_backend/app/services/model_connection_service.py @@ -15,6 +15,7 @@ from app.db import Connection, Model, ModelSource from app.services.model_resolver import ensure_v1, to_litellm from app.services.openrouter_model_normalizer import normalize_openrouter_models from app.services.provider_registry import Transport, provider_label, spec_for +from app.services.requesty_model_normalizer import normalize_requesty_models logger = logging.getLogger(__name__) @@ -148,7 +149,7 @@ async def verify_connection(conn: Connection) -> VerifyResult: if spec.transport == Transport.OLLAMA and base_url: url = f"{base_url.rstrip('/')}/api/version" - elif spec.discovery in {"openai_models", "openrouter"} and base_url: + elif spec.discovery in {"openai_models", "openrouter", "requesty"} and base_url: url = f"{ensure_v1(base_url)}/models" elif spec.discovery == "anthropic_models" and base_url: url = f"{base_url.rstrip('/')}/models" @@ -363,6 +364,16 @@ async def _openrouter_models(conn: Connection) -> list[dict[str, Any]]: return normalize_openrouter_models(response.json().get("data", [])) +async def _requesty_models(conn: Connection) -> list[dict[str, Any]]: + base_url = _base_url_or_default(conn) or "https://router.requesty.ai/v1" + async with httpx.AsyncClient(timeout=DISCOVERY_TIMEOUT_SECONDS) as client: + response = await client.get( + f"{ensure_v1(base_url)}/models", headers=_auth_headers(conn) + ) + response.raise_for_status() + return normalize_requesty_models(response.json().get("data", [])) + + def _litellm_static_models(conn: Connection) -> list[dict[str, Any]]: provider = conn.provider prefix = spec_for(provider).litellm_prefix or provider @@ -446,6 +457,8 @@ async def discover_models(conn: Connection) -> list[dict[str, Any]]: results = await _ollama_tags_then_show(conn) elif spec.discovery == "openrouter": results = await _openrouter_models(conn) + elif spec.discovery == "requesty": + results = await _requesty_models(conn) elif spec.discovery == "anthropic_models": results = await _discover_anthropic_models(conn) elif spec.discovery == "openai_models": diff --git a/surfsense_backend/app/services/provider_registry.py b/surfsense_backend/app/services/provider_registry.py index 67d1c4db4..3461500e6 100644 --- a/surfsense_backend/app/services/provider_registry.py +++ b/surfsense_backend/app/services/provider_registry.py @@ -23,6 +23,7 @@ DiscoveryKind = Literal[ "anthropic_models", "bedrock_models", "openrouter", + "requesty", "static", "none", ] @@ -78,6 +79,15 @@ REGISTRY: dict[str, ProviderSpec] = { "bearer", "OpenRouter", ), + "requesty": ProviderSpec( + Transport.OPENAI_COMPATIBLE, + "openai", + "requesty", + "https://router.requesty.ai/v1", + False, + "bearer", + "Requesty", + ), "openai_compatible": ProviderSpec( Transport.OPENAI_COMPATIBLE, "openai", diff --git a/surfsense_backend/app/services/quality_score.py b/surfsense_backend/app/services/quality_score.py index 737dd7c2f..1aa3e7eda 100644 --- a/surfsense_backend/app/services/quality_score.py +++ b/surfsense_backend/app/services/quality_score.py @@ -123,6 +123,7 @@ PROVIDER_PRESTIGE_YAML: dict[str, int] = { "perplexity": 28, "bedrock": 28, "openrouter": 25, + "requesty": 25, "ollama_chat": 12, "custom": 12, } diff --git a/surfsense_backend/app/services/requesty_model_normalizer.py b/surfsense_backend/app/services/requesty_model_normalizer.py new file mode 100644 index 000000000..7710eb174 --- /dev/null +++ b/surfsense_backend/app/services/requesty_model_normalizer.py @@ -0,0 +1,123 @@ +"""Shared Requesty model normalization. + +Requesty (https://router.requesty.ai) is an OpenAI-compatible LLM router. +Its ``/v1/models`` catalogue carries richer, Requesty-specific capability +metadata than a generic OpenAI-compatible ``/models`` response, so keep all +Requesty filtering and capability extraction here -- mirroring +``openrouter_model_normalizer`` -- so GLOBAL catalogue generation and BYOK +discovery agree. + +Unlike OpenRouter, Requesty exposes capabilities as flat booleans +(``supports_tool_calling`` / ``supports_reasoning`` / ``supports_vision`` / +``supports_image_generation``) rather than an ``architecture`` block plus a +``supported_parameters`` array, and it reports context size as +``context_window`` rather than ``context_length``. This module maps those +fields onto the same normalized shape the rest of the backend consumes. +""" + +from __future__ import annotations + +from typing import Any + +from app.db import ModelSource + +MIN_CONTEXT_LENGTH = 100_000 + +EXCLUDED_PROVIDER_SLUGS: set[str] = {"amazon"} +EXCLUDED_MODEL_IDS: set[str] = set() +EXCLUDED_MODEL_SUFFIXES: tuple[str, ...] = ("-deep-research",) + + +def is_image_output_model(model: dict[str, Any]) -> bool: + return bool(model.get("supports_image_generation")) + + +def is_text_output_model(model: dict[str, Any]) -> bool: + # Requesty entries are chat-completion models (``api == "chat"``). Treat a + # model as text output whenever it is not an image-generation model. + return not is_image_output_model(model) + + +def supports_image_input(model: dict[str, Any]) -> bool: + return bool(model.get("supports_vision")) + + +def supports_tool_calling(model: dict[str, Any]) -> bool: + return bool(model.get("supports_tool_calling")) + + +def has_sufficient_context(model: dict[str, Any]) -> bool: + return int(model.get("context_window") or 0) >= MIN_CONTEXT_LENGTH + + +def is_compatible_provider(model: dict[str, Any]) -> bool: + model_id = str(model.get("id") or "") + slug = model_id.split("/", 1)[0] if "/" in model_id else "" + return slug not in EXCLUDED_PROVIDER_SLUGS + + +def is_allowed_model(model: dict[str, Any]) -> bool: + model_id = str(model.get("id") or "") + if model_id in EXCLUDED_MODEL_IDS: + return False + base_id = model_id.split(":")[0] + return not base_id.endswith(EXCLUDED_MODEL_SUFFIXES) + + +def is_requesty_chat_model(model: dict[str, Any]) -> bool: + return ( + "/" in str(model.get("id") or "") + and is_text_output_model(model) + and supports_tool_calling(model) + and has_sufficient_context(model) + and is_compatible_provider(model) + and is_allowed_model(model) + ) + + +def is_requesty_image_model(model: dict[str, Any]) -> bool: + return ( + "/" in str(model.get("id") or "") + and is_image_output_model(model) + and is_compatible_provider(model) + and is_allowed_model(model) + ) + + +def normalize_requesty_models( + raw_models: list[dict[str, Any]], +) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for model in raw_models: + if not is_requesty_chat_model(model): + continue + model_id = str(model.get("id") or "") + normalized.append( + { + "model_id": model_id, + "display_name": model.get("name") or model_id, + "source": ModelSource.DISCOVERED, + "supports_chat": True, + "max_input_tokens": model.get("context_window"), + "supports_image_input": supports_image_input(model), + "supports_tools": supports_tool_calling(model), + "supports_image_generation": False, + "metadata": model, + } + ) + return normalized + + +__all__ = [ + "MIN_CONTEXT_LENGTH", + "has_sufficient_context", + "is_allowed_model", + "is_compatible_provider", + "is_image_output_model", + "is_requesty_chat_model", + "is_requesty_image_model", + "is_text_output_model", + "normalize_requesty_models", + "supports_image_input", + "supports_tool_calling", +] diff --git a/surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py b/surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py new file mode 100644 index 000000000..36b867923 --- /dev/null +++ b/surfsense_backend/tests/unit/services/test_requesty_model_normalizer.py @@ -0,0 +1,107 @@ +"""Unit tests for Requesty model normalization. + +Mirrors the OpenRouter normalizer coverage but exercises Requesty's flat +boolean capability fields (``supports_tool_calling`` / ``supports_vision``) +and ``context_window`` sizing. +""" + +from __future__ import annotations + +import pytest + +from app.services.requesty_model_normalizer import ( + is_requesty_chat_model, + is_requesty_image_model, + normalize_requesty_models, + supports_image_input, + supports_tool_calling, +) + +pytestmark = pytest.mark.unit + + +def _requesty_model( + *, + model_id: str, + context_window: int = 128_000, + tools: bool = True, + vision: bool = False, + image_generation: bool = False, + name: str | None = None, +) -> dict: + """Return a synthetic Requesty ``/v1/models`` entry. + + Only the fields the normalizer inspects are populated; the live payload + carries many more (pricing, ``supports_caching``, ``description``, ...). + """ + return { + "id": model_id, + "name": name or model_id, + "api": "chat", + "object": "model", + "context_window": context_window, + "supports_tool_calling": tools, + "supports_vision": vision, + "supports_image_generation": image_generation, + } + + +def test_chat_model_requires_slash_tools_and_context(): + assert is_requesty_chat_model(_requesty_model(model_id="openai/gpt-4o-mini")) + assert not is_requesty_chat_model( + _requesty_model(model_id="openai/gpt-4o-mini", tools=False) + ) + assert not is_requesty_chat_model( + _requesty_model(model_id="openai/gpt-4o-mini", context_window=8_000) + ) + assert not is_requesty_chat_model(_requesty_model(model_id="bare-model")) + + +def test_excluded_provider_slug_is_filtered(): + assert not is_requesty_chat_model( + _requesty_model(model_id="amazon/nova-pro-v1") + ) + + +def test_image_generation_models_excluded_from_chat_and_flagged(): + image_model = _requesty_model( + model_id="google/gemini-2.5-flash-image", image_generation=True + ) + assert not is_requesty_chat_model(image_model) + assert is_requesty_image_model(image_model) + + +def test_capability_helpers_read_flat_booleans(): + model = _requesty_model( + model_id="anthropic/claude-sonnet-4-5", vision=True, tools=True + ) + assert supports_image_input(model) is True + assert supports_tool_calling(model) is True + + +def test_normalize_maps_context_window_and_capabilities(): + normalized = normalize_requesty_models( + [ + _requesty_model( + model_id="openai/gpt-4o-mini", + context_window=128_000, + vision=True, + name="GPT-4o mini", + ), + _requesty_model(model_id="openai/gpt-4o-mini", tools=False), + _requesty_model( + model_id="black-forest-labs/flux", image_generation=True + ), + ] + ) + + assert len(normalized) == 1 + entry = normalized[0] + assert entry["model_id"] == "openai/gpt-4o-mini" + assert entry["display_name"] == "GPT-4o mini" + assert entry["supports_chat"] is True + assert entry["max_input_tokens"] == 128_000 + assert entry["supports_image_input"] is True + assert entry["supports_tools"] is True + assert entry["supports_image_generation"] is False + assert entry["metadata"]["id"] == "openai/gpt-4o-mini" diff --git a/surfsense_web/components/icons/providers/index.ts b/surfsense_web/components/icons/providers/index.ts index 5c8276e62..f03fbec68 100644 --- a/surfsense_web/components/icons/providers/index.ts +++ b/surfsense_web/components/icons/providers/index.ts @@ -27,6 +27,7 @@ export { default as PerplexityIcon } from "./perplexity.svg"; export { default as QwenIcon } from "./qwen.svg"; export { default as RecraftIcon } from "./recraft.svg"; export { default as ReplicateIcon } from "./replicate.svg"; +export { default as RequestyIcon } from "./requesty.svg"; export { default as SambaNovaIcon } from "./sambanova.svg"; export { default as TogetherAiIcon } from "./togetherai.svg"; export { default as VertexAiIcon } from "./vertexai.svg"; diff --git a/surfsense_web/components/icons/providers/requesty.svg b/surfsense_web/components/icons/providers/requesty.svg new file mode 100644 index 000000000..a804601b6 --- /dev/null +++ b/surfsense_web/components/icons/providers/requesty.svg @@ -0,0 +1 @@ + diff --git a/surfsense_web/components/settings/model-connections/default-connect-form.tsx b/surfsense_web/components/settings/model-connections/default-connect-form.tsx index e3111202d..91f31f2f0 100644 --- a/surfsense_web/components/settings/model-connections/default-connect-form.tsx +++ b/surfsense_web/components/settings/model-connections/default-connect-form.tsx @@ -11,7 +11,12 @@ function baseUrlHint(provider: string) { if (provider === "openai_compatible") { return "Enter the full endpoint URL."; } - if (provider === "openai" || provider === "anthropic" || provider === "openrouter") { + if ( + provider === "openai" || + provider === "anthropic" || + provider === "openrouter" || + provider === "requesty" + ) { return "Override only if you route through a proxy or gateway."; } return undefined; diff --git a/surfsense_web/components/settings/model-connections/provider-metadata.tsx b/surfsense_web/components/settings/model-connections/provider-metadata.tsx index 8b8a877b9..e6c0e8cd8 100644 --- a/surfsense_web/components/settings/model-connections/provider-metadata.tsx +++ b/surfsense_web/components/settings/model-connections/provider-metadata.tsx @@ -7,6 +7,7 @@ export const PROVIDER_ORDER = [ "bedrock", "azure", "openrouter", + "requesty", "ollama_chat", "lm_studio", "openai_compatible", @@ -43,6 +44,12 @@ export const PROVIDER_DISPLAY: Record< iconKey: "openrouter", defaultBaseUrl: "https://openrouter.ai/api/v1", }, + requesty: { + name: "Requesty", + subtitle: "Requesty", + iconKey: "requesty", + defaultBaseUrl: "https://router.requesty.ai/v1", + }, vertex_ai: { name: "Gemini", subtitle: "Google Cloud Vertex AI", iconKey: "vertex_ai" }, }; diff --git a/surfsense_web/lib/provider-icons.tsx b/surfsense_web/lib/provider-icons.tsx index d3e799720..20016fdf2 100644 --- a/surfsense_web/lib/provider-icons.tsx +++ b/surfsense_web/lib/provider-icons.tsx @@ -29,6 +29,7 @@ import { QwenIcon, RecraftIcon, ReplicateIcon, + RequestyIcon, SambaNovaIcon, TogetherAiIcon, VertexAiIcon, @@ -117,6 +118,8 @@ export function getProviderIcon( return ; case "REPLICATE": return ; + case "REQUESTY": + return ; case "SAMBANOVA": return ; case "TOGETHER_AI": From 0d663358842295dd2b15607a70f323236343d85c Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 14:15:38 -0700 Subject: [PATCH 05/28] Update metadata and hero section text to reflect rebranding of SurfSense as NotebookLM for competitive intelligence. Adjusted titles and descriptions for improved clarity and alignment with new positioning. --- surfsense_web/app/layout.tsx | 6 +++--- surfsense_web/components/homepage/hero-section.tsx | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index d9e915d0f..fff38e83b 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -50,7 +50,7 @@ export const metadata: Metadata = { alternates: { canonical: "https://www.surfsense.com", }, - title: "SurfSense - Competitive Intelligence Platform for AI Agents", + title: "SurfSense - NotebookLM for Competitive Intelligence", description: "SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.", keywords: [ @@ -68,7 +68,7 @@ export const metadata: Metadata = { "SurfSense", ], openGraph: { - title: "SurfSense - Competitive Intelligence Platform for AI Agents", + title: "SurfSense - NotebookLM for Competitive Intelligence", description: "SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.", url: "https://www.surfsense.com", @@ -86,7 +86,7 @@ export const metadata: Metadata = { }, twitter: { card: "summary_large_image", - title: "SurfSense - Competitive Intelligence Platform for AI Agents", + title: "SurfSense - NotebookLM for Competitive Intelligence", description: "SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.", creator: "@SurfSenseAI", diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index fc8356041..05db2781f 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -708,7 +708,7 @@ export function HeroSection() { "relative mt-4 max-w-4xl text-left text-4xl font-bold tracking-tight text-balance text-neutral-900 sm:text-5xl md:text-6xl dark:text-neutral-50" )} > - Give your AI agents competitive intelligence. + NotebookLM for competitive intelligence research.
@@ -717,9 +717,10 @@ export function HeroSection() { "relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400" )} > - SurfSense is an open-source competitive intelligence platform. Your AI agents monitor - competitors, track rankings, and listen to your market with live data from platforms - like Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web. + SurfSense is an open-source competitive intelligence platform, like NotebookLM but with + live scraping connectors. Your AI agents monitor competitors, track rankings, and listen + to your market with live data from platforms like Reddit, YouTube, Instagram, TikTok, + Google Maps, Google Search, and the open web.

From b5cefd25ef376f2140a1b42d5ca1c276310db68e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 06/28] feat(chat): add module-level chat stream store for resumable streaming --- surfsense_web/lib/chat/stream-engine/store.ts | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/store.ts diff --git a/surfsense_web/lib/chat/stream-engine/store.ts b/surfsense_web/lib/chat/stream-engine/store.ts new file mode 100644 index 000000000..44627663f --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/store.ts @@ -0,0 +1,177 @@ +import type { ThreadMessageLike } from "@assistant-ui/react"; +import { createTokenUsageStore } from "@/components/assistant-ui/token-usage-context"; +import type { PendingInterruptState } from "@/features/chat-messages/hitl"; + +/** + * Durable, per-thread streaming state for a single in-flight chat turn. + * + * Lives at module scope (see {@link chatStreamStore}) so a running turn's + * messages / interrupts survive the chat page unmounting during in-app + * navigation. The React tree consumes it through ``useChatStream`` via + * ``useSyncExternalStore`` (state lives outside the render cycle). + */ +export interface ThreadStreamState { + threadId: number; + messages: ThreadMessageLike[]; + isRunning: boolean; + pendingInterrupts: PendingInterruptState[]; +} + +type Listener = () => void; + +/** + * Module-level singleton external store for chat streaming. + * + * Single-active-stream invariant: at most one turn streams at a time, + * tracked by {@link active}. Per-thread state is still keyed by threadId so + * the currently-streaming thread and the currently-viewed thread can differ + * (e.g. a stream finishes while the user is on another chat). + * + * ponytail: unbounded ``states`` map is bounded in practice by + * ``clearInactive`` (called on navigation) + ``clear`` (called after the DB + * re-hydrates a finished turn). Upgrade path if that ever leaks: LRU cap. + */ +class ChatStreamStore { + private states = new Map(); + private listeners = new Set(); + + /** Shared, cross-navigation token-usage store (one instance app-wide). */ + readonly tokenUsage = createTokenUsageStore(); + + /** The one in-flight turn's abort handle, or null when idle. */ + private active: { threadId: number; controller: AbortController } | null = null; + + /** Timestamp of the last explicit cancel, for the THREAD_BUSY retry window. */ + recentCancelRequestedAt = 0; + + subscribe = (listener: Listener): (() => void) => { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }; + + private notify(): void { + for (const l of this.listeners) l(); + } + + /** Snapshot for ``useSyncExternalStore``; stable ref between mutations. */ + getSnapshot = (threadId: number | null): ThreadStreamState | null => { + if (threadId == null) return null; + return this.states.get(threadId) ?? null; + }; + + isRunning(threadId: number | null): boolean { + if (threadId == null) return false; + return this.states.get(threadId)?.isRunning ?? false; + } + + getMessages(threadId: number): ThreadMessageLike[] { + return this.states.get(threadId)?.messages ?? []; + } + + getPendingInterrupts(threadId: number): PendingInterruptState[] { + return this.states.get(threadId)?.pendingInterrupts ?? []; + } + + private ensure(threadId: number): ThreadStreamState { + let s = this.states.get(threadId); + if (!s) { + s = { threadId, messages: [], isRunning: false, pendingInterrupts: [] }; + this.states.set(threadId, s); + } + return s; + } + + private commit(threadId: number, next: ThreadStreamState): void { + this.states.set(threadId, next); + this.notify(); + } + + /** Seed a thread's state at the start of a fresh turn (running=true). */ + begin(threadId: number, messages: ThreadMessageLike[]): void { + this.commit(threadId, { threadId, messages, isRunning: true, pendingInterrupts: [] }); + } + + setMessages(threadId: number, updater: (prev: ThreadMessageLike[]) => ThreadMessageLike[]): void { + const prev = this.ensure(threadId); + const messages = updater(prev.messages); + if (messages === prev.messages) return; + this.commit(threadId, { ...prev, messages }); + } + + setRunning(threadId: number, running: boolean): void { + const prev = this.ensure(threadId); + if (prev.isRunning === running) return; + this.commit(threadId, { ...prev, isRunning: running }); + } + + setPendingInterrupts( + threadId: number, + updater: (prev: PendingInterruptState[]) => PendingInterruptState[] + ): void { + const prev = this.ensure(threadId); + const pendingInterrupts = updater(prev.pendingInterrupts); + if (pendingInterrupts === prev.pendingInterrupts) return; + this.commit(threadId, { ...prev, pendingInterrupts }); + } + + /** + * A thread whose overlay must survive DB re-hydration / navigation: it is + * either streaming or paused awaiting a HITL decision (the pending + * interrupts + interrupt cards live only in the overlay). + */ + private isPinned(s: ThreadStreamState): boolean { + return s.isRunning || s.pendingInterrupts.length > 0; + } + + /** Drop a thread's overlay once the DB is authoritative. No-op while pinned. */ + clear(threadId: number): void { + const s = this.states.get(threadId); + if (!s || this.isPinned(s)) return; + this.states.delete(threadId); + this.notify(); + } + + /** Evict every non-pinned thread except ``exceptThreadId`` (memory bound). */ + clearInactive(exceptThreadId: number | null): void { + let changed = false; + for (const [id, s] of this.states) { + if (id === exceptThreadId || this.isPinned(s)) continue; + this.states.delete(id); + changed = true; + } + if (changed) this.notify(); + } + + // ---- active-stream lifecycle ------------------------------------------- + + /** Register a new in-flight turn, aborting any previous one first. */ + beginActive(threadId: number, controller: AbortController): void { + this.abortActive(); + this.active = { threadId, controller }; + } + + get activeThreadId(): number | null { + return this.active?.threadId ?? null; + } + + /** Clear the active handle iff it still points at ``controller``. */ + clearActive(controller: AbortController): void { + if (this.active?.controller === controller) this.active = null; + } + + /** Abort the in-flight turn's fetch (client disconnect, not a server stop). */ + abortActive(): void { + if (this.active) { + this.active.controller.abort(); + this.active = null; + } + } + + markRecentCancel(): void { + this.recentCancelRequestedAt = Date.now(); + } +} + +export const chatStreamStore = new ChatStreamStore(); From 0fb577b71ff7075a0cafe770ee7e469f31f2d347 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 07/28] feat(chat): add useChatStream hook over the stream store --- .../lib/chat/stream-engine/use-chat-stream.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/use-chat-stream.ts diff --git a/surfsense_web/lib/chat/stream-engine/use-chat-stream.ts b/surfsense_web/lib/chat/stream-engine/use-chat-stream.ts new file mode 100644 index 000000000..ccfb32481 --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/use-chat-stream.ts @@ -0,0 +1,18 @@ +"use client"; + +import { useCallback, useSyncExternalStore } from "react"; +import { chatStreamStore, type ThreadStreamState } from "./store"; + +/** + * Subscribe to the durable streaming state for a given thread. + * + * Returns the live ``ThreadStreamState`` while a turn is streaming (or + * pending re-hydration from the DB), or ``null`` when the thread has no + * active overlay — in which case the page falls back to its DB-hydrated + * messages. State lives in the module-level {@link chatStreamStore}, so it + * survives the chat page unmounting during in-app navigation. + */ +export function useChatStream(threadId: number | null): ThreadStreamState | null { + const getSnapshot = useCallback(() => chatStreamStore.getSnapshot(threadId), [threadId]); + return useSyncExternalStore(chatStreamStore.subscribe, getSnapshot, () => null); +} From 9984a88d6d65c5fd319008ac3f97c74d3b377415 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 08/28] feat(chat): extract shared stream-engine helpers --- .../lib/chat/stream-engine/helpers.ts | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/helpers.ts diff --git a/surfsense_web/lib/chat/stream-engine/helpers.ts b/surfsense_web/lib/chat/stream-engine/helpers.ts new file mode 100644 index 000000000..afa1851e1 --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/helpers.ts @@ -0,0 +1,154 @@ +import { z } from "zod"; +import type { MentionedDocumentInfo } from "@/atoms/chat/mentioned-documents.atom"; +import type { ToolUIGate } from "@/lib/chat/streaming-state"; + +/** + * Every tool call renders a card. The sentinel ``"all"`` matches every tool + * — the legacy ``BASE_TOOLS_WITH_UI`` allowlist was dropped so unknown tool + * calls route through the generic ``ToolFallback``. Persisted payload size + * stays bounded because the backend's ``format_thinking_step`` summarisation + * and the ``result_length``-only default for unknown tools keep the JSON + * from ballooning. + */ +export const TOOLS_WITH_UI_ALL: ToolUIGate = "all"; + +export const TURN_CANCELLING_INITIAL_DELAY_MS = 200; +export const TURN_CANCELLING_BACKOFF_FACTOR = 2; +export const TURN_CANCELLING_MAX_DELAY_MS = 1500; +export const RECENT_CANCEL_WINDOW_MS = 5_000; + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function computeFallbackTurnCancellingRetryDelay(attempt: number): number { + const safeAttempt = Math.max(1, attempt); + const raw = + TURN_CANCELLING_INITIAL_DELAY_MS * TURN_CANCELLING_BACKOFF_FACTOR ** (safeAttempt - 1); + return Math.min(raw, TURN_CANCELLING_MAX_DELAY_MS); +} + +/** + * Generate a synthetic ``toolCallId`` for an action_request that has no + * matching streamed tool-call card (HITL-blocked subagent calls don't surface + * as tool-call events). Suffixes a counter when the base id is already taken + * — sequential interrupts for the same tool name otherwise collide on + * ``interrupt-${name}-${i}`` and crash assistant-ui with a duplicate-key error. + */ +export function freshSynthToolCallId( + toolCallIndices: Map, + toolName: string, + index: number +): string { + const base = `interrupt-${toolName}-${index}`; + if (!toolCallIndices.has(base)) return base; + let n = 1; + while (toolCallIndices.has(`${base}-${n}`)) n++; + return `${base}-${n}`; +} + +/** + * Pair each ``action_request`` to a unique pending tool-call card, preserving + * order so ``decisions[i]`` lines up with ``action_requests[i]`` on the wire. + * + * Same-name bundles (e.g. three ``create_jira_issue``) used to collapse onto + * one card because the matcher keyed by name; this consumes each card via the + * ``claimed`` set and walks forward in DOM order. + */ +export function pairBundleToolCallIds( + toolCallIndices: Map, + contentParts: Array<{ + type: string; + toolName?: string; + result?: unknown; + }>, + actionRequests: ReadonlyArray<{ name: string }> +): Array { + const claimed = new Set(); + const paired: Array = []; + for (const action of actionRequests) { + let matched: string | null = null; + for (const [tcId, idx] of toolCallIndices) { + if (claimed.has(tcId)) continue; + const part = contentParts[idx]; + if (!part || part.type !== "tool-call" || part.toolName !== action.name) continue; + const result = part.result as Record | undefined | null; + if (result == null || (result.__interrupt__ === true && !result.__decided__)) { + matched = tcId; + claimed.add(tcId); + break; + } + } + paired.push(matched); + } + return paired; +} + +/** + * Zod schema for mentioned document info (for type-safe parsing). + * + * ``kind`` defaults to ``"doc"`` so messages persisted before folder + * mentions existed deserialise unchanged. + */ +const MentionedDocumentInfoSchema = z.object({ + id: z.number(), + title: z.string(), + document_type: z.string().optional(), + kind: z + .union([z.literal("doc"), z.literal("folder"), z.literal("connector"), z.literal("thread")]) + .optional() + .default("doc"), + connector_type: z.string().optional(), + account_name: z.string().optional(), +}); + +const MentionedDocumentsPartSchema = z.object({ + type: z.literal("mentioned-documents"), + documents: z.array(MentionedDocumentInfoSchema), +}); + +/** + * Extract mentioned documents from message content (type-safe with Zod). + */ +export function extractMentionedDocuments(content: unknown): MentionedDocumentInfo[] { + if (!Array.isArray(content)) return []; + + for (const part of content) { + const result = MentionedDocumentsPartSchema.safeParse(part); + if (result.success) { + return result.data.documents.map((doc) => { + if (doc.kind === "connector") { + return { + id: doc.id, + title: doc.title, + kind: "connector", + connector_type: doc.connector_type ?? doc.document_type ?? "UNKNOWN", + account_name: doc.account_name ?? doc.title, + }; + } + if (doc.kind === "folder") { + return { + id: doc.id, + title: doc.title, + kind: "folder", + }; + } + if (doc.kind === "thread") { + return { + id: doc.id, + title: doc.title, + kind: "thread", + }; + } + return { + id: doc.id, + title: doc.title, + document_type: doc.document_type ?? "UNKNOWN", + kind: "doc", + }; + }); + } + } + + return []; +} From 204b3aa56cdcab49c6eaab3a27e3a019b8a8a90a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 09/28] feat(chat): add stream engine driving turns into the store --- .../lib/chat/stream-engine/engine.ts | 1433 +++++++++++++++++ 1 file changed, 1433 insertions(+) create mode 100644 surfsense_web/lib/chat/stream-engine/engine.ts diff --git a/surfsense_web/lib/chat/stream-engine/engine.ts b/surfsense_web/lib/chat/stream-engine/engine.ts new file mode 100644 index 000000000..a4a0c02a7 --- /dev/null +++ b/surfsense_web/lib/chat/stream-engine/engine.ts @@ -0,0 +1,1433 @@ +import type { AppendMessage, ThreadMessageLike } from "@assistant-ui/react"; +import { getDefaultStore } from "jotai"; +import type { Dispatch, SetStateAction } from "react"; +import { toast } from "sonner"; +import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; +import { disabledToolsAtom } from "@/atoms/agent-tools/agent-tools.atoms"; +import { + deriveMentionedPayload, + type MentionedDocumentInfo, + mentionedDocumentsAtom, + messageDocumentsMapAtom, + submittedMentionsAtom, +} from "@/atoms/chat/mentioned-documents.atom"; +import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; +import { setPremiumAlertForThreadAtom } from "@/atoms/chat/premium-alert.atom"; +import { type AgentCreatedDocument, agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; +import { updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import type { HitlDecision, PendingInterruptState } from "@/features/chat-messages/hitl"; +import { + applyActionLogSse, + applyActionLogUpdatedSse, + markActionRevertedInCache, +} from "@/hooks/use-agent-actions-query"; +import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; +import { authenticatedFetch } from "@/lib/auth-fetch"; +import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; +import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; +import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; +import { createStreamFlushHelpers } from "@/lib/chat/stream-flush"; +import { consumeSseEvents, processSharedStreamEvent } from "@/lib/chat/stream-pipeline"; +import { + applyTurnIdToAssistantMessageList, + mergeChatTurnIdIntoMessage, + readStreamedChatTurnId, + readStreamedMessageId, +} from "@/lib/chat/stream-side-effects"; +import { + addToolCall, + buildContentForUI, + type ContentPartsState, + type FrameBatchedUpdater, + type ThinkingStepData, + updateToolCall, +} from "@/lib/chat/streaming-state"; +import { + appendMessage, + createThread, + getRegenerateUrl, + type ThreadListItem, + type ThreadListResponse, + type ThreadRecord, +} from "@/lib/chat/thread-persistence"; +import { + extractUserTurnForNewChatApi, + type NewChatUserImagePayload, +} from "@/lib/chat/user-turn-api-parts"; +import { buildBackendUrl } from "@/lib/env-config"; +import { + trackChatBlocked, + trackChatCreated, + trackChatErrorDetailed, + trackChatMessageSent, + trackChatResponseReceived, +} from "@/lib/posthog/events"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { + computeFallbackTurnCancellingRetryDelay, + freshSynthToolCallId, + pairBundleToolCallIds, + RECENT_CANCEL_WINDOW_MS, + sleep, + TOOLS_WITH_UI_ALL, +} from "./helpers"; +import { chatStreamStore } from "./store"; + +const jotaiStore = getDefaultStore(); +const tokenUsageStore = chatStreamStore.tokenUsage; +const toolsWithUI = TOOLS_WITH_UI_ALL; + +/** + * Display-only setters the page provides while it is mounted. After the chat + * page unmounts (in-app navigation) these are stale but harmless — the stream + * keeps writing to the module {@link chatStreamStore}, and the page re-derives + * its thread from the URL when it remounts. + */ +export interface EngineView { + setThreadId: Dispatch>; + setCurrentThread: Dispatch>; +} + +/** Route/view context the page passes into every engine call. */ +export interface EngineContext { + workspaceId: number; + /** Currently viewed thread id (``activeThreadId`` in the page). */ + threadId: number | null; + /** The page's current displayed messages — history/slice seed. */ + priorMessages: ThreadMessageLike[]; + view: EngineView; +} + +// --------------------------------------------------------------------------- +// Error handling (relocated from the chat page) +// --------------------------------------------------------------------------- + +async function persistAssistantErrorMessage({ + threadId, + assistantMsgId, + text, +}: { + threadId: number | null; + assistantMsgId: string; + text: string; +}): Promise { + if (threadId != null) { + chatStreamStore.setMessages(threadId, (prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, content: [{ type: "text", text }] } : m)) + ); + } + + if (!threadId) return; + + // Persist only temporary assistant placeholders to avoid duplicate rows + // when the message already has a database-backed ID. + if (!assistantMsgId.startsWith("msg-assistant-")) return; + + try { + const savedMessage = await appendMessage(threadId, { + role: "assistant", + content: [{ type: "text", text }], + }); + const newMsgId = `msg-${savedMessage.id}`; + tokenUsageStore.rename(assistantMsgId, newMsgId); + chatStreamStore.setMessages(threadId, (prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)) + ); + } catch (persistErr) { + console.error("Failed to persist assistant error message:", persistErr); + } +} + +async function handleChatFailure({ + error, + flow, + threadId, + assistantMsgId, + workspaceId, +}: { + error: unknown; + flow: ChatFlow; + threadId: number | null; + assistantMsgId: string; + workspaceId: number; +}): Promise { + const normalized = classifyChatError({ + error, + flow, + context: { workspaceId, threadId }, + }); + + const logger = + normalized.severity === "error" + ? console.error + : normalized.severity === "warn" + ? console.warn + : console.info; + logger(`[chat-engine] ${flow} ${normalized.kind}:`, error); + + const telemetryPayload = { + flow, + kind: normalized.kind, + error_code: normalized.errorCode, + severity: normalized.severity, + is_expected: normalized.isExpected, + message: normalized.userMessage, + }; + if (normalized.telemetryEvent === "chat_blocked") { + trackChatBlocked(workspaceId, threadId, telemetryPayload); + } else { + trackChatErrorDetailed(workspaceId, threadId, telemetryPayload); + } + + if (normalized.channel === "silent") { + return; + } + + if (normalized.channel === "pinned_inline") { + if (threadId) { + const currentUser = jotaiStore.get(currentUserAtom).data; + jotaiStore.set(setPremiumAlertForThreadAtom, { + threadId, + message: normalized.userMessage, + userId: currentUser?.id ?? null, + }); + } + if (normalized.assistantMessage) { + await persistAssistantErrorMessage({ + threadId, + assistantMsgId, + text: normalized.assistantMessage, + }); + } + return; + } + + if (normalized.channel === "inline") { + if (normalized.assistantMessage) { + await persistAssistantErrorMessage({ + threadId, + assistantMsgId, + text: normalized.assistantMessage, + }); + } + toast.error(normalized.userMessage); + return; + } + + toast.error(normalized.userMessage); +} + +async function handleStreamTerminalError({ + error, + flow, + threadId, + assistantMsgId, + accepted, + workspaceId, + onAbort, + onPreAcceptFailure, + onAcceptedStreamError, +}: { + error: unknown; + flow: ChatFlow; + threadId: number | null; + assistantMsgId: string; + accepted: boolean; + workspaceId: number; + onAbort?: () => Promise; + onPreAcceptFailure?: () => Promise; + onAcceptedStreamError?: () => Promise; +}): Promise { + if (error instanceof Error && error.name === "AbortError") { + await onAbort?.(); + return; + } + + if (!accepted) { + await onPreAcceptFailure?.(); + } else { + await onAcceptedStreamError?.(); + } + + await handleChatFailure({ + error: !accepted ? tagPreAcceptSendFailure(error) : error, + flow, + threadId, + assistantMsgId: accepted ? assistantMsgId : "no-persist-assistant", + workspaceId, + }); +} + +async function fetchWithTurnCancellingRetry(runFetch: () => Promise): Promise { + const maxAttempts = 4; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const response = await runFetch(); + if (response.ok) { + return response; + } + const error = await toHttpResponseError(response); + const withMeta = error as Error & { errorCode?: string; retryAfterMs?: number }; + const isTurnCancelling = withMeta.errorCode === "TURN_CANCELLING"; + const isRecentThreadBusyAfterCancel = + withMeta.errorCode === "THREAD_BUSY" && + Date.now() - chatStreamStore.recentCancelRequestedAt <= RECENT_CANCEL_WINDOW_MS; + if ((isTurnCancelling || isRecentThreadBusyAfterCancel) && attempt < maxAttempts) { + const waitMs = withMeta.retryAfterMs ?? computeFallbackTurnCancellingRetryDelay(attempt); + await sleep(waitMs); + continue; + } + throw error; + } + + throw Object.assign(new Error("Turn cancellation retry limit exceeded"), { + errorCode: "TURN_CANCELLING", + }); +} + +// --------------------------------------------------------------------------- +// Cancel +// --------------------------------------------------------------------------- + +/** + * Cancel the single in-flight turn. Targets the active stream's OWNER thread + * (not the currently-viewed thread) so cancel works even after navigation and + * for the lazy-create case. + */ +export async function cancelActiveTurn(): Promise { + const threadId = chatStreamStore.activeThreadId; + if (threadId) { + try { + const response = await authenticatedFetch( + buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), + { method: "POST" } + ); + if (response.ok) { + const payload = (await response.json()) as { error_code?: string }; + if (payload.error_code === "TURN_CANCELLING") { + chatStreamStore.markRecentCancel(); + } + } + } catch (error) { + console.warn("[chat-engine] Failed to signal cancel-active-turn:", error); + } + chatStreamStore.setRunning(threadId, false); + } + chatStreamStore.abortActive(); +} + +// --------------------------------------------------------------------------- +// New chat turn +// --------------------------------------------------------------------------- + +export async function startNewChat(ctx: EngineContext, message: AppendMessage): Promise { + const { workspaceId, threadId, priorMessages, view } = ctx; + + // Supersede any previous in-flight turn. + chatStreamStore.abortActive(); + + // Prefer the submit-time snapshot; fall back to the live atom for the + // send-button path. + const submittedSnapshot = jotaiStore.get(submittedMentionsAtom); + jotaiStore.set(submittedMentionsAtom, null); + const mentionedDocuments = jotaiStore.get(mentionedDocumentsAtom); + const activeMentions = submittedSnapshot ?? mentionedDocuments; + const mentionPayload = deriveMentionedPayload(activeMentions); + if (activeMentions.length > 0) { + jotaiStore.set(mentionedDocumentsAtom, []); + } + + const pendingUserImageUrls = jotaiStore.get(pendingUserImageDataUrlsAtom); + const urlsSnapshot = [...pendingUserImageUrls]; + const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot); + + if (!userQuery.trim() && userImages.length === 0) return; + + const localFilesystemEnabled = + jotaiStore.get(agentFlagsAtom).data?.enable_desktop_local_filesystem === true; + const disabledTools = jotaiStore.get(disabledToolsAtom); + const currentUser = jotaiStore.get(currentUserAtom).data; + + // Resolve filesystem selection BEFORE any optimistic UI / lazy thread + // creation so an unsatisfied "Local Folder" requirement bails cleanly + // with nothing to roll back and no thread left stuck "running". + let selection: Awaited>; + try { + selection = await getAgentFilesystemSelection(workspaceId, { localFilesystemEnabled }); + } catch (error) { + await handleChatFailure({ + error: tagPreAcceptSendFailure(error), + flow: "new", + threadId, + assistantMsgId: "no-persist-assistant", + workspaceId, + }); + return; + } + if ( + selection.filesystem_mode === "desktop_local_folder" && + (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0) + ) { + toast.error("Select a local folder before using Local Folder mode."); + return; + } + + // Lazy thread creation: create thread on first message if it doesn't exist. + let currentThreadId = threadId; + let isNewThread = false; + if (!currentThreadId) { + try { + const newThread = await createThread(workspaceId, "New Chat"); + currentThreadId = newThread.id; + view.setThreadId(currentThreadId); + view.setCurrentThread(newThread); + queryClient.setQueryData(cacheKeys.threads.detail(newThread.id), newThread); + queryClient.setQueryData(cacheKeys.threads.messages(newThread.id), { messages: [] }); + + trackChatCreated(workspaceId, currentThreadId); + + isNewThread = true; + // Update URL silently using browser API (not router.replace) to avoid + // interrupting the ongoing fetch/streaming with React navigation. + window.history.replaceState( + null, + "", + `/dashboard/${workspaceId}/new-chat/${currentThreadId}` + ); + } catch (error) { + console.error("[chat-engine] Failed to create thread:", error); + await handleChatFailure({ + error: tagPreAcceptSendFailure(error), + flow: "new", + threadId: currentThreadId, + assistantMsgId: "no-persist-assistant", + workspaceId, + }); + return; + } + } + + // Seed the durable per-thread overlay with the pre-turn conversation and + // flip it to running so the page renders the live stream. + const streamThreadId = currentThreadId; + chatStreamStore.begin(streamThreadId, priorMessages); + + if (urlsSnapshot.length > 0) { + jotaiStore.set(pendingUserImageDataUrlsAtom, (prev) => + prev.filter((u) => !urlsSnapshot.includes(u)) + ); + } + + // Add user message to state. Mutable because the SSE + // ``data-user-message-id`` handler renames this optimistic id to the + // canonical ``msg-{db_id}`` once ``persist_user_turn`` resolves. + let userMsgId = `msg-user-${Date.now()}`; + + const authorMetadata = currentUser + ? { + custom: { + author: { + displayName: currentUser.display_name ?? null, + avatarUrl: currentUser.avatar_url ?? null, + }, + }, + } + : undefined; + + const existingImageUrls = new Set( + message.content + .filter( + (p): p is { type: "image"; image: string } => + typeof p === "object" && p !== null && "type" in p && p.type === "image" && "image" in p + ) + .map((p) => p.image) + ); + const extraImageParts = urlsSnapshot + .filter((u) => !existingImageUrls.has(u)) + .map((image) => ({ type: "image" as const, image })); + const userDisplayContent = [...message.content, ...extraImageParts]; + + const userMessage: ThreadMessageLike = { + id: userMsgId, + role: "user", + content: userDisplayContent, + createdAt: new Date(), + metadata: authorMetadata, + }; + chatStreamStore.setMessages(streamThreadId, (prev) => [...prev, userMessage]); + + trackChatMessageSent(workspaceId, streamThreadId, { + hasAttachments: userImages.length > 0, + hasMentionedDocuments: + mentionPayload.document_ids.length > 0 || + mentionPayload.folder_ids.length > 0 || + mentionPayload.connector_ids.length > 0, + messageLength: userQuery.length, + }); + + // Collect unique mention chips for display & persistence. + const allMentionedDocs: MentionedDocumentInfo[] = []; + const seenDocKeys = new Set(); + for (const doc of activeMentions) { + const key = getMentionDocKey(doc); + if (seenDocKeys.has(key)) continue; + seenDocKeys.add(key); + allMentionedDocs.push(doc); + } + + if (allMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => ({ + ...prev, + [userMsgId]: allMentionedDocs, + })); + } + + const controller = new AbortController(); + chatStreamStore.beginActive(streamThreadId, controller); + + // Prepare assistant message. Mutable for the same reason as ``userMsgId``. + let assistantMsgId = `msg-assistant-${Date.now()}`; + const currentThinkingSteps = new Map(); + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + currentReasoningPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts } = contentPartsState; + let wasInterrupted = false; + let newAccepted = false; + let streamBatcher: FrameBatchedUpdater | null = null; + + try { + // Build message history for context. + const messageHistory = priorMessages + .filter((m) => m.role === "user" || m.role === "assistant") + .map((m) => { + let text = ""; + for (const part of m.content) { + if (typeof part === "object" && part.type === "text" && "text" in part) { + text += part.text; + } + } + return { role: m.role, content: text }; + }) + .filter((m) => m.content.length > 0); + + const hasDocumentIds = mentionPayload.document_ids.length > 0; + const hasFolderIds = mentionPayload.folder_ids.length > 0; + const hasConnectorIds = mentionPayload.connector_ids.length > 0; + const hasThreadIds = mentionPayload.thread_ids.length > 0; + + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + chat_id: streamThreadId, + user_query: userQuery.trim(), + workspace_id: workspaceId, + filesystem_mode: selection.filesystem_mode, + client_platform: selection.client_platform, + local_filesystem_mounts: selection.local_filesystem_mounts, + messages: messageHistory, + mentioned_document_ids: hasDocumentIds ? mentionPayload.document_ids : undefined, + mentioned_folder_ids: hasFolderIds ? mentionPayload.folder_ids : undefined, + mentioned_connector_ids: hasConnectorIds ? mentionPayload.connector_ids : undefined, + mentioned_connectors: hasConnectorIds ? mentionPayload.connectors : undefined, + mentioned_thread_ids: hasThreadIds ? mentionPayload.thread_ids : undefined, + mentioned_documents: allMentionedDocs.length > 0 ? allMentionedDocs : undefined, + disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + ...(userImages.length > 0 ? { user_images: userImages } : {}), + }), + signal: controller.signal, + }) + ); + + if (!response.ok) { + throw await toHttpResponseError(response); + } + newAccepted = true; + chatStreamStore.setMessages(streamThreadId, (prev) => [ + ...prev, + { + id: assistantMsgId, + role: "assistant", + content: [{ type: "text", text: "" }], + createdAt: new Date(), + }, + ]); + + const flushMessages = () => { + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + }; + const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); + streamBatcher = batcher; + + await consumeSseEvents(response, async (parsed) => { + if ( + processSharedStreamEvent(parsed, { + contentPartsState, + toolsWithUI, + currentThinkingSteps, + scheduleFlush, + forceFlush, + onTokenUsage: (data) => { + tokenUsageStore.set(assistantMsgId, data); + }, + onTurnStatus: (data) => { + if (data.status === "cancelling") { + chatStreamStore.markRecentCancel(); + } + }, + }) + ) { + return; + } + switch (parsed.type) { + case "data-thread-title-update": { + const titleData = parsed.data as { threadId: number; title: string }; + if (titleData?.title && titleData?.threadId === streamThreadId) { + view.setCurrentThread((prev) => (prev ? { ...prev, title: titleData.title } : prev)); + jotaiStore.set(updateChatTabTitleAtom, { + chatId: streamThreadId, + title: titleData.title, + }); + queryClient.setQueriesData( + { queryKey: ["threads", String(workspaceId)] }, + (old) => { + if (!old) return old; + const updateTitle = (list: ThreadListItem[]) => + list.map((t) => + t.id === titleData.threadId ? { ...t, title: titleData.title } : t + ); + return { + ...old, + threads: updateTitle(old.threads), + archived_threads: updateTitle(old.archived_threads), + }; + } + ); + } + break; + } + + case "data-documents-updated": { + const docEvent = parsed.data as { + action: string; + document: AgentCreatedDocument; + }; + if (docEvent?.document?.id) { + jotaiStore.set(agentCreatedDocumentsAtom, (prev) => { + if (prev.some((d) => d.id === docEvent.document.id)) return prev; + return [...prev, docEvent.document]; + }); + } + break; + } + + case "data-interrupt-request": { + wasInterrupted = true; + const interruptData = parsed.data as Record; + const actionRequests = (interruptData.action_requests ?? []) as Array<{ + name: string; + args: Record; + }>; + const paired = pairBundleToolCallIds( + contentPartsState.toolCallIndices, + contentPartsState.contentParts, + actionRequests + ); + const bundleToolCallIds: string[] = []; + for (let i = 0; i < actionRequests.length; i++) { + const action = actionRequests[i]; + let targetTcId = paired[i]; + if (!targetTcId) { + targetTcId = freshSynthToolCallId(contentPartsState.toolCallIndices, action.name, i); + addToolCall( + contentPartsState, + toolsWithUI, + targetTcId, + action.name, + action.args, + true + ); + } + updateToolCall(contentPartsState, targetTcId, { + result: { __interrupt__: true, ...interruptData }, + }); + bundleToolCallIds.push(targetTcId); + } + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + // ``tool_call_id`` is stamped on the backend by + // ``checkpointed_subagent_middleware``. Without it we can't + // address the paused subagent on resume — skip rather than + // fabricate a synthetic key. + const interruptId = String(interruptData.tool_call_id ?? ""); + if (interruptId) { + const incoming: PendingInterruptState = { + interruptId, + threadId: streamThreadId, + assistantMsgId, + interruptData, + bundleToolCallIds, + }; + chatStreamStore.setPendingInterrupts(streamThreadId, (prev) => { + const without = prev.filter((p) => p.interruptId !== interruptId); + return [...without, incoming]; + }); + } + break; + } + + case "data-action-log": { + applyActionLogSse(queryClient, streamThreadId, workspaceId, parsed.data); + break; + } + + case "data-action-log-updated": { + applyActionLogUpdatedSse( + queryClient, + streamThreadId, + parsed.data.id, + parsed.data.reversible + ); + break; + } + + case "data-turn-info": { + const turnId = readStreamedChatTurnId(parsed.data); + if (turnId) { + chatStreamStore.setMessages(streamThreadId, (prev) => + applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) + ); + } + break; + } + + case "data-user-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newUserMsgId = `msg-${parsedMsg.messageId}`; + const oldUserMsgId = userMsgId; + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldUserMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) + : m + ) + ); + if (allMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => { + if (!(oldUserMsgId in prev)) { + return { ...prev, [newUserMsgId]: allMentionedDocs }; + } + const { [oldUserMsgId]: _removed, ...rest } = prev; + return { ...rest, [newUserMsgId]: allMentionedDocs }; + }); + } + userMsgId = newUserMsgId; + if (isNewThread) { + queryClient.invalidateQueries({ + queryKey: ["threads", String(workspaceId)], + }); + } + break; + } + + case "data-assistant-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newAssistantMsgId = `msg-${parsedMsg.messageId}`; + const oldAssistantMsgId = assistantMsgId; + tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldAssistantMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) + : m + ) + ); + chatStreamStore.setPendingInterrupts(streamThreadId, (prev) => + prev.map((p) => + p.assistantMsgId === oldAssistantMsgId + ? { ...p, assistantMsgId: newAssistantMsgId } + : p + ) + ); + assistantMsgId = newAssistantMsgId; + break; + } + } + }); + + batcher.flush(); + + if (contentParts.length > 0 && !wasInterrupted) { + trackChatResponseReceived(workspaceId, streamThreadId); + } + } catch (error) { + streamBatcher?.dispose(); + await handleStreamTerminalError({ + error, + flow: "new", + threadId: streamThreadId, + assistantMsgId, + accepted: newAccepted, + workspaceId, + onPreAcceptFailure: async () => { + // Pre-accept failure means the BE never accepted the request — no + // server-side persistence ran. Roll back the optimistic UI. + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.filter((m) => m.id !== userMsgId) + ); + jotaiStore.set(messageDocumentsMapAtom, (prev) => { + if (!(userMsgId in prev)) return prev; + const { [userMsgId]: _removed, ...rest } = prev; + return rest; + }); + }, + }); + } finally { + chatStreamStore.setRunning(streamThreadId, false); + chatStreamStore.clearActive(controller); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(streamThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(streamThreadId), + }); + } +} + +// --------------------------------------------------------------------------- +// Resume (HITL decisions) +// --------------------------------------------------------------------------- + +export async function resumeChat( + ctx: EngineContext, + decisions: Array<{ + type: string; + message?: string; + edited_action?: { name: string; args: Record }; + }> +): Promise { + const { workspaceId, threadId } = ctx; + if (threadId == null) return; + const pendingInterrupts = chatStreamStore.getPendingInterrupts(threadId); + if (pendingInterrupts.length === 0) return; + + const resumeThreadId = pendingInterrupts[0].threadId; + let assistantMsgId = pendingInterrupts[0].assistantMsgId; + const allBundleToolCallIds = pendingInterrupts.flatMap((p) => p.bundleToolCallIds); + chatStreamStore.setPendingInterrupts(resumeThreadId, () => []); + chatStreamStore.setRunning(resumeThreadId, true); + + const controller = new AbortController(); + chatStreamStore.beginActive(resumeThreadId, controller); + + const localFilesystemEnabled = + jotaiStore.get(agentFlagsAtom).data?.enable_desktop_local_filesystem === true; + const disabledTools = jotaiStore.get(disabledToolsAtom); + + const currentThinkingSteps = new Map(); + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + currentReasoningPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts, toolCallIndices } = contentPartsState; + let resumeAccepted = false; + let streamBatcher: FrameBatchedUpdater | null = null; + + const existingMsg = chatStreamStore + .getMessages(resumeThreadId) + .find((m) => m.id === assistantMsgId); + if (existingMsg && Array.isArray(existingMsg.content)) { + contentPartsState.suppressStepSeparators = true; + for (const part of existingMsg.content) { + if (typeof part === "object" && part !== null) { + const p = part as Record; + if (p.type === "text") { + contentParts.push({ type: "text", text: String(p.text ?? "") }); + contentPartsState.currentTextPartIndex = contentParts.length - 1; + } else if (p.type === "tool-call") { + toolCallIndices.set(String(p.toolCallId), contentParts.length); + contentParts.push({ + type: "tool-call", + toolCallId: String(p.toolCallId), + toolName: String(p.toolName), + args: (p.args as Record) ?? {}, + result: p.result as unknown, + ...(typeof p.argsText === "string" ? { argsText: p.argsText } : {}), + ...(typeof p.langchainToolCallId === "string" + ? { langchainToolCallId: p.langchainToolCallId } + : {}), + ...(p.metadata && typeof p.metadata === "object" + ? { metadata: p.metadata as Record } + : {}), + }); + contentPartsState.currentTextPartIndex = -1; + } else if (p.type === "data-thinking-steps") { + const stepsData = p.data as { steps: ThinkingStepData[] } | undefined; + contentParts.push({ + type: "data-thinking-steps", + data: { steps: stepsData?.steps ?? [] }, + }); + for (const step of stepsData?.steps ?? []) { + currentThinkingSteps.set(step.id, step); + } + } + } + } + } + + // Apply each decision to its own card by toolCallId so mixed bundles + // (approve/edit/reject) do not collapse onto ``decisions[0]``. + const decisionByTcId = new Map(); + const tcIds = allBundleToolCallIds; + if (decisions.length === tcIds.length) { + for (let i = 0; i < tcIds.length; i++) decisionByTcId.set(tcIds[i], decisions[i]); + } + if (decisionByTcId.size > 0) { + for (const part of contentParts) { + if (part.type !== "tool-call") continue; + const tcId = part.toolCallId as string | undefined; + const d = tcId ? decisionByTcId.get(tcId) : undefined; + if (!d) continue; + if (typeof part.result !== "object" || part.result === null) continue; + if (!("__interrupt__" in (part.result as Record))) continue; + const decided = d.type; + if (decided === "edit" && d.edited_action) { + const mergedArgs = { ...part.args, ...d.edited_action.args }; + part.args = mergedArgs; + part.argsText = JSON.stringify(mergedArgs, null, 2); + } + part.result = { + ...(part.result as Record), + __decided__: decided, + }; + } + } + + try { + const selection = await getAgentFilesystemSelection(workspaceId, { localFilesystemEnabled }); + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + workspace_id: workspaceId, + decisions, + disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + filesystem_mode: selection.filesystem_mode, + client_platform: selection.client_platform, + local_filesystem_mounts: selection.local_filesystem_mounts, + }), + signal: controller.signal, + }) + ); + + if (!response.ok) { + throw await toHttpResponseError(response); + } + resumeAccepted = true; + + const flushMessages = () => { + chatStreamStore.setMessages(resumeThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + }; + const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); + streamBatcher = batcher; + + await consumeSseEvents(response, async (parsed) => { + if ( + processSharedStreamEvent(parsed, { + contentPartsState, + toolsWithUI, + currentThinkingSteps, + scheduleFlush, + forceFlush, + onTokenUsage: (data) => { + tokenUsageStore.set(assistantMsgId, data); + }, + onTurnStatus: (data) => { + if (data.status === "cancelling") { + chatStreamStore.markRecentCancel(); + } + }, + }) + ) { + return; + } + switch (parsed.type) { + case "data-interrupt-request": { + const interruptData = parsed.data as Record; + const actionRequests = (interruptData.action_requests ?? []) as Array<{ + name: string; + args: Record; + }>; + const paired = pairBundleToolCallIds( + contentPartsState.toolCallIndices, + contentPartsState.contentParts, + actionRequests + ); + const bundleToolCallIds: string[] = []; + for (let i = 0; i < actionRequests.length; i++) { + const action = actionRequests[i]; + let targetTcId = paired[i]; + if (!targetTcId) { + targetTcId = freshSynthToolCallId(contentPartsState.toolCallIndices, action.name, i); + addToolCall( + contentPartsState, + toolsWithUI, + targetTcId, + action.name, + action.args, + true + ); + } + updateToolCall(contentPartsState, targetTcId, { + result: { __interrupt__: true, ...interruptData }, + }); + bundleToolCallIds.push(targetTcId); + } + chatStreamStore.setMessages(resumeThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + { + const interruptId = String(interruptData.tool_call_id ?? ""); + if (interruptId) { + const incoming: PendingInterruptState = { + interruptId, + threadId: resumeThreadId, + assistantMsgId, + interruptData, + bundleToolCallIds, + }; + chatStreamStore.setPendingInterrupts(resumeThreadId, (prev) => { + const without = prev.filter((p) => p.interruptId !== interruptId); + return [...without, incoming]; + }); + } + } + break; + } + + case "data-action-log": { + applyActionLogSse(queryClient, resumeThreadId, workspaceId, parsed.data); + break; + } + + case "data-action-log-updated": { + applyActionLogUpdatedSse( + queryClient, + resumeThreadId, + parsed.data.id, + parsed.data.reversible + ); + break; + } + + case "data-turn-info": { + const turnId = readStreamedChatTurnId(parsed.data); + if (turnId) { + chatStreamStore.setMessages(resumeThreadId, (prev) => + applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) + ); + } + break; + } + + case "data-assistant-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newAssistantMsgId = `msg-${parsedMsg.messageId}`; + const oldAssistantMsgId = assistantMsgId; + tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); + chatStreamStore.setMessages(resumeThreadId, (prev) => + prev.map((m) => + m.id === oldAssistantMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) + : m + ) + ); + assistantMsgId = newAssistantMsgId; + break; + } + } + }); + + batcher.flush(); + } catch (error) { + streamBatcher?.dispose(); + await handleStreamTerminalError({ + error, + flow: "resume", + threadId: resumeThreadId, + assistantMsgId, + accepted: resumeAccepted, + workspaceId, + }); + } finally { + chatStreamStore.setRunning(resumeThreadId, false); + chatStreamStore.clearActive(controller); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(resumeThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(resumeThreadId), + }); + } +} + +// --------------------------------------------------------------------------- +// Regenerate (edit / reload) +// --------------------------------------------------------------------------- + +export async function regenerateChat( + ctx: EngineContext, + newUserQuery: string | null, + editExtras?: { + userMessageContent: ThreadMessageLike["content"]; + userImages: NewChatUserImagePayload[]; + sourceUserMessageId?: string; + }, + editFromPosition?: { + fromMessageId?: number | null; + revertActions?: boolean; + } +): Promise { + const { workspaceId, threadId, priorMessages } = ctx; + if (!threadId) { + toast.error("Cannot regenerate: no active chat thread"); + return; + } + const streamThreadId = threadId; + + const isEdit = newUserQuery !== null; + + // Supersede any previous in-flight turn. + chatStreamStore.abortActive(); + + const messageDocumentsMap = jotaiStore.get(messageDocumentsMapAtom); + const localFilesystemEnabled = + jotaiStore.get(agentFlagsAtom).data?.enable_desktop_local_filesystem === true; + const disabledTools = jotaiStore.get(disabledToolsAtom); + + // Extract the original user query BEFORE removing messages (reload mode). + let userQueryToDisplay: string | undefined; + let originalUserMessageContent: ThreadMessageLike["content"] | null = null; + let originalUserMessageMetadata: ThreadMessageLike["metadata"] | undefined; + let sourceUserMessageId: string | undefined = editExtras?.sourceUserMessageId; + + if (!isEdit) { + const lastUserMessage = [...priorMessages].reverse().find((m) => m.role === "user"); + if (lastUserMessage) { + sourceUserMessageId = lastUserMessage.id; + originalUserMessageContent = lastUserMessage.content; + originalUserMessageMetadata = lastUserMessage.metadata; + for (const part of lastUserMessage.content) { + if (typeof part === "object" && part.type === "text" && "text" in part) { + userQueryToDisplay = part.text; + break; + } + } + } + } else { + userQueryToDisplay = newUserQuery; + } + + // Seed the durable overlay with the pre-regenerate conversation and flip + // to running so the page renders the live stream. + chatStreamStore.begin(streamThreadId, priorMessages); + + const controller = new AbortController(); + chatStreamStore.beginActive(streamThreadId, controller); + + let userMsgId = `msg-user-${Date.now()}`; + let assistantMsgId = `msg-assistant-${Date.now()}`; + const currentThinkingSteps = new Map(); + + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + currentReasoningPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts } = contentPartsState; + let regenerateAccepted = false; + let streamBatcher: FrameBatchedUpdater | null = null; + + const userMessage: ThreadMessageLike = { + id: userMsgId, + role: "user", + content: isEdit + ? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }]) + : originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }], + createdAt: new Date(), + metadata: isEdit ? undefined : originalUserMessageMetadata, + }; + const sourceMentionedDocs = + sourceUserMessageId && messageDocumentsMap[sourceUserMessageId] + ? messageDocumentsMap[sourceUserMessageId] + : []; + try { + const selection = await getAgentFilesystemSelection(workspaceId, { localFilesystemEnabled }); + const regenerateDocIds = sourceMentionedDocs.filter((d) => d.kind === "doc").map((d) => d.id); + const regenerateFolderIds = sourceMentionedDocs + .filter((d) => d.kind === "folder") + .map((d) => d.id); + const regenerateConnectors = sourceMentionedDocs.filter((d) => d.kind === "connector"); + const regenerateThreadIds = sourceMentionedDocs + .filter((d) => d.kind === "thread") + .map((d) => d.id); + + const requestBody: Record = { + workspace_id: workspaceId, + user_query: newUserQuery, + disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, + filesystem_mode: selection.filesystem_mode, + client_platform: selection.client_platform, + local_filesystem_mounts: selection.local_filesystem_mounts, + mentioned_document_ids: regenerateDocIds.length > 0 ? regenerateDocIds : undefined, + mentioned_folder_ids: regenerateFolderIds.length > 0 ? regenerateFolderIds : undefined, + mentioned_connector_ids: + regenerateConnectors.length > 0 ? regenerateConnectors.map((d) => d.id) : undefined, + mentioned_connectors: regenerateConnectors.length > 0 ? regenerateConnectors : undefined, + mentioned_thread_ids: regenerateThreadIds.length > 0 ? regenerateThreadIds : undefined, + mentioned_documents: sourceMentionedDocs.length > 0 ? sourceMentionedDocs : undefined, + }; + if (isEdit) { + requestBody.user_images = editExtras?.userImages ?? []; + } + if (editFromPosition?.fromMessageId != null) { + requestBody.from_message_id = editFromPosition.fromMessageId; + if (editFromPosition.revertActions) { + requestBody.revert_actions = true; + } + } + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(getRegenerateUrl(streamThreadId), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(requestBody), + signal: controller.signal, + }) + ); + + if (!response.ok) { + throw await toHttpResponseError(response); + } + regenerateAccepted = true; + + chatStreamStore.setMessages(streamThreadId, (prev) => { + let base = prev; + if (editFromPosition?.fromMessageId != null) { + const targetId = `msg-${editFromPosition.fromMessageId}`; + const sliceIndex = prev.findIndex((m) => m.id === targetId); + if (sliceIndex >= 0) { + base = prev.slice(0, sliceIndex); + } + } else if (prev.length >= 2) { + base = prev.slice(0, -2); + } + return [ + ...base, + userMessage, + { + id: assistantMsgId, + role: "assistant", + content: [{ type: "text", text: "" }], + createdAt: new Date(), + }, + ]; + }); + if (sourceMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => ({ + ...prev, + [userMsgId]: sourceMentionedDocs, + })); + } + + const flushMessages = () => { + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } + : m + ) + ); + }; + const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); + streamBatcher = batcher; + + await consumeSseEvents(response, async (parsed) => { + if ( + processSharedStreamEvent(parsed, { + contentPartsState, + toolsWithUI, + currentThinkingSteps, + scheduleFlush, + forceFlush, + onTokenUsage: (data) => { + tokenUsageStore.set(assistantMsgId, data); + }, + onTurnStatus: (data) => { + if (data.status === "cancelling") { + chatStreamStore.markRecentCancel(); + } + }, + }) + ) { + return; + } + switch (parsed.type) { + case "data-action-log": { + applyActionLogSse(queryClient, streamThreadId, workspaceId, parsed.data); + break; + } + + case "data-action-log-updated": { + applyActionLogUpdatedSse( + queryClient, + streamThreadId, + parsed.data.id, + parsed.data.reversible + ); + break; + } + + case "data-turn-info": { + const turnId = readStreamedChatTurnId(parsed.data); + if (turnId) { + chatStreamStore.setMessages(streamThreadId, (prev) => + applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) + ); + } + break; + } + + case "data-user-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newUserMsgId = `msg-${parsedMsg.messageId}`; + const oldUserMsgId = userMsgId; + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldUserMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) + : m + ) + ); + if (sourceMentionedDocs.length > 0) { + jotaiStore.set(messageDocumentsMapAtom, (prev) => { + if (!(oldUserMsgId in prev)) { + return { ...prev, [newUserMsgId]: sourceMentionedDocs }; + } + const { [oldUserMsgId]: _removed, ...rest } = prev; + return { ...rest, [newUserMsgId]: sourceMentionedDocs }; + }); + } + userMsgId = newUserMsgId; + break; + } + + case "data-assistant-message-id": { + const parsedMsg = readStreamedMessageId(parsed.data); + if (!parsedMsg) break; + const newAssistantMsgId = `msg-${parsedMsg.messageId}`; + const oldAssistantMsgId = assistantMsgId; + tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); + chatStreamStore.setMessages(streamThreadId, (prev) => + prev.map((m) => + m.id === oldAssistantMsgId + ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) + : m + ) + ); + assistantMsgId = newAssistantMsgId; + break; + } + + case "data-revert-results": { + const summary = parsed.data; + const failureCount = + summary.failed + summary.not_reversible + (summary.permission_denied ?? 0); + if (failureCount > 0) { + toast.warning( + `Pre-revert: ${summary.reverted}/${summary.total} undone, ${failureCount} could not be rolled back.` + ); + } else if (summary.reverted > 0) { + toast.success( + summary.reverted === 1 + ? "Reverted 1 downstream action before regenerating." + : `Reverted ${summary.reverted} downstream actions before regenerating.` + ); + } + for (const r of summary.results) { + if (r.status === "reverted" || r.status === "already_reverted") { + markActionRevertedInCache( + queryClient, + streamThreadId, + r.action_id, + r.new_action_id ?? null + ); + } + } + break; + } + } + }); + + batcher.flush(); + + if (contentParts.length > 0) { + trackChatResponseReceived(workspaceId, streamThreadId); + } + } catch (error) { + streamBatcher?.dispose(); + await handleStreamTerminalError({ + error, + flow: "regenerate", + threadId: streamThreadId, + assistantMsgId, + accepted: regenerateAccepted, + workspaceId, + }); + } finally { + chatStreamStore.setRunning(streamThreadId, false); + chatStreamStore.clearActive(controller); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(streamThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(streamThreadId), + }); + } +} + +export type { HitlDecision }; From 5c3aa72c1a825f28e6ec1cae5b2287eb6ef54ddc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 10/28] feat(chat): add ActiveChatStreamRunner to persist streams across nav --- .../chat/active-chat-stream-runner.tsx | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 surfsense_web/components/chat/active-chat-stream-runner.tsx diff --git a/surfsense_web/components/chat/active-chat-stream-runner.tsx b/surfsense_web/components/chat/active-chat-stream-runner.tsx new file mode 100644 index 000000000..da86c0aa8 --- /dev/null +++ b/surfsense_web/components/chat/active-chat-stream-runner.tsx @@ -0,0 +1,29 @@ +"use client"; + +import { useEffect } from "react"; +import { chatStreamStore } from "@/lib/chat/stream-engine/store"; + +/** + * Persistent, render-null host for app-wide chat-stream lifecycle. + * + * Mounted in the workspace shell (``LayoutDataProvider``), it persists across + * in-app navigation between workspace routes (``/new-chat`` -> ``/chats`` -> + * ``/automations`` and doc-tab switches) and only unmounts on real teardown + * (workspace change / app teardown). On that teardown it aborts the single + * in-flight turn — this replaces the old chat-page unmount abort, which was + * what killed streams on ordinary navigation. + * + * NOTE: the ``hitl-decision`` bridge and ``useChatSessionStateSync`` stay in + * the chat page: both need the currently-viewed thread id (HITL resume can + * only be triggered from the page's approval UI), which the shell-level runner + * does not cleanly have. + */ +export function ActiveChatStreamRunner() { + useEffect(() => { + return () => { + chatStreamStore.abortActive(); + }; + }, []); + + return null; +} From 5e453cefee886d95cdd24aa90049b769cfb7792a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 11/28] feat(chat): mount ActiveChatStreamRunner in the workspace shell --- .../components/layout/providers/LayoutDataProvider.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index f9c2c9072..81f0d974f 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -18,6 +18,7 @@ import { workspacesAtom } from "@/atoms/workspaces/workspace-query.atoms"; import { ActionLogDialog } from "@/components/agent-action-log/action-log-dialog"; import { AnnouncementSpotlight } from "@/components/announcements/AnnouncementSpotlight"; import { AnnouncementsDialog } from "@/components/announcements/AnnouncementsDialog"; +import { ActiveChatStreamRunner } from "@/components/chat/active-chat-stream-runner"; import { AlertDialog, AlertDialogAction, @@ -644,6 +645,9 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider return ( <> + {/* Persistent host: keeps an in-flight chat turn streaming across + in-app navigation and aborts it only on workspace teardown. */} + Date: Mon, 13 Jul 2026 22:17:38 +0200 Subject: [PATCH 12/28] refactor(chat): consume stream store and delegate turns to the engine --- .../new-chat/[[...chat_id]]/page.tsx | 2218 ++--------------- 1 file changed, 236 insertions(+), 1982 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx index 4bb355049..f5b857ce1 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx @@ -7,14 +7,11 @@ import { useExternalStoreRuntime, } from "@assistant-ui/react"; import { useQueryClient } from "@tanstack/react-query"; -import { useAtomValue, useSetAtom, useStore } from "jotai"; +import { useAtomValue, useSetAtom } from "jotai"; import dynamic from "next/dynamic"; import { useParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; -import { z } from "zod"; -import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; -import { disabledToolsAtom } from "@/atoms/agent-tools/agent-tools.atoms"; import { clearTargetCommentIdAtom, currentThreadAtom, @@ -22,24 +19,15 @@ import { setTargetCommentIdAtom, } from "@/atoms/chat/current-thread.atom"; import { - deriveMentionedPayload, type MentionedDocumentInfo, mentionedDocumentsAtom, messageDocumentsMapAtom, - submittedMentionsAtom, } from "@/atoms/chat/mentioned-documents.atom"; -import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; -import { - clearPlanOwnerRegistry, - // extractWriteTodosFromContent, -} from "@/atoms/chat/plan-state.atom"; -import { setPremiumAlertForThreadAtom } from "@/atoms/chat/premium-alert.atom"; +import { clearPlanOwnerRegistry } from "@/atoms/chat/plan-state.atom"; import { closeReportPanelAtom } from "@/atoms/chat/report-panel.atom"; -import { type AgentCreatedDocument, agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; import { closeEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { membersAtom } from "@/atoms/members/members-query.atoms"; -import { removeChatTabAtom, syncChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import { removeChatTabAtom, syncChatTabAtom } from "@/atoms/tabs/tabs.atom"; import { EditMessageDialog, type EditMessageDialogChoice, @@ -47,7 +35,6 @@ import { import { StepSeparatorDataUI } from "@/components/assistant-ui/step-separator"; import { Thread } from "@/components/assistant-ui/thread"; import { - createTokenUsageStore, type TokenUsageData, TokenUsageProvider, } from "@/components/assistant-ui/token-usage-context"; @@ -60,64 +47,31 @@ import { type PendingInterruptState, } from "@/features/chat-messages/hitl"; import { TimelineDataUI } from "@/features/chat-messages/timeline"; -import { - applyActionLogSse, - applyActionLogUpdatedSse, - markActionRevertedInCache, - useAgentActionsQuery, -} from "@/hooks/use-agent-actions-query"; +import { useAgentActionsQuery } from "@/hooks/use-agent-actions-query"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesSync } from "@/hooks/use-messages-sync"; import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; -import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; -import { authenticatedFetch } from "@/lib/auth-fetch"; -import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; -import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; -import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; import { convertToThreadMessage, reconcileInterruptedAssistantMessages, } from "@/lib/chat/message-utils"; -import { createStreamFlushHelpers } from "@/lib/chat/stream-flush"; -import { consumeSseEvents, processSharedStreamEvent } from "@/lib/chat/stream-pipeline"; import { - applyTurnIdToAssistantMessageList, - mergeChatTurnIdIntoMessage, - readStreamedChatTurnId, - readStreamedMessageId, -} from "@/lib/chat/stream-side-effects"; -import { - addToolCall, - buildContentForUI, - type ContentPartsState, - type FrameBatchedUpdater, - type ThinkingStepData, - type ToolUIGate, - updateToolCall, -} from "@/lib/chat/streaming-state"; -import { - appendMessage, - createThread, - getRegenerateUrl, - type ThreadListItem, - type ThreadListResponse, - type ThreadRecord, -} from "@/lib/chat/thread-persistence"; + cancelActiveTurn, + type EngineContext, + regenerateChat, + resumeChat, + startNewChat, +} from "@/lib/chat/stream-engine/engine"; +import { extractMentionedDocuments } from "@/lib/chat/stream-engine/helpers"; +import { chatStreamStore } from "@/lib/chat/stream-engine/store"; +import { useChatStream } from "@/lib/chat/stream-engine/use-chat-stream"; +import type { ThreadRecord } from "@/lib/chat/thread-persistence"; import { extractUserTurnForNewChatApi, type NewChatUserImagePayload, } from "@/lib/chat/user-turn-api-parts"; -import { buildBackendUrl } from "@/lib/env-config"; import { NotFoundError } from "@/lib/error"; -import { - trackChatBlocked, - trackChatCreated, - trackChatErrorDetailed, - trackChatMessageSent, - trackChatResponseReceived, -} from "@/lib/posthog/events"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; const MobileEditorPanel = dynamic( () => @@ -148,156 +102,8 @@ const MobileArtifactsPanel = dynamic( { ssr: false } ); -/** - * Generate a synthetic ``toolCallId`` for an action_request that has no - * matching streamed tool-call card (HITL-blocked subagent calls don't surface - * as tool-call events). Suffixes a counter when the base id is already taken - * — sequential interrupts for the same tool name otherwise collide on - * ``interrupt-${name}-${i}`` and crash assistant-ui with a duplicate-key error. - */ -function freshSynthToolCallId( - toolCallIndices: Map, - toolName: string, - index: number -): string { - const base = `interrupt-${toolName}-${index}`; - if (!toolCallIndices.has(base)) return base; - let n = 1; - while (toolCallIndices.has(`${base}-${n}`)) n++; - return `${base}-${n}`; -} - -/** - * Pair each ``action_request`` to a unique pending tool-call card, preserving - * order so ``decisions[i]`` lines up with ``action_requests[i]`` on the wire. - * - * Same-name bundles (e.g. three ``create_jira_issue``) used to collapse onto - * one card because the matcher keyed by name; this consumes each card via the - * ``claimed`` set and walks forward in DOM order. - */ -function pairBundleToolCallIds( - toolCallIndices: Map, - contentParts: Array<{ - type: string; - toolName?: string; - result?: unknown; - }>, - actionRequests: ReadonlyArray<{ name: string }> -): Array { - const claimed = new Set(); - const paired: Array = []; - for (const action of actionRequests) { - let matched: string | null = null; - for (const [tcId, idx] of toolCallIndices) { - if (claimed.has(tcId)) continue; - const part = contentParts[idx]; - if (!part || part.type !== "tool-call" || part.toolName !== action.name) continue; - const result = part.result as Record | undefined | null; - if (result == null || (result.__interrupt__ === true && !result.__decided__)) { - matched = tcId; - claimed.add(tcId); - break; - } - } - paired.push(matched); - } - return paired; -} - -/** - * Zod schema for mentioned document info (for type-safe parsing). - * - * ``kind`` defaults to ``"doc"`` so messages persisted before folder - * mentions existed deserialise unchanged. - */ -const MentionedDocumentInfoSchema = z.object({ - id: z.number(), - title: z.string(), - document_type: z.string().optional(), - kind: z - .union([z.literal("doc"), z.literal("folder"), z.literal("connector"), z.literal("thread")]) - .optional() - .default("doc"), - connector_type: z.string().optional(), - account_name: z.string().optional(), -}); - -const MentionedDocumentsPartSchema = z.object({ - type: z.literal("mentioned-documents"), - documents: z.array(MentionedDocumentInfoSchema), -}); - -/** - * Extract mentioned documents from message content (type-safe with Zod) - */ -function extractMentionedDocuments(content: unknown): MentionedDocumentInfo[] { - if (!Array.isArray(content)) return []; - - for (const part of content) { - const result = MentionedDocumentsPartSchema.safeParse(part); - if (result.success) { - return result.data.documents.map((doc) => { - if (doc.kind === "connector") { - return { - id: doc.id, - title: doc.title, - kind: "connector", - connector_type: doc.connector_type ?? doc.document_type ?? "UNKNOWN", - account_name: doc.account_name ?? doc.title, - }; - } - if (doc.kind === "folder") { - return { - id: doc.id, - title: doc.title, - kind: "folder", - }; - } - if (doc.kind === "thread") { - return { - id: doc.id, - title: doc.title, - kind: "thread", - }; - } - return { - id: doc.id, - title: doc.title, - document_type: doc.document_type ?? "UNKNOWN", - kind: "doc", - }; - }); - } - } - - return []; -} - -/** - * Every tool call renders a card. The legacy - * ``BASE_TOOLS_WITH_UI`` allowlist used to drop unknown tool calls on the - * floor; we now route everything through ``ToolFallback``. Persisted - * payload size stays bounded because the backend's - * ``format_thinking_step`` summarisation and the - * ``result_length``-only default for unknown tools (see - * ``stream_new_chat.py``) keep the JSON from ballooning. - */ -const TOOLS_WITH_UI_ALL: ToolUIGate = "all"; -const TURN_CANCELLING_INITIAL_DELAY_MS = 200; -const TURN_CANCELLING_BACKOFF_FACTOR = 2; -const TURN_CANCELLING_MAX_DELAY_MS = 1500; -const RECENT_CANCEL_WINDOW_MS = 5_000; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function computeFallbackTurnCancellingRetryDelay(attempt: number): number { - const safeAttempt = Math.max(1, attempt); - const raw = - TURN_CANCELLING_INITIAL_DELAY_MS * TURN_CANCELLING_BACKOFF_FACTOR ** (safeAttempt - 1); - return Math.min(raw, TURN_CANCELLING_MAX_DELAY_MS); -} +/** Stable empty reference so idle threads don't re-render the interrupt provider. */ +const EMPTY_PENDING_INTERRUPTS: PendingInterruptState[] = []; function parseUrlChatId(id: string | string[] | undefined): number { let parsed = 0; @@ -372,103 +178,37 @@ export default function NewChatPage() { const activeThreadId = urlChatId > 0 ? urlChatId : threadId; const handledLoadErrorThreadRef = useRef(null); const [currentThread, setCurrentThread] = useState(null); + // DB-hydrated messages for the viewed thread (idle display). While a turn + // is streaming, the live overlay in ``chatStreamStore`` takes precedence + // (see ``displayMessages``) so it survives this page unmounting on nav. const [messages, setMessages] = useState([]); - const [isRunning, setIsRunning] = useState(false); - const [tokenUsageStore] = useState(() => createTokenUsageStore()); - const abortControllerRef = useRef(null); - const recentCancelRequestedAtRef = useRef(0); - // One entry per paused subagent, in receipt order (which matches the - // backend's ``state.interrupts`` traversal — and therefore the order - // ``slice_decisions_by_tool_call`` consumes on resume). Cleared on submit - // or on a fresh user turn. - const [pendingInterrupts, setPendingInterrupts] = useState([]); - // Per-card staged decisions held until every pending card has submitted, - // at which point we batch them into one ``hitl-decision`` event in the - // same order as ``pendingInterrupts``. Using a ref because partial - // progress should not re-render the page. - const stagedDecisionsByInterruptIdRef = useRef>(new Map()); - const toolsWithUI = TOOLS_WITH_UI_ALL; + + // Durable, cross-navigation streaming state for the viewed thread. + const streamState = useChatStream(activeThreadId); + const isRunning = streamState?.isRunning ?? false; + const pendingInterrupts = streamState?.pendingInterrupts ?? EMPTY_PENDING_INTERRUPTS; + // Live overlay while a turn is streaming / awaiting HITL; DB-hydrated + // messages once the overlay is cleared (the hydration effect drops it only + // after the DB catches up, so there is no finish->refetch gap). + const displayMessages = streamState ? streamState.messages : messages; + + // One shared token-usage store, alive across navigation. + const tokenUsageStore = chatStreamStore.tokenUsage; + const setMessageDocumentsMap = useSetAtom(messageDocumentsMapAtom); - - const persistAssistantErrorMessage = useCallback( - async ({ - threadId, - assistantMsgId, - text, - }: { - threadId: number | null; - assistantMsgId: string; - text: string; - }) => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { - ...m, - content: [{ type: "text", text }], - } - : m - ) - ); - - if (!threadId) return; - - // Persist only temporary assistant placeholders to avoid duplicate rows - // when the message already has a database-backed ID. - if (!assistantMsgId.startsWith("msg-assistant-")) return; - - try { - const savedMessage = await appendMessage(threadId, { - role: "assistant", - content: [{ type: "text", text }], - }); - const newMsgId = `msg-${savedMessage.id}`; - tokenUsageStore.rename(assistantMsgId, newMsgId); - setMessages((prev) => - prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)) - ); - } catch (persistErr) { - console.error("Failed to persist assistant error message:", persistErr); - } - }, - [tokenUsageStore] - ); - - // NOTE: ``persistUserTurn`` / ``persistAssistantTurn`` callbacks - // were removed in the SSE-based message ID handshake refactor. - // ``stream_new_chat`` and ``stream_resume_chat`` now persist both - // the user and assistant rows server-side via - // ``persist_user_turn`` / ``persist_assistant_shell`` and emit - // ``data-user-message-id`` / ``data-assistant-message-id`` SSE - // events; the consumers below rename the optimistic ids in real - // time. ``persistAssistantErrorMessage`` (above) is intentionally - // kept — it is the pre-stream-error fallback fired when the - // server NEVER accepted the request, and the BE has nothing to - // persist in that case. - - // Get disabled tools from the tool toggle UI - const disabledTools = useAtomValue(disabledToolsAtom); - - const jotaiStore = useStore(); - const mentionedDocuments = useAtomValue(mentionedDocumentsAtom); - const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); const setMentionedDocuments = useSetAtom(mentionedDocumentsAtom); const currentThreadState = useAtomValue(currentThreadAtom); const setCurrentThreadMetadata = useSetAtom(setCurrentThreadMetadataAtom); - const setPremiumAlertForThread = useSetAtom(setPremiumAlertForThreadAtom); const setTargetCommentId = useSetAtom(setTargetCommentIdAtom); const clearTargetCommentId = useSetAtom(clearTargetCommentIdAtom); const closeReportPanel = useSetAtom(closeReportPanelAtom); const closeEditorPanel = useSetAtom(closeEditorPanelAtom); const syncChatTab = useSetAtom(syncChatTabAtom); - const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom); const removeChatTab = useSetAtom(removeChatTabAtom); - const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom); - const pendingUserImageUrls = useAtomValue(pendingUserImageDataUrlsAtom); - const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom); - // Edit dialog state. Holds the message id being edited and - // the (already extracted) regenerate args so we can resume the edit - // after the user picks "revert all" / "continue" / "cancel". + + // Edit dialog state. Holds the message id being edited and the (already + // extracted) regenerate args so we can resume the edit after the user picks + // "revert all" / "continue" / "cancel". const [editDialogState, setEditDialogState] = useState<{ fromMessageId: number; userQuery: string | null; @@ -478,14 +218,17 @@ export default function NewChatPage() { downstreamTotalCount: number; } | null>(null); - // Get current user for author info in shared chats - const { data: currentUser } = useAtomValue(currentUserAtom); - const { data: agentFlags } = useAtomValue(agentFlagsAtom); - const localFilesystemEnabled = agentFlags?.enable_desktop_local_filesystem === true; + // Per-card staged decisions held until every pending card has submitted, at + // which point we batch them into one ``hitl-decision`` event in the same + // order as ``pendingInterrupts``. A ref because partial progress should not + // re-render the page. + const stagedDecisionsByInterruptIdRef = useRef>(new Map()); + const threadDetailQuery = useThreadDetail(activeThreadId); const threadMessagesQuery = useThreadMessages(activeThreadId); - // Live collaboration: sync session state and messages via Zero + // Live collaboration: sync session state and messages via Zero. Kept on the + // page because "AI responding" reflects the currently-viewed thread. useChatSessionStateSync(activeThreadId); const { data: membersData } = useAtomValue(membersAtom); @@ -498,10 +241,6 @@ export default function NewChatPage() { content: unknown; author_id: string | null; created_at: string; - // Forwarded so ``convertToThreadMessage`` can rebuild the - // ``metadata.custom.chatTurnId`` on the - // ``ThreadMessageLike``. Required by the inline Revert - // button's per-turn fallback. turn_id?: string | null; }[] ) => { @@ -520,7 +259,6 @@ export default function NewChatPage() { return reconcileInterruptedAssistantMessages(syncedMessages).map((msg) => { const member = msg.author_id ? (memberById.get(msg.author_id) ?? null) : null; - // Preserve existing author info if member lookup fails (e.g., cloned chats) const existingMsg = prevById.get(`msg-${msg.id}`); const existingAuthor = existingMsg?.metadata?.custom?.author as | { displayName?: string | null; avatarUrl?: string | null } @@ -535,10 +273,6 @@ export default function NewChatPage() { created_at: msg.created_at, author_display_name: member?.user_display_name ?? existingAuthor?.displayName ?? null, author_avatar_url: member?.user_avatar_url ?? existingAuthor?.avatarUrl ?? null, - // Forward the per-turn correlation id so the - // inline Revert button's ``(chat_turn_id, - // tool_name, position)`` fallback survives the - // post-stream Zero re-sync. turn_id: msg.turn_id ?? null, }); }); @@ -556,169 +290,33 @@ export default function NewChatPage() { return Number.isNaN(parsed) ? 0 : parsed; }, [params.workspace_id]); - // Unified store for agent-action rows (the same react-query cache - // the agent-actions dialog, the inline Revert button, and the - // per-turn Revert button all read). Hydrates from - // ``GET /threads/{id}/actions`` and is updated incrementally by the - // SSE handlers + revert-batch results below — no atom side-channel. + // Unified store for agent-action rows (react-query cache). Used by the + // edit pre-flight to count reversible downstream actions. const { items: agentActionItems } = useAgentActionsQuery(activeThreadId); - const handleChatFailure = useCallback( - async ({ - error, - flow, - threadId, - assistantMsgId, - }: { - error: unknown; - flow: ChatFlow; - threadId: number | null; - assistantMsgId: string; - }) => { - const normalized = classifyChatError({ - error, - flow, - context: { - workspaceId: workspaceId, - threadId, - }, - }); + // Latest displayed messages, read by the engine wrappers at call time so + // history/slice seeds stay fresh without re-creating the callbacks. + const messagesRef = useRef(displayMessages); + messagesRef.current = displayMessages; - const logger = - normalized.severity === "error" - ? console.error - : normalized.severity === "warn" - ? console.warn - : console.info; - logger(`[NewChatPage] ${flow} ${normalized.kind}:`, error); - - const telemetryPayload = { - flow, - kind: normalized.kind, - error_code: normalized.errorCode, - severity: normalized.severity, - is_expected: normalized.isExpected, - message: normalized.userMessage, - }; - if (normalized.telemetryEvent === "chat_blocked") { - trackChatBlocked(workspaceId, threadId, telemetryPayload); - } else { - trackChatErrorDetailed(workspaceId, threadId, telemetryPayload); - } - - if (normalized.channel === "silent") { - return; - } - - if (normalized.channel === "pinned_inline") { - if (threadId) { - setPremiumAlertForThread({ - threadId, - message: normalized.userMessage, - userId: currentUser?.id ?? null, - }); - } - if (normalized.assistantMessage) { - await persistAssistantErrorMessage({ - threadId, - assistantMsgId, - text: normalized.assistantMessage, - }); - } - return; - } - - if (normalized.channel === "inline") { - if (normalized.assistantMessage) { - await persistAssistantErrorMessage({ - threadId, - assistantMsgId, - text: normalized.assistantMessage, - }); - } - toast.error(normalized.userMessage); - return; - } - - toast.error(normalized.userMessage); - }, - [currentUser?.id, persistAssistantErrorMessage, workspaceId, setPremiumAlertForThread] + const buildCtx = useCallback( + (): EngineContext => ({ + workspaceId, + threadId: activeThreadId, + priorMessages: messagesRef.current, + view: { setThreadId, setCurrentThread }, + }), + [workspaceId, activeThreadId] ); - const handleStreamTerminalError = useCallback( - async ({ - error, - flow, - threadId, - assistantMsgId, - accepted, - onAbort, - onPreAcceptFailure, - onAcceptedStreamError, - }: { - error: unknown; - flow: ChatFlow; - threadId: number | null; - assistantMsgId: string; - accepted: boolean; - onAbort?: () => Promise; - onPreAcceptFailure?: () => Promise; - onAcceptedStreamError?: () => Promise; - }) => { - if (error instanceof Error && error.name === "AbortError") { - await onAbort?.(); - return; - } - - if (!accepted) { - await onPreAcceptFailure?.(); - } else { - await onAcceptedStreamError?.(); - } - - await handleChatFailure({ - error: !accepted ? tagPreAcceptSendFailure(error) : error, - flow, - threadId, - assistantMsgId: accepted ? assistantMsgId : "no-persist-assistant", - }); - }, - [handleChatFailure] - ); - - const fetchWithTurnCancellingRetry = useCallback(async (runFetch: () => Promise) => { - const maxAttempts = 4; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const response = await runFetch(); - if (response.ok) { - return response; - } - const error = await toHttpResponseError(response); - const withMeta = error as Error & { errorCode?: string; retryAfterMs?: number }; - const isTurnCancelling = withMeta.errorCode === "TURN_CANCELLING"; - const isRecentThreadBusyAfterCancel = - withMeta.errorCode === "THREAD_BUSY" && - Date.now() - recentCancelRequestedAtRef.current <= RECENT_CANCEL_WINDOW_MS; - if ((isTurnCancelling || isRecentThreadBusyAfterCancel) && attempt < maxAttempts) { - const waitMs = withMeta.retryAfterMs ?? computeFallbackTurnCancellingRetryDelay(attempt); - await sleep(waitMs); - continue; - } - throw error; - } - - throw Object.assign(new Error("Turn cancellation retry limit exceeded"), { - errorCode: "TURN_CANCELLING", - }); - }, []); - const hydratedMessagesRef = useRef<{ threadId: number | null; data: typeof threadMessagesQuery.data; }>({ threadId: null, data: undefined }); - // Reset thread-local runtime state on route/workspace changes. Data fetching - // is handled by React Query below so the chat shell can render immediately. + // Reset thread-local runtime state on route/workspace changes. The durable + // streaming overlay is preserved for any still-running thread (and the newly + // viewed thread) via ``clearInactive`` so an in-flight turn survives nav. useEffect(() => { const nextThreadId = urlChatId > 0 ? urlChatId : null; handledLoadErrorThreadRef.current = null; @@ -732,13 +330,11 @@ export default function NewChatPage() { clearPlanOwnerRegistry(); closeReportPanel(); closeEditorPanel(); - // Note: agent-action data is keyed by threadId in react-query so - // switching threads naturally swaps caches; no explicit reset. + chatStreamStore.clearInactive(nextThreadId); }, [ urlChatId, setMentionedDocuments, setMessageDocumentsMap, - tokenUsageStore, closeReportPanel, closeEditorPanel, ]); @@ -773,6 +369,7 @@ export default function NewChatPage() { return; } + // Per-thread gate: never overwrite the live overlay of a running turn. if (isRunning) { return; } @@ -800,13 +397,18 @@ export default function NewChatPage() { } setMessageDocumentsMap(restoredDocsMap); hydratedMessagesRef.current = { threadId: activeThreadId, data: messagesResponse }; + + // The DB is now authoritative for this thread — drop the streaming + // overlay so we render DB messages (no-op while running / HITL-pending). + if (loadedMessages.length >= chatStreamStore.getMessages(activeThreadId).length) { + chatStreamStore.clear(activeThreadId); + } }, [ activeThreadId, isRunning, messages.length, setMessageDocumentsMap, threadMessagesQuery.data, - tokenUsageStore, ]); useEffect(() => { @@ -838,8 +440,7 @@ export default function NewChatPage() { threadMessagesQuery.error, ]); - // Prefetch document titles for @ mention picker - // Runs when user lands on page so data is ready when they type @ + // Prefetch document titles for @ mention picker so data is ready on type. useEffect(() => { if (!workspaceId) return; @@ -856,10 +457,7 @@ export default function NewChatPage() { }); }, [workspaceId, queryClient]); - // Handle scroll to comment from URL query params (e.g., from inbox item click) - // Read from window.location.search inside the effect instead of subscribing via - // useSearchParams() — avoids re-rendering this heavy component tree on every - // unrelated query-string change. (Vercel Best Practice: rerender-defer-reads 5.2) + // Handle scroll to comment from URL query params (e.g., from inbox click). useEffect(() => { const readAndApplyCommentId = () => { const params = new URLSearchParams(window.location.search); @@ -874,10 +472,8 @@ export default function NewChatPage() { readAndApplyCommentId(); - // Also respond to SPA navigations (back/forward) that change the query string window.addEventListener("popstate", readAndApplyCommentId); - // Cleanup on unmount or when navigating away return () => { window.removeEventListener("popstate", readAndApplyCommentId); clearTargetCommentId(); @@ -919,934 +515,158 @@ export default function NewChatPage() { setCurrentThreadMetadata, ]); - // Cleanup on unmount - abort any in-flight requests - useEffect(() => { - return () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - }; - }, []); - - // Cancel ongoing request - const cancelRun = useCallback(async () => { - if (threadId) { - try { - const response = await authenticatedFetch( - buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), - { - method: "POST", - } - ); - if (response.ok) { - const payload = (await response.json()) as { - error_code?: string; - }; - if (payload.error_code === "TURN_CANCELLING") { - recentCancelRequestedAtRef.current = Date.now(); - } - } - } catch (error) { - console.warn("[NewChatPage] Failed to signal cancel-active-turn:", error); - } - } - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - setIsRunning(false); - }, [threadId]); - // Handle new message from user const onNew = useCallback( + (message: AppendMessage) => startNewChat(buildCtx(), message), + [buildCtx] + ); + + // Cancel the in-flight turn (targets the active stream's owner thread). + const onCancel = useCallback(async () => { + await cancelActiveTurn(); + }, []); + + // Convert message (pass through since already in correct format) + const convertMessage = useCallback( + (message: ThreadMessageLike): ThreadMessageLike => message, + [] + ); + + // Handle editing a message - truncates history and regenerates with new + // query. When ``message.sourceId`` is set we pin ``from_message_id`` so the + // backend rewinds to the right checkpoint, and prompt the user to revert / + // continue / cancel before regenerating. + const onEdit = useCallback( async (message: AppendMessage) => { - // Abort any previous streaming request to prevent race conditions - // when user sends a second query while the first is still streaming - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; + const { userQuery, userImages } = extractUserTurnForNewChatApi(message, []); + const queryForApi = userQuery.trim(); + if (!queryForApi && userImages.length === 0) { + toast.error("Cannot edit with empty message"); + return; } - // Prefer the submit-time snapshot; fall back to the live atom - // for the send-button path. - const submittedSnapshot = jotaiStore.get(submittedMentionsAtom); - jotaiStore.set(submittedMentionsAtom, null); - const activeMentions = submittedSnapshot ?? mentionedDocuments; - const mentionPayload = deriveMentionedPayload(activeMentions); - if (activeMentions.length > 0) { - setMentionedDocuments([]); + const userMessageContent = message.content as unknown as ThreadMessageLike["content"]; + + const sourceId = (message as { sourceId?: string }).sourceId; + const fromMessageId = + sourceId && /^msg-\d+$/.test(sourceId) + ? Number.parseInt(sourceId.replace(/^msg-/, ""), 10) + : null; + + if (fromMessageId == null) { + await regenerateChat(buildCtx(), queryForApi, { + userMessageContent, + userImages, + sourceUserMessageId: sourceId, + }); + return; } - const urlsSnapshot = [...pendingUserImageUrls]; - const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot); - - if (!userQuery.trim() && userImages.length === 0) return; - - // Lazy thread creation: create thread on first message if it doesn't exist - let currentThreadId = threadId; - let isNewThread = false; - if (!currentThreadId) { - try { - const newThread = await createThread(workspaceId, "New Chat"); - currentThreadId = newThread.id; - setThreadId(currentThreadId); - // Set currentThread so share button in header appears immediately - setCurrentThread(newThread); - queryClient.setQueryData(cacheKeys.threads.detail(newThread.id), newThread); - queryClient.setQueryData(cacheKeys.threads.messages(newThread.id), { messages: [] }); - - // Track chat creation - trackChatCreated(workspaceId, currentThreadId); - - isNewThread = true; - // Update URL silently using browser API (not router.replace) to avoid - // interrupting the ongoing fetch/streaming with React navigation - window.history.replaceState( - null, - "", - `/dashboard/${workspaceId}/new-chat/${currentThreadId}` - ); - } catch (error) { - console.error("[NewChatPage] Failed to create thread:", error); - await handleChatFailure({ - error: tagPreAcceptSendFailure(error), - flow: "new", - threadId: currentThreadId, - assistantMsgId: "no-persist-assistant", - }); - return; + const msgs = messagesRef.current; + const editedIndex = msgs.findIndex((m) => m.id === `msg-${fromMessageId}`); + let downstreamReversibleCount = 0; + let downstreamTotalCount = 0; + if (editedIndex >= 0) { + const downstream = msgs.slice(editedIndex + 1); + downstreamTotalCount = downstream.length; + const seenTurns = new Set(); + const downstreamTurnIds = new Set(); + for (const m of downstream) { + const meta = (m.metadata ?? {}) as { custom?: { chatTurnId?: string } }; + const tid = meta.custom?.chatTurnId; + if (!tid || seenTurns.has(tid)) continue; + seenTurns.add(tid); + downstreamTurnIds.add(tid); + } + for (const a of agentActionItems) { + if (!a.chat_turn_id || !downstreamTurnIds.has(a.chat_turn_id)) continue; + if ( + a.reversible && + (a.reverted_by_action_id === null || a.reverted_by_action_id === undefined) && + !a.is_revert_action && + (a.error === null || a.error === undefined) + ) { + downstreamReversibleCount += 1; + } } } - if (urlsSnapshot.length > 0) { - setPendingUserImageUrls((prev) => prev.filter((u) => !urlsSnapshot.includes(u))); + if (downstreamReversibleCount === 0) { + await regenerateChat( + buildCtx(), + queryForApi, + { userMessageContent, userImages, sourceUserMessageId: sourceId }, + { fromMessageId, revertActions: false } + ); + return; } - // Add user message to state. Mutable because the SSE - // ``data-user-message-id`` handler (below) renames this - // optimistic id to the canonical ``msg-{db_id}`` once the - // backend's ``persist_user_turn`` resolves the row, and - // the in-stream flush / interrupt closures need to see - // the post-rename value via this live ``let`` binding. - let userMsgId = `msg-user-${Date.now()}`; - - // Always include author metadata so the UI layer can decide visibility - const authorMetadata = currentUser - ? { - custom: { - author: { - displayName: currentUser.display_name ?? null, - avatarUrl: currentUser.avatar_url ?? null, - }, - }, - } - : undefined; - - const existingImageUrls = new Set( - message.content - .filter( - (p): p is { type: "image"; image: string } => - typeof p === "object" && - p !== null && - "type" in p && - p.type === "image" && - "image" in p - ) - .map((p) => p.image) - ); - const extraImageParts = urlsSnapshot - .filter((u) => !existingImageUrls.has(u)) - .map((image) => ({ type: "image" as const, image })); - const userDisplayContent = [...message.content, ...extraImageParts]; - - const userMessage: ThreadMessageLike = { - id: userMsgId, - role: "user", - content: userDisplayContent, - createdAt: new Date(), - metadata: authorMetadata, - }; - setMessages((prev) => [...prev, userMessage]); - - // Track message sent - trackChatMessageSent(workspaceId, currentThreadId, { - hasAttachments: userImages.length > 0, - hasMentionedDocuments: - mentionPayload.document_ids.length > 0 || - mentionPayload.folder_ids.length > 0 || - mentionPayload.connector_ids.length > 0, - messageLength: userQuery.length, + setEditDialogState({ + fromMessageId, + userQuery: queryForApi, + userMessageContent, + userImages, + downstreamReversibleCount, + downstreamTotalCount, }); + }, + [buildCtx, agentActionItems] + ); - // Collect unique mention chips for display & persistence. - // The ``kind`` field is forwarded to the backend - // so the persisted ``mentioned-documents`` content part - // can render the correct chip type on reload. - const allMentionedDocs: MentionedDocumentInfo[] = []; - const seenDocKeys = new Set(); - for (const doc of activeMentions) { - const key = getMentionDocKey(doc); - if (seenDocKeys.has(key)) continue; - seenDocKeys.add(key); - allMentionedDocs.push(doc); + const handleApprovalSubmit = useCallback( + (interruptId: string, decisions: HitlDecision[]) => { + // Stage this card's decisions; only fire the resume once every pending + // card in the current turn has submitted, so the backend slicer sees a + // single concatenated decisions list. + stagedDecisionsByInterruptIdRef.current.set(interruptId, decisions); + if (stagedDecisionsByInterruptIdRef.current.size < pendingInterrupts.length) { + return; } - - if (allMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => ({ - ...prev, - [userMsgId]: allMentionedDocs, - })); - } - - // Start streaming response - setIsRunning(true); - const controller = new AbortController(); - abortControllerRef.current = controller; - - // Prepare assistant message. Mutable for the same reason - // as ``userMsgId`` above — the ``data-assistant-message-id`` - // SSE handler reassigns this once - // ``persist_assistant_shell`` returns its canonical id. - let assistantMsgId = `msg-assistant-${Date.now()}`; - const currentThinkingSteps = new Map(); - const contentPartsState: ContentPartsState = { - contentParts: [], - currentTextPartIndex: -1, - currentReasoningPartIndex: -1, - toolCallIndices: new Map(), - }; - const { contentParts } = contentPartsState; - let wasInterrupted = false; - let newAccepted = false; - let streamBatcher: FrameBatchedUpdater | null = null; - - try { - const selection = await getAgentFilesystemSelection(workspaceId, { - localFilesystemEnabled, - }); - if ( - selection.filesystem_mode === "desktop_local_folder" && - (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0) - ) { - toast.error("Select a local folder before using Local Folder mode."); + const ordered: HitlDecision[] = []; + for (const pi of pendingInterrupts) { + const staged = stagedDecisionsByInterruptIdRef.current.get(pi.interruptId); + if (!staged) { return; } - - // Build message history for context - const messageHistory = messages - .filter((m) => m.role === "user" || m.role === "assistant") - .map((m) => { - let text = ""; - for (const part of m.content) { - if (typeof part === "object" && part.type === "text" && "text" in part) { - text += part.text; - } - } - return { role: m.role, content: text }; - }) - .filter((m) => m.content.length > 0); - - // Backend expects each mention kind in its own payload bucket. - const hasDocumentIds = mentionPayload.document_ids.length > 0; - const hasFolderIds = mentionPayload.folder_ids.length > 0; - const hasConnectorIds = mentionPayload.connector_ids.length > 0; - const hasThreadIds = mentionPayload.thread_ids.length > 0; - - const response = await fetchWithTurnCancellingRetry(() => - authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - chat_id: currentThreadId, - user_query: userQuery.trim(), - workspace_id: workspaceId, - filesystem_mode: selection.filesystem_mode, - client_platform: selection.client_platform, - local_filesystem_mounts: selection.local_filesystem_mounts, - messages: messageHistory, - mentioned_document_ids: hasDocumentIds ? mentionPayload.document_ids : undefined, - mentioned_folder_ids: hasFolderIds ? mentionPayload.folder_ids : undefined, - mentioned_connector_ids: hasConnectorIds ? mentionPayload.connector_ids : undefined, - mentioned_connectors: hasConnectorIds ? mentionPayload.connectors : undefined, - mentioned_thread_ids: hasThreadIds ? mentionPayload.thread_ids : undefined, - // Full mention metadata so the backend can persist a - // ``mentioned-documents`` ContentPart on the user message. - mentioned_documents: allMentionedDocs.length > 0 ? allMentionedDocs : undefined, - disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, - ...(userImages.length > 0 ? { user_images: userImages } : {}), - }), - signal: controller.signal, - }) - ); - - if (!response.ok) { - throw await toHttpResponseError(response); - } - newAccepted = true; - setMessages((prev) => [ - ...prev, - { - id: assistantMsgId, - role: "assistant", - content: [{ type: "text", text: "" }], - createdAt: new Date(), - }, - ]); - - const flushMessages = () => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - }; - const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); - streamBatcher = batcher; - - await consumeSseEvents(response, async (parsed) => { - if ( - processSharedStreamEvent(parsed, { - contentPartsState, - toolsWithUI, - currentThinkingSteps, - scheduleFlush, - forceFlush, - onTokenUsage: (data) => { - tokenUsageStore.set(assistantMsgId, data); - }, - onTurnStatus: (data) => { - if (data.status === "cancelling") { - recentCancelRequestedAtRef.current = Date.now(); - } - }, - }) - ) { - return; - } - switch (parsed.type) { - case "data-thread-title-update": { - const titleData = parsed.data as { threadId: number; title: string }; - if (titleData?.title && titleData?.threadId === currentThreadId) { - setCurrentThread((prev) => (prev ? { ...prev, title: titleData.title } : prev)); - updateChatTabTitle({ chatId: currentThreadId, title: titleData.title }); - queryClient.setQueriesData( - { queryKey: ["threads", String(workspaceId)] }, - (old) => { - if (!old) return old; - const updateTitle = (list: ThreadListItem[]) => - list.map((t) => - t.id === titleData.threadId ? { ...t, title: titleData.title } : t - ); - return { - ...old, - threads: updateTitle(old.threads), - archived_threads: updateTitle(old.archived_threads), - }; - } - ); - } - break; - } - - case "data-documents-updated": { - const docEvent = parsed.data as { - action: string; - document: AgentCreatedDocument; - }; - if (docEvent?.document?.id) { - setAgentCreatedDocuments((prev) => { - if (prev.some((d) => d.id === docEvent.document.id)) return prev; - return [...prev, docEvent.document]; - }); - } - break; - } - - case "data-interrupt-request": { - wasInterrupted = true; - const interruptData = parsed.data as Record; - const actionRequests = (interruptData.action_requests ?? []) as Array<{ - name: string; - args: Record; - }>; - const paired = pairBundleToolCallIds( - contentPartsState.toolCallIndices, - contentPartsState.contentParts, - actionRequests - ); - const bundleToolCallIds: string[] = []; - for (let i = 0; i < actionRequests.length; i++) { - const action = actionRequests[i]; - let targetTcId = paired[i]; - if (!targetTcId) { - targetTcId = freshSynthToolCallId( - contentPartsState.toolCallIndices, - action.name, - i - ); - addToolCall( - contentPartsState, - toolsWithUI, - targetTcId, - action.name, - action.args, - true - ); - } - updateToolCall(contentPartsState, targetTcId, { - result: { __interrupt__: true, ...interruptData }, - }); - bundleToolCallIds.push(targetTcId); - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - if (currentThreadId) { - // ``tool_call_id`` is stamped on the backend by - // ``checkpointed_subagent_middleware``. Without it we - // can't address the paused subagent on resume — skip - // rather than fabricate a synthetic key. - const interruptId = String(interruptData.tool_call_id ?? ""); - if (interruptId) { - const incoming: PendingInterruptState = { - interruptId, - threadId: currentThreadId, - assistantMsgId, - interruptData, - bundleToolCallIds, - }; - setPendingInterrupts((prev) => { - const without = prev.filter((p) => p.interruptId !== interruptId); - return [...without, incoming]; - }); - } - } - break; - } - - case "data-action-log": { - applyActionLogSse(queryClient, currentThreadId, workspaceId, parsed.data); - break; - } - - case "data-action-log-updated": { - applyActionLogUpdatedSse( - queryClient, - currentThreadId, - parsed.data.id, - parsed.data.reversible - ); - break; - } - - case "data-turn-info": { - const turnId = readStreamedChatTurnId(parsed.data); - if (turnId) { - setMessages((prev) => - applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) - ); - } - break; - } - - case "data-user-message-id": { - // Server-authoritative user message id resolved by - // ``persist_user_turn`` (or recovered via ON CONFLICT). - // Rename the optimistic ``msg-user-XXX`` placeholder to - // the canonical ``msg-{db_id}`` so DB-id-gated UI - // (comments, edit-from-this-message) unlocks immediately, - // migrate the local mentioned-documents map, and reassign - // the closure variable so all downstream - // ``m.id === userMsgId`` checks see the new value. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newUserMsgId = `msg-${parsedMsg.messageId}`; - const oldUserMsgId = userMsgId; - setMessages((prev) => - prev.map((m) => - m.id === oldUserMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) - : m - ) - ); - if (allMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => { - if (!(oldUserMsgId in prev)) { - return { ...prev, [newUserMsgId]: allMentionedDocs }; - } - const { [oldUserMsgId]: _removed, ...rest } = prev; - return { ...rest, [newUserMsgId]: allMentionedDocs }; - }); - } - userMsgId = newUserMsgId; - if (isNewThread) { - // First user-side row landed in ``new_chat_messages``; - // refresh the sidebar so the freshly-bumped - // ``thread.updated_at`` reorders this thread. - queryClient.invalidateQueries({ - queryKey: ["threads", String(workspaceId)], - }); - } - break; - } - - case "data-assistant-message-id": { - // Server-authoritative assistant message id resolved - // by ``persist_assistant_shell``. Rename the optimistic - // id, migrate ``tokenUsageStore`` so any pending - // ``data-token-usage`` payload binds to the new id, - // remap any in-flight ``pendingInterrupts`` entries, - // and reassign the closure variable so the in-stream - // flush callback (line ~1074) keeps writing to the - // renamed message. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newAssistantMsgId = `msg-${parsedMsg.messageId}`; - const oldAssistantMsgId = assistantMsgId; - tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); - setMessages((prev) => - prev.map((m) => - m.id === oldAssistantMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) - : m - ) - ); - setPendingInterrupts((prev) => - prev.map((p) => - p.assistantMsgId === oldAssistantMsgId - ? { ...p, assistantMsgId: newAssistantMsgId } - : p - ) - ); - assistantMsgId = newAssistantMsgId; - break; - } - } - }); - - batcher.flush(); - - // Server-authoritative persistence: ``stream_new_chat`` - // already wrote the user row in ``persist_user_turn`` - // (the FE renamed the optimistic id mid-stream via - // ``data-user-message-id``) and finalises the assistant - // row in ``finalize_assistant_turn`` from a shielded - // ``finally`` block. Nothing left for the FE to persist - // here — track the response and unblock the UI. - if (contentParts.length > 0 && !wasInterrupted) { - trackChatResponseReceived(workspaceId, currentThreadId); - } - } catch (error) { - streamBatcher?.dispose(); - await handleStreamTerminalError({ - error, - flow: "new", - threadId: currentThreadId, - assistantMsgId, - accepted: newAccepted, - // Server-side ``finalize_assistant_turn`` runs from a - // shielded ``anyio.CancelScope(shield=True)`` finally - // block, so partial content (incl. abort-mid-stream) - // is already persisted by the BE for the assistant - // row, and ``persist_user_turn`` ran before any LLM - // call. The FE's only remaining responsibility on - // abort / accepted-stream-error is to surface the - // error toast (handled by ``handleStreamTerminalError`` - // itself). - onPreAcceptFailure: async () => { - // Pre-accept failure means the BE never accepted the - // request — no server-side persistence ran. Roll - // back the optimistic UI insertions we made before - // the fetch so the user message and any local - // mentioned-docs metadata don't linger. - setMessages((prev) => prev.filter((m) => m.id !== userMsgId)); - setMessageDocumentsMap((prev) => { - if (!(userMsgId in prev)) return prev; - const { [userMsgId]: _removed, ...rest } = prev; - return rest; - }); - }, - }); - } finally { - setIsRunning(false); - abortControllerRef.current = null; - if (currentThreadId) { - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.messages(currentThreadId), - }); - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.detail(currentThreadId), - }); - } + ordered.push(...staged); } - }, - [ - threadId, - workspaceId, - messages, - jotaiStore, - mentionedDocuments, - setMentionedDocuments, - setMessageDocumentsMap, - setAgentCreatedDocuments, - queryClient, - currentUser, - localFilesystemEnabled, - disabledTools, - updateChatTabTitle, - tokenUsageStore, - pendingUserImageUrls, - setPendingUserImageUrls, - fetchWithTurnCancellingRetry, - handleStreamTerminalError, - handleChatFailure, - ] - ); - - const handleResume = useCallback( - async ( - decisions: Array<{ - type: string; - message?: string; - edited_action?: { name: string; args: Record }; - }> - ) => { - if (pendingInterrupts.length === 0) return; - // All cards in this turn share the same threadId/assistantMsgId - // (they're siblings of one parent agent step), so reading from - // the first entry is safe. - const resumeThreadId = pendingInterrupts[0].threadId; - // Destructured separately as ``let`` so the SSE - // ``data-assistant-message-id`` handler (resume always - // allocates a fresh server-side row) can rename it to - // the canonical ``msg-{db_id}`` mid-stream. - let assistantMsgId = pendingInterrupts[0].assistantMsgId; - // Concatenate every card's tool-call ids in pendingInterrupts order; - // this matches the ``decisions`` ordering produced by - // ``handleApprovalSubmit`` and the backend slicer's traversal of - // ``state.interrupts``. - const allBundleToolCallIds = pendingInterrupts.flatMap((p) => p.bundleToolCallIds); - setPendingInterrupts([]); stagedDecisionsByInterruptIdRef.current.clear(); - setIsRunning(true); - - const controller = new AbortController(); - abortControllerRef.current = controller; - - const currentThinkingSteps = new Map(); - - const contentPartsState: ContentPartsState = { - contentParts: [], - currentTextPartIndex: -1, - currentReasoningPartIndex: -1, - toolCallIndices: new Map(), - }; - const { contentParts, toolCallIndices } = contentPartsState; - let resumeAccepted = false; - let streamBatcher: FrameBatchedUpdater | null = null; - - const existingMsg = messages.find((m) => m.id === assistantMsgId); - if (existingMsg && Array.isArray(existingMsg.content)) { - // See ``ContentPartsState.suppressStepSeparators`` doc. - contentPartsState.suppressStepSeparators = true; - for (const part of existingMsg.content) { - if (typeof part === "object" && part !== null) { - const p = part as Record; - if (p.type === "text") { - contentParts.push({ type: "text", text: String(p.text ?? "") }); - contentPartsState.currentTextPartIndex = contentParts.length - 1; - } else if (p.type === "tool-call") { - toolCallIndices.set(String(p.toolCallId), contentParts.length); - contentParts.push({ - type: "tool-call", - toolCallId: String(p.toolCallId), - toolName: String(p.toolName), - args: (p.args as Record) ?? {}, - result: p.result as unknown, - // argsText: assistant-ui prefers it over - // JSON.stringify(args), so restoring it keeps - // pretty-printed JSON across reloads. - ...(typeof p.argsText === "string" ? { argsText: p.argsText } : {}), - ...(typeof p.langchainToolCallId === "string" - ? { langchainToolCallId: p.langchainToolCallId } - : {}), - // metadata: spanId / thinkingStepId drive the - // timeline's step↔tool join. Dropping these - // here orphans every rehydrated tool-call. - ...(p.metadata && typeof p.metadata === "object" - ? { metadata: p.metadata as Record } - : {}), - }); - contentPartsState.currentTextPartIndex = -1; - } else if (p.type === "data-thinking-steps") { - const stepsData = p.data as { steps: ThinkingStepData[] } | undefined; - contentParts.push({ - type: "data-thinking-steps", - data: { steps: stepsData?.steps ?? [] }, - }); - for (const step of stepsData?.steps ?? []) { - currentThinkingSteps.set(step.id, step); - } - } - } - } - } - - // Apply each decision to its own card by toolCallId so mixed - // bundles (approve/edit/reject) and multi-edit bundles do not - // collapse onto ``decisions[0]``. Cards outside the bundle are - // untouched. Mirrors the host ``hitl-decision`` handler. - const decisionByTcId = new Map(); - const tcIds = allBundleToolCallIds; - if (decisions.length === tcIds.length) { - for (let i = 0; i < tcIds.length; i++) decisionByTcId.set(tcIds[i], decisions[i]); - } - if (decisionByTcId.size > 0) { - for (const part of contentParts) { - if (part.type !== "tool-call") continue; - const tcId = part.toolCallId as string | undefined; - const d = tcId ? decisionByTcId.get(tcId) : undefined; - if (!d) continue; - if (typeof part.result !== "object" || part.result === null) continue; - if (!("__interrupt__" in (part.result as Record))) continue; - const decided = d.type; - if (decided === "edit" && d.edited_action) { - const mergedArgs = { ...part.args, ...d.edited_action.args }; - part.args = mergedArgs; - // Sync argsText so the rendered card shows the - // edited inputs (assistant-ui prefers it over - // JSON.stringify(args)). - part.argsText = JSON.stringify(mergedArgs, null, 2); - } - part.result = { - ...(part.result as Record), - __decided__: decided, - }; - } - } - - try { - const selection = await getAgentFilesystemSelection(workspaceId, { - localFilesystemEnabled, - }); - const response = await fetchWithTurnCancellingRetry(() => - authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - workspace_id: workspaceId, - decisions, - disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, - filesystem_mode: selection.filesystem_mode, - client_platform: selection.client_platform, - local_filesystem_mounts: selection.local_filesystem_mounts, - }), - signal: controller.signal, - }) - ); - - if (!response.ok) { - throw await toHttpResponseError(response); - } - resumeAccepted = true; - - const flushMessages = () => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - }; - const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); - streamBatcher = batcher; - - await consumeSseEvents(response, async (parsed) => { - if ( - processSharedStreamEvent(parsed, { - contentPartsState, - toolsWithUI, - currentThinkingSteps, - scheduleFlush, - forceFlush, - onTokenUsage: (data) => { - tokenUsageStore.set(assistantMsgId, data); - }, - onTurnStatus: (data) => { - if (data.status === "cancelling") { - recentCancelRequestedAtRef.current = Date.now(); - } - }, - }) - ) { - return; - } - switch (parsed.type) { - case "data-interrupt-request": { - const interruptData = parsed.data as Record; - const actionRequests = (interruptData.action_requests ?? []) as Array<{ - name: string; - args: Record; - }>; - const paired = pairBundleToolCallIds( - contentPartsState.toolCallIndices, - contentPartsState.contentParts, - actionRequests - ); - const bundleToolCallIds: string[] = []; - for (let i = 0; i < actionRequests.length; i++) { - const action = actionRequests[i]; - let targetTcId = paired[i]; - if (!targetTcId) { - targetTcId = freshSynthToolCallId( - contentPartsState.toolCallIndices, - action.name, - i - ); - addToolCall( - contentPartsState, - toolsWithUI, - targetTcId, - action.name, - action.args, - true - ); - } - updateToolCall(contentPartsState, targetTcId, { - result: { __interrupt__: true, ...interruptData }, - }); - bundleToolCallIds.push(targetTcId); - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - { - const interruptId = String(interruptData.tool_call_id ?? ""); - if (interruptId) { - const incoming: PendingInterruptState = { - interruptId, - threadId: resumeThreadId, - assistantMsgId, - interruptData, - bundleToolCallIds, - }; - setPendingInterrupts((prev) => { - const without = prev.filter((p) => p.interruptId !== interruptId); - return [...without, incoming]; - }); - } - } - break; - } - - case "data-action-log": { - applyActionLogSse(queryClient, resumeThreadId, workspaceId, parsed.data); - break; - } - - case "data-action-log-updated": { - applyActionLogUpdatedSse( - queryClient, - resumeThreadId, - parsed.data.id, - parsed.data.reversible - ); - break; - } - - case "data-turn-info": { - const turnId = readStreamedChatTurnId(parsed.data); - if (turnId) { - setMessages((prev) => - applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) - ); - } - break; - } - - case "data-assistant-message-id": { - // Resume always allocates a fresh ``new_chat_messages`` - // row anchored to a new ``turn_id`` (the original - // interrupted turn's row stays as-is), so this is a - // real id swap. Rename the optimistic placeholder to - // ``msg-{db_id}`` and reassign closure state. Resume - // does NOT emit ``data-user-message-id`` — the user - // row belongs to the original interrupted turn. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newAssistantMsgId = `msg-${parsedMsg.messageId}`; - const oldAssistantMsgId = assistantMsgId; - tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); - setMessages((prev) => - prev.map((m) => - m.id === oldAssistantMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) - : m - ) - ); - assistantMsgId = newAssistantMsgId; - break; - } - } - }); - - batcher.flush(); - - // Server-authoritative persistence: ``stream_resume_chat`` - // finalises the assistant row in - // ``finalize_assistant_turn`` from a shielded - // ``finally`` block (covers both happy-path and - // abort-mid-stream). FE has no remaining persistence - // work here. - } catch (error) { - streamBatcher?.dispose(); - await handleStreamTerminalError({ - error, - flow: "resume", - threadId: resumeThreadId, - assistantMsgId, - accepted: resumeAccepted, - }); - } finally { - setIsRunning(false); - abortControllerRef.current = null; - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.messages(resumeThreadId), - }); - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.detail(resumeThreadId), - }); - } + window.dispatchEvent(new CustomEvent("hitl-decision", { detail: { decisions: ordered } })); }, - [ - pendingInterrupts, - messages, - workspaceId, - localFilesystemEnabled, - disabledTools, - queryClient, - tokenUsageStore, - fetchWithTurnCancellingRetry, - handleStreamTerminalError, - ] + [pendingInterrupts] ); + const handleEditDialogChoice = useCallback( + async (choice: EditMessageDialogChoice) => { + const pending = editDialogState; + if (!pending) return; + setEditDialogState(null); + if (choice === "cancel") return; + await regenerateChat( + buildCtx(), + pending.userQuery, + { + userMessageContent: pending.userMessageContent, + userImages: pending.userImages, + sourceUserMessageId: `msg-${pending.fromMessageId}`, + }, + { + fromMessageId: pending.fromMessageId, + revertActions: choice === "revert", + } + ); + }, + [editDialogState, buildCtx] + ); + + // Handle reloading/refreshing the last AI response + const onReload = useCallback(async () => { + await regenerateChat(buildCtx(), null); + }, [buildCtx]); + + // HITL resume bridge. Submit always happens from this page's approval UI, so + // the currently-viewed thread owns the pending interrupts. Applies each + // decision to its card, then resumes the (durable) stream. useEffect(() => { const handler = (e: Event) => { const detail = (e as CustomEvent).detail as { @@ -1859,15 +679,9 @@ export default function NewChatPage() { if (!detail?.decisions || pendingInterrupts.length === 0) return; const incoming = detail.decisions; if (incoming.length === 0) return; - // Concatenated tool-call ids across every pending card, in the - // order ``handleApprovalSubmit`` produced ``incoming``. const tcIds = pendingInterrupts.flatMap((p) => p.bundleToolCallIds); const N = tcIds.length; - // Refuse rather than silently broadcast or drop. The orchestrator - // only fires ``hitl-decision`` once every pending card has - // submitted, so a count mismatch indicates a contract drift - // (and would later make the backend slicer raise). if (incoming.length !== N) { toast.error( `Cannot resume: ${incoming.length} decision(s) submitted for ${N} pending actions.` @@ -1890,631 +704,71 @@ export default function NewChatPage() { submittedDecisions.push(decision); } - // All pending cards belong to the same assistant message, so a - // single content-update pass suffices. const targetAssistantMsgId = pendingInterrupts[0].assistantMsgId; - setMessages((prev) => - prev.map((m) => { - if (m.id !== targetAssistantMsgId) return m; - const parts = m.content as unknown as Array>; - const newContent = parts.map((part) => { - const tcId = part.toolCallId as string | undefined; - const d = tcId ? byTcId.get(tcId) : undefined; - if (!d || part.type !== "tool-call") return part; - if (typeof part.result !== "object" || part.result === null) return part; - if (!("__interrupt__" in (part.result as Record))) return part; - const decided = d.type; - if (decided === "edit" && d.edited_action) { + if (activeThreadId != null) { + chatStreamStore.setMessages(activeThreadId, (prev) => + prev.map((m) => { + if (m.id !== targetAssistantMsgId) return m; + const parts = m.content as unknown as Array>; + const newContent = parts.map((part) => { + const tcId = part.toolCallId as string | undefined; + const d = tcId ? byTcId.get(tcId) : undefined; + if (!d || part.type !== "tool-call") return part; + if (typeof part.result !== "object" || part.result === null) return part; + if (!("__interrupt__" in (part.result as Record))) return part; + const decided = d.type; + if (decided === "edit" && d.edited_action) { + return { + ...part, + args: d.edited_action.args, + argsText: JSON.stringify(d.edited_action.args, null, 2), + result: { + ...(part.result as Record), + __decided__: decided, + }, + }; + } return { ...part, - args: d.edited_action.args, - // Sync argsText so the card renders the edited - // inputs (assistant-ui prefers it over JSON.stringify). - argsText: JSON.stringify(d.edited_action.args, null, 2), result: { ...(part.result as Record), __decided__: decided, }, }; - } - return { - ...part, - result: { - ...(part.result as Record), - __decided__: decided, - }, - }; - }); - return { ...m, content: newContent as unknown as ThreadMessageLike["content"] }; - }) - ); - handleResume(submittedDecisions); + }); + return { ...m, content: newContent as unknown as ThreadMessageLike["content"] }; + }) + ); + } + void resumeChat(buildCtx(), submittedDecisions); }; window.addEventListener("hitl-decision", handler); return () => window.removeEventListener("hitl-decision", handler); - }, [handleResume, pendingInterrupts]); - - // Convert message (pass through since already in correct format) - const convertMessage = useCallback( - (message: ThreadMessageLike): ThreadMessageLike => message, - [] - ); - - /** - * Handle regeneration (edit or reload) by calling the regenerate endpoint - * and streaming the response. This rewinds the LangGraph checkpointer state. - * - * @param newUserQuery - `null` = reload with same turn from the server. A string = edit - * (including an empty string when the edited turn is images-only); pass `editExtras` for images/content. - */ - const handleRegenerate = useCallback( - async ( - newUserQuery: string | null, - editExtras?: { - userMessageContent: ThreadMessageLike["content"]; - userImages: NewChatUserImagePayload[]; - sourceUserMessageId?: string; - }, - editFromPosition?: { - /** Message id (numeric, parsed from ``msg-``) to rewind to. */ - fromMessageId?: number | null; - /** When true, revert reversible downstream actions before stream. */ - revertActions?: boolean; - } - ) => { - if (!threadId) { - toast.error("Cannot regenerate: no active chat thread"); - return; - } - - const isEdit = newUserQuery !== null; - - // Abort any previous streaming request - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - abortControllerRef.current = null; - } - - // Extract the original user query BEFORE removing messages (for reload mode) - let userQueryToDisplay: string | undefined; - let originalUserMessageContent: ThreadMessageLike["content"] | null = null; - let originalUserMessageMetadata: ThreadMessageLike["metadata"] | undefined; - let sourceUserMessageId: string | undefined = editExtras?.sourceUserMessageId; - - if (!isEdit) { - // Reload mode - find and preserve the last user message content - const lastUserMessage = [...messages].reverse().find((m) => m.role === "user"); - if (lastUserMessage) { - sourceUserMessageId = lastUserMessage.id; - originalUserMessageContent = lastUserMessage.content; - originalUserMessageMetadata = lastUserMessage.metadata; - // Extract text for the API request - for (const part of lastUserMessage.content) { - if (typeof part === "object" && part.type === "text" && "text" in part) { - userQueryToDisplay = part.text; - break; - } - } - } - } else { - userQueryToDisplay = newUserQuery; - } - - // Start streaming - setIsRunning(true); - const controller = new AbortController(); - abortControllerRef.current = controller; - - // Add placeholder user message if we have a new query (edit mode). - // Mutable for the same reason as in ``onNew`` — both ids are - // renamed mid-stream by the new ``data-user-message-id`` / - // ``data-assistant-message-id`` SSE handlers below. - let userMsgId = `msg-user-${Date.now()}`; - let assistantMsgId = `msg-assistant-${Date.now()}`; - const currentThinkingSteps = new Map(); - - const contentPartsState: ContentPartsState = { - contentParts: [], - currentTextPartIndex: -1, - currentReasoningPartIndex: -1, - toolCallIndices: new Map(), - }; - const { contentParts } = contentPartsState; - let regenerateAccepted = false; - let streamBatcher: FrameBatchedUpdater | null = null; - - // Add placeholder messages to UI - // Always add back the user message (with new query for edit, or original content for reload) - const userMessage: ThreadMessageLike = { - id: userMsgId, - role: "user", - content: isEdit - ? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }]) - : originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }], - createdAt: new Date(), - metadata: isEdit ? undefined : originalUserMessageMetadata, - }; - const sourceMentionedDocs = - sourceUserMessageId && messageDocumentsMap[sourceUserMessageId] - ? messageDocumentsMap[sourceUserMessageId] - : []; - try { - const selection = await getAgentFilesystemSelection(workspaceId, { - localFilesystemEnabled, - }); - // Partition the source mentions back into doc/folder id buckets - // so the regenerate route can pass them to ``stream_new_chat`` - // and the priority middleware sees the same ``[USER-MENTIONED]`` - // priority entries the original turn did. Without this partition - // the regenerate flow silently dropped the agent's mention - // awareness — same architectural bug we fixed on the new-chat path. - const regenerateDocIds = sourceMentionedDocs - .filter((d) => d.kind === "doc") - .map((d) => d.id); - const regenerateFolderIds = sourceMentionedDocs - .filter((d) => d.kind === "folder") - .map((d) => d.id); - const regenerateConnectors = sourceMentionedDocs.filter((d) => d.kind === "connector"); - const regenerateThreadIds = sourceMentionedDocs - .filter((d) => d.kind === "thread") - .map((d) => d.id); - - const requestBody: Record = { - workspace_id: workspaceId, - user_query: newUserQuery, - disabled_tools: disabledTools.length > 0 ? disabledTools : undefined, - filesystem_mode: selection.filesystem_mode, - client_platform: selection.client_platform, - local_filesystem_mounts: selection.local_filesystem_mounts, - mentioned_document_ids: regenerateDocIds.length > 0 ? regenerateDocIds : undefined, - mentioned_folder_ids: regenerateFolderIds.length > 0 ? regenerateFolderIds : undefined, - mentioned_connector_ids: - regenerateConnectors.length > 0 ? regenerateConnectors.map((d) => d.id) : undefined, - mentioned_connectors: regenerateConnectors.length > 0 ? regenerateConnectors : undefined, - mentioned_thread_ids: regenerateThreadIds.length > 0 ? regenerateThreadIds : undefined, - // Full mention metadata for the regenerate-specific - // source list. Only meaningful for edit (the BE only - // re-persists a user row when ``user_query`` is set); - // reload reuses the original turn's mentioned_documents. - mentioned_documents: sourceMentionedDocs.length > 0 ? sourceMentionedDocs : undefined, - }; - if (isEdit) { - requestBody.user_images = editExtras?.userImages ?? []; - } - // Explicit edit-from-arbitrary-position. Only send - // ``from_message_id`` / ``revert_actions`` when the - // caller asked for them; otherwise the backend keeps the - // legacy "last 2 messages" behaviour for back-compat. - if (editFromPosition?.fromMessageId != null) { - requestBody.from_message_id = editFromPosition.fromMessageId; - if (editFromPosition.revertActions) { - requestBody.revert_actions = true; - } - } - const response = await fetchWithTurnCancellingRetry(() => - authenticatedFetch(getRegenerateUrl(threadId), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(requestBody), - signal: controller.signal, - }) - ); - - if (!response.ok) { - throw await toHttpResponseError(response); - } - regenerateAccepted = true; - - // Only switch UI to regenerated placeholder messages after the backend accepts - // regenerate. This avoids local message loss when regenerate fails early (e.g. 400). - // - // When an explicit ``editFromPosition.fromMessageId`` is passed, slice from - // that message forward so edit-from-arbitrary-position drops every downstream - // message; otherwise fall back to the legacy "drop the last 2" behaviour. - setMessages((prev) => { - let base = prev; - if (editFromPosition?.fromMessageId != null) { - const targetId = `msg-${editFromPosition.fromMessageId}`; - const sliceIndex = prev.findIndex((m) => m.id === targetId); - if (sliceIndex >= 0) { - base = prev.slice(0, sliceIndex); - } - } else if (prev.length >= 2) { - base = prev.slice(0, -2); - } - return [ - ...base, - userMessage, - { - id: assistantMsgId, - role: "assistant", - content: [{ type: "text", text: "" }], - createdAt: new Date(), - }, - ]; - }); - if (sourceMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => ({ - ...prev, - [userMsgId]: sourceMentionedDocs, - })); - } - - const flushMessages = () => { - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId - ? { ...m, content: buildContentForUI(contentPartsState, toolsWithUI) } - : m - ) - ); - }; - const { batcher, scheduleFlush, forceFlush } = createStreamFlushHelpers(flushMessages); - streamBatcher = batcher; - - await consumeSseEvents(response, async (parsed) => { - if ( - processSharedStreamEvent(parsed, { - contentPartsState, - toolsWithUI, - currentThinkingSteps, - scheduleFlush, - forceFlush, - onTokenUsage: (data) => { - tokenUsageStore.set(assistantMsgId, data); - }, - onTurnStatus: (data) => { - if (data.status === "cancelling") { - recentCancelRequestedAtRef.current = Date.now(); - } - }, - }) - ) { - return; - } - switch (parsed.type) { - case "data-action-log": { - if (threadId !== null) { - applyActionLogSse(queryClient, threadId, workspaceId, parsed.data); - } - break; - } - - case "data-action-log-updated": { - if (threadId !== null) { - applyActionLogUpdatedSse( - queryClient, - threadId, - parsed.data.id, - parsed.data.reversible - ); - } - break; - } - - case "data-turn-info": { - const turnId = readStreamedChatTurnId(parsed.data); - if (turnId) { - setMessages((prev) => - applyTurnIdToAssistantMessageList(prev, assistantMsgId, turnId) - ); - } - break; - } - - case "data-user-message-id": { - // Same role as in ``onNew`` but the regenerate-specific - // mention metadata (``sourceMentionedDocs``) is the - // list to migrate onto the canonical id key. - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newUserMsgId = `msg-${parsedMsg.messageId}`; - const oldUserMsgId = userMsgId; - setMessages((prev) => - prev.map((m) => - m.id === oldUserMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newUserMsgId }, parsedMsg.turnId) - : m - ) - ); - if (sourceMentionedDocs.length > 0) { - setMessageDocumentsMap((prev) => { - if (!(oldUserMsgId in prev)) { - return { ...prev, [newUserMsgId]: sourceMentionedDocs }; - } - const { [oldUserMsgId]: _removed, ...rest } = prev; - return { ...rest, [newUserMsgId]: sourceMentionedDocs }; - }); - } - userMsgId = newUserMsgId; - break; - } - - case "data-assistant-message-id": { - const parsedMsg = readStreamedMessageId(parsed.data); - if (!parsedMsg) break; - const newAssistantMsgId = `msg-${parsedMsg.messageId}`; - const oldAssistantMsgId = assistantMsgId; - tokenUsageStore.rename(oldAssistantMsgId, newAssistantMsgId); - setMessages((prev) => - prev.map((m) => - m.id === oldAssistantMsgId - ? mergeChatTurnIdIntoMessage({ ...m, id: newAssistantMsgId }, parsedMsg.turnId) - : m - ) - ); - assistantMsgId = newAssistantMsgId; - break; - } - - case "data-revert-results": { - const summary = parsed.data; - // failureCount must include every "not undone" bucket - // (not_reversible, permission_denied, failed) so the - // toast's "X could not be rolled back" math matches - // the response invariant ``total === sum(counters)``. - // ``skipped`` rows are batch revert artefacts (revert - // rows themselves) and are not user-facing failures. - const failureCount = - summary.failed + summary.not_reversible + (summary.permission_denied ?? 0); - if (failureCount > 0) { - toast.warning( - `Pre-revert: ${summary.reverted}/${summary.total} undone, ${failureCount} could not be rolled back.` - ); - } else if (summary.reverted > 0) { - toast.success( - summary.reverted === 1 - ? "Reverted 1 downstream action before regenerating." - : `Reverted ${summary.reverted} downstream actions before regenerating.` - ); - } - if (threadId !== null) { - for (const r of summary.results) { - if (r.status === "reverted" || r.status === "already_reverted") { - markActionRevertedInCache( - queryClient, - threadId, - r.action_id, - r.new_action_id ?? null - ); - } - } - } - break; - } - } - }); - - batcher.flush(); - - // Server-authoritative persistence: ``stream_new_chat`` - // (regenerate flow) wrote the user row in - // ``persist_user_turn`` and finalises the assistant row - // in ``finalize_assistant_turn`` from a shielded - // ``finally`` block (covers both happy-path and - // abort-mid-stream). FE only needs to track the - // successful response here. - if (contentParts.length > 0) { - trackChatResponseReceived(workspaceId, threadId); - } - } catch (error) { - streamBatcher?.dispose(); - await handleStreamTerminalError({ - error, - flow: "regenerate", - threadId, - assistantMsgId, - accepted: regenerateAccepted, - }); - } finally { - setIsRunning(false); - abortControllerRef.current = null; - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.messages(threadId), - }); - void queryClient.invalidateQueries({ - queryKey: cacheKeys.threads.detail(threadId), - }); - } - }, - [ - threadId, - workspaceId, - messages, - disabledTools, - localFilesystemEnabled, - messageDocumentsMap, - setMessageDocumentsMap, - queryClient, - tokenUsageStore, - fetchWithTurnCancellingRetry, - handleStreamTerminalError, - ] - ); - - // Handle editing a message - truncates history and regenerates with new query. - // - // When ``message.sourceId`` is set (the assistant-ui way to say - // "this edit replaces an older message"), we pin - // ``from_message_id`` so the backend rewinds to the right LangGraph - // checkpoint instead of relying on the legacy "last 2 messages" - // rewind. We also count downstream reversible actions and prompt the - // user to revert / continue / cancel before regenerating. - const onEdit = useCallback( - async (message: AppendMessage) => { - const { userQuery, userImages } = extractUserTurnForNewChatApi(message, []); - const queryForApi = userQuery.trim(); - if (!queryForApi && userImages.length === 0) { - toast.error("Cannot edit with empty message"); - return; - } - - const userMessageContent = message.content as unknown as ThreadMessageLike["content"]; - - // ``sourceId`` per @assistant-ui/core's ``AppendMessage`` is - // "the ID of the message that was edited". Parse the numeric - // suffix so we can map it back to a DB row. - const sourceId = (message as { sourceId?: string }).sourceId; - const fromMessageId = - sourceId && /^msg-\d+$/.test(sourceId) - ? Number.parseInt(sourceId.replace(/^msg-/, ""), 10) - : null; - - if (fromMessageId == null) { - // No source id (or non-DB id) — fall back to today's - // last-2 behaviour. The user gets the legacy edit flow. - await handleRegenerate(queryForApi, { - userMessageContent, - userImages, - sourceUserMessageId: sourceId, - }); - return; - } - - // Pre-flight: count reversible downstream actions so we can - // auto-skip the dialog for harmless edits. - // - // "Downstream" means messages AFTER the edited one. The - // previous slice ``messages.slice(editedIndex)`` included - // the edited message itself in both the total - // count and the reversibility scan (any actions on the - // edited turn would be double-counted). Slice from - // ``editedIndex + 1`` so the dialog text matches reality: - // "N downstream messages will be dropped". - const editedIndex = messages.findIndex((m) => m.id === `msg-${fromMessageId}`); - let downstreamReversibleCount = 0; - let downstreamTotalCount = 0; - if (editedIndex >= 0) { - const downstream = messages.slice(editedIndex + 1); - downstreamTotalCount = downstream.length; - const seenTurns = new Set(); - const downstreamTurnIds = new Set(); - for (const m of downstream) { - const meta = (m.metadata ?? {}) as { custom?: { chatTurnId?: string } }; - const tid = meta.custom?.chatTurnId; - if (!tid || seenTurns.has(tid)) continue; - seenTurns.add(tid); - downstreamTurnIds.add(tid); - } - // Source of truth: the unified react-query cache. Every - // action whose ``chat_turn_id`` belongs to the slice we're - // about to drop counts toward the prompt. - for (const a of agentActionItems) { - if (!a.chat_turn_id || !downstreamTurnIds.has(a.chat_turn_id)) continue; - if ( - a.reversible && - (a.reverted_by_action_id === null || a.reverted_by_action_id === undefined) && - !a.is_revert_action && - (a.error === null || a.error === undefined) - ) { - downstreamReversibleCount += 1; - } - } - } - - if (downstreamReversibleCount === 0) { - // Nothing to revert — submit silently. - await handleRegenerate( - queryForApi, - { userMessageContent, userImages, sourceUserMessageId: sourceId }, - { fromMessageId, revertActions: false } - ); - return; - } - - setEditDialogState({ - fromMessageId, - userQuery: queryForApi, - userMessageContent, - userImages, - downstreamReversibleCount, - downstreamTotalCount, - }); - }, - [handleRegenerate, messages, agentActionItems] - ); - - const handleApprovalSubmit = useCallback( - (interruptId: string, decisions: HitlDecision[]) => { - // Stage this card's decisions; only fire the resume once every - // pending card in the current turn has submitted, so the - // backend slicer sees a single concatenated decisions list - // whose total matches the parent state's pending action count. - stagedDecisionsByInterruptIdRef.current.set(interruptId, decisions); - if (stagedDecisionsByInterruptIdRef.current.size < pendingInterrupts.length) { - return; - } - const ordered: HitlDecision[] = []; - for (const pi of pendingInterrupts) { - const staged = stagedDecisionsByInterruptIdRef.current.get(pi.interruptId); - if (!staged) { - // Defensive: a missing entry means the staging map and - // the pending list disagreed for one cycle. Bail rather - // than dispatch a count-mismatched batch. - return; - } - ordered.push(...staged); - } - stagedDecisionsByInterruptIdRef.current.clear(); - window.dispatchEvent(new CustomEvent("hitl-decision", { detail: { decisions: ordered } })); - }, - [pendingInterrupts] - ); - - const handleEditDialogChoice = useCallback( - async (choice: EditMessageDialogChoice) => { - const pending = editDialogState; - if (!pending) return; - setEditDialogState(null); - if (choice === "cancel") return; - await handleRegenerate( - pending.userQuery, - { - userMessageContent: pending.userMessageContent, - userImages: pending.userImages, - sourceUserMessageId: `msg-${pending.fromMessageId}`, - }, - { - fromMessageId: pending.fromMessageId, - revertActions: choice === "revert", - } - ); - }, - [editDialogState, handleRegenerate] - ); - - // Handle reloading/refreshing the last AI response - const onReload = useCallback(async () => { - // parentId is the ID of the message to reload from (the user message) - // We call regenerate without a query to use the same query - await handleRegenerate(null); - }, [handleRegenerate]); + }, [buildCtx, pendingInterrupts, activeThreadId]); // Surface the thread's deliverables to the layout-level artifacts sidebar. - useSyncChatArtifacts(messages); + useSyncChatArtifacts(displayMessages); // Create external store runtime const runtime = useExternalStoreRuntime({ - messages, + messages: displayMessages, isRunning, onNew, onEdit, onReload, convertMessage, - onCancel: cancelRun, + onCancel, }); const threadLoadError = activeThreadId ? (threadDetailQuery.error ?? threadMessagesQuery.error) : null; const shouldShowThreadLoadError = - !!threadLoadError && !!activeThreadId && !currentThread && messages.length === 0; + !!threadLoadError && !!activeThreadId && !currentThread && displayMessages.length === 0; const isThreadMessagesLoading = !!activeThreadId && threadMessagesQuery.isPending && - messages.length === 0 && + displayMessages.length === 0 && !threadMessagesQuery.error; if (shouldShowThreadLoadError) { From 9879306ccc4eafb32ad8e02ac7396b912e2672c9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:39:28 +0200 Subject: [PATCH 13/28] docs(chat): tighten ActiveChatStreamRunner comment to intent --- .../chat/active-chat-stream-runner.tsx | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/surfsense_web/components/chat/active-chat-stream-runner.tsx b/surfsense_web/components/chat/active-chat-stream-runner.tsx index da86c0aa8..dd27922a0 100644 --- a/surfsense_web/components/chat/active-chat-stream-runner.tsx +++ b/surfsense_web/components/chat/active-chat-stream-runner.tsx @@ -4,19 +4,13 @@ import { useEffect } from "react"; import { chatStreamStore } from "@/lib/chat/stream-engine/store"; /** - * Persistent, render-null host for app-wide chat-stream lifecycle. + * Persistent, render-null host that scopes the in-flight chat turn's lifetime + * to the workspace shell, not the chat page. * - * Mounted in the workspace shell (``LayoutDataProvider``), it persists across - * in-app navigation between workspace routes (``/new-chat`` -> ``/chats`` -> - * ``/automations`` and doc-tab switches) and only unmounts on real teardown - * (workspace change / app teardown). On that teardown it aborts the single - * in-flight turn — this replaces the old chat-page unmount abort, which was - * what killed streams on ordinary navigation. - * - * NOTE: the ``hitl-decision`` bridge and ``useChatSessionStateSync`` stay in - * the chat page: both need the currently-viewed thread id (HITL resume can - * only be triggered from the page's approval UI), which the shell-level runner - * does not cleanly have. + * Mounted in ``LayoutDataProvider``, it survives in-app navigation between + * workspace routes and aborts the single active turn only on workspace/app + * teardown, so ordinary navigation disconnects the view without stopping the + * stream. */ export function ActiveChatStreamRunner() { useEffect(() => { From a44c67da978720cabd8ed873bf840edf408a8e70 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 22:39:29 +0200 Subject: [PATCH 14/28] docs(chat): tighten engine comments to intent --- surfsense_web/lib/chat/stream-engine/engine.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/surfsense_web/lib/chat/stream-engine/engine.ts b/surfsense_web/lib/chat/stream-engine/engine.ts index a4a0c02a7..57ff47f61 100644 --- a/surfsense_web/lib/chat/stream-engine/engine.ts +++ b/surfsense_web/lib/chat/stream-engine/engine.ts @@ -80,10 +80,9 @@ const tokenUsageStore = chatStreamStore.tokenUsage; const toolsWithUI = TOOLS_WITH_UI_ALL; /** - * Display-only setters the page provides while it is mounted. After the chat - * page unmounts (in-app navigation) these are stale but harmless — the stream - * keeps writing to the module {@link chatStreamStore}, and the page re-derives - * its thread from the URL when it remounts. + * Display-only setters, valid only while the page is mounted. The stream drives + * durable state through {@link chatStreamStore}; these just sync the mounted + * page's local view and are ignored once it unmounts. */ export interface EngineView { setThreadId: Dispatch>; @@ -101,7 +100,7 @@ export interface EngineContext { } // --------------------------------------------------------------------------- -// Error handling (relocated from the chat page) +// Error handling // --------------------------------------------------------------------------- async function persistAssistantErrorMessage({ From f0cd20bcbd3afda0cb024bb067cd630315e15cb7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 15/28] feat(chat): add showMessageTimestamps preference atom (localStorage, off by default) --- surfsense_web/atoms/chat/show-timestamps.atom.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 surfsense_web/atoms/chat/show-timestamps.atom.ts diff --git a/surfsense_web/atoms/chat/show-timestamps.atom.ts b/surfsense_web/atoms/chat/show-timestamps.atom.ts new file mode 100644 index 000000000..434f065ff --- /dev/null +++ b/surfsense_web/atoms/chat/show-timestamps.atom.ts @@ -0,0 +1,11 @@ +import { atomWithStorage } from "jotai/utils"; + +/** + * Per-device preference: show a timestamp under each chat message. + * + * Off by default to match streaming-AI chat convention (ChatGPT/Claude keep + * the message stream clean and put time in the conversation list). Persisted + * in localStorage, so it does not sync across devices — acceptable for a + * cosmetic display toggle. + */ +export const showMessageTimestampsAtom = atomWithStorage("chat-show-timestamps:v1", false); From f4870ad1c20b8cd138a2ef20bce137ef317d1fbd Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 16/28] feat(chat): add shared formatMessageTimestamp helper --- surfsense_web/lib/format-date.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/surfsense_web/lib/format-date.ts b/surfsense_web/lib/format-date.ts index c2f445537..a062b08fa 100644 --- a/surfsense_web/lib/format-date.ts +++ b/surfsense_web/lib/format-date.ts @@ -68,3 +68,17 @@ export function formatRelativeFutureDate(dateString: string): string { export function formatThreadTimestamp(dateString: string): string { return format(new Date(dateString), "MMM d, yyyy 'at' h:mm a"); } + +/** + * Format a chat message timestamp for inline display under a bubble. + * Locale-aware, 12-hour clock. Example: "Jul 13, 10:42 PM". + */ +export function formatMessageTimestamp(date: Date): string { + return date.toLocaleDateString(undefined, { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + hour12: true, + }); +} From 00fbc24e4bb91d217a0da1c616ac23a58366b501 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 17/28] feat(chat): add MessageTimestamp component gated on the preference --- .../assistant-ui/message-timestamp.tsx | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 surfsense_web/components/assistant-ui/message-timestamp.tsx diff --git a/surfsense_web/components/assistant-ui/message-timestamp.tsx b/surfsense_web/components/assistant-ui/message-timestamp.tsx new file mode 100644 index 000000000..65a1da9be --- /dev/null +++ b/surfsense_web/components/assistant-ui/message-timestamp.tsx @@ -0,0 +1,24 @@ +import { useAuiState } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import type { FC } from "react"; +import { showMessageTimestampsAtom } from "@/atoms/chat/show-timestamps.atom"; +import { formatMessageTimestamp } from "@/lib/format-date"; +import { cn } from "@/lib/utils"; + +/** + * Muted, always-visible timestamp under a chat message. Renders only when the + * user has opted in via {@link showMessageTimestampsAtom} and the message + * carries a ``createdAt`` (absent on optimistic pre-persist messages). + */ +export const MessageTimestamp: FC<{ className?: string }> = ({ className }) => { + const show = useAtomValue(showMessageTimestampsAtom); + const createdAt = useAuiState(({ message }) => message?.createdAt); + + if (!show || !createdAt) return null; + + return ( +
+ {formatMessageTimestamp(createdAt)} +
+ ); +}; From 0093664fd10c6e1e0b0d9d490c9011d20189307c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:06 +0200 Subject: [PATCH 18/28] feat(chat): render timestamp under user messages --- surfsense_web/components/assistant-ui/user-message.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_web/components/assistant-ui/user-message.tsx b/surfsense_web/components/assistant-ui/user-message.tsx index 592dc4224..09a700ff1 100644 --- a/surfsense_web/components/assistant-ui/user-message.tsx +++ b/surfsense_web/components/assistant-ui/user-message.tsx @@ -22,6 +22,7 @@ import { currentThreadAtom } from "@/atoms/chat/current-thread.atom"; import { messageDocumentsMapAtom } from "@/atoms/chat/mentioned-documents.atom"; import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { MentionChip } from "@/components/assistant-ui/mention-chip"; +import { MessageTimestamp } from "@/components/assistant-ui/message-timestamp"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; @@ -182,6 +183,7 @@ export const UserMessage: FC = () => {
)}
+
); From 855f82409712f3b54369b5f63e7b58690403f2c6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 19/28] feat(chat): render timestamp under assistant messages and reuse shared formatter --- .../assistant-ui/assistant-message.tsx | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index c4a0e86dc..a45c651b1 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -35,6 +35,7 @@ import { useAllCitationMetadata, } from "@/components/assistant-ui/citation-metadata-context"; import { MarkdownText } from "@/components/assistant-ui/markdown-text"; +import { MessageTimestamp } from "@/components/assistant-ui/message-timestamp"; import { ReasoningMessagePart } from "@/components/assistant-ui/reasoning-message-part"; import { RevertTurnButton } from "@/components/assistant-ui/revert-turn-button"; import { @@ -62,6 +63,7 @@ import { withArtifactAnchor } from "@/features/chat-artifacts"; import { useComments } from "@/hooks/use-comments"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; +import { formatMessageTimestamp } from "@/lib/format-date"; import { getProviderIcon } from "@/lib/provider-icons"; import { tryGetHostname } from "@/lib/url"; import { cn } from "@/lib/utils"; @@ -249,16 +251,6 @@ export const MessageError: FC = () => { ); }; -function formatMessageDate(date: Date): string { - return date.toLocaleDateString(undefined, { - month: "short", - day: "numeric", - hour: "numeric", - minute: "2-digit", - hour12: true, - }); -} - /** * Format provider USD cost (in micro-USD) for inline display next to a * token count. Falls back to ``"<$0.001"`` for sub-tenth-of-a-cent @@ -367,7 +359,7 @@ const MessageInfoDropdown: FC<{ chatTurnId: string | null | undefined }> = ({ ch > {createdAt && ( - {formatMessageDate(createdAt)} + {formatMessageTimestamp(createdAt)} )} {hasUsage && ( @@ -463,6 +455,8 @@ const AssistantMessageInner: FC = () => { + + {isMobile && (
From 4b2f009233b06cdfb190d0fc06379dab9a9e5560 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 20/28] feat(settings): add Appearance content with show-timestamps toggle --- .../components/AppearanceContent.tsx | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx b/surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx new file mode 100644 index 000000000..258a31dfe --- /dev/null +++ b/surfsense_web/app/dashboard/[workspace_id]/user-settings/components/AppearanceContent.tsx @@ -0,0 +1,43 @@ +"use client"; + +import { useAtom } from "jotai"; +import { showMessageTimestampsAtom } from "@/atoms/chat/show-timestamps.atom"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; + +export function AppearanceContent() { + const [showTimestamps, setShowTimestamps] = useAtom(showMessageTimestampsAtom); + + return ( +
+
+
+

Chat

+

+ Control how messages are displayed in your conversations. +

+
+
+
+
+ +

+ Display the time under each message in a chat. Saved on this device. +

+
+ +
+
+
+
+ ); +} From b3999a754e06e4a385fc3e441b7411e7113e70c2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 21/28] feat(settings): add Appearance user-settings page --- .../[workspace_id]/user-settings/appearance/page.tsx | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx new file mode 100644 index 000000000..6ae0952d5 --- /dev/null +++ b/surfsense_web/app/dashboard/[workspace_id]/user-settings/appearance/page.tsx @@ -0,0 +1,5 @@ +import { AppearanceContent } from "../components/AppearanceContent"; + +export default function Page() { + return ; +} From e30a7afaf167accbb6ce1fc3318af5246d33ee6f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 13 Jul 2026 23:12:07 +0200 Subject: [PATCH 22/28] feat(settings): register Appearance tab in user-settings nav --- .../[workspace_id]/user-settings/layout-shell.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx b/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx index aa73917ef..b18db9e64 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/user-settings/layout-shell.tsx @@ -7,6 +7,7 @@ import { Library, MessageCircle, Monitor, + Palette, ReceiptText, ShieldCheck, WandSparkles, @@ -20,6 +21,7 @@ import { usePlatform } from "@/hooks/use-platform"; export type UserSettingsTab = | "profile" + | "appearance" | "api-key" | "prompts" | "community-prompts" @@ -49,6 +51,12 @@ export function UserSettingsLayoutShell({ workspaceId, children }: UserSettingsL href: `/dashboard/${workspaceId}/user-settings/profile`, icon: , }, + { + value: "appearance" as const, + label: "Appearance", + href: `/dashboard/${workspaceId}/user-settings/appearance`, + icon: , + }, { value: "api-key" as const, label: t("api_key_nav_label"), From f70b83582fa813a812aff3062d4117f0bb4f7682 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 14:18:15 -0700 Subject: [PATCH 23/28] Update README files in multiple languages to reflect the rebranding of SurfSense as NotebookLM for competitive intelligence research, emphasizing live scraping capabilities and clarifying the platform's features. --- README.es.md | 4 ++-- README.hi.md | 4 ++-- README.md | 4 ++-- README.pt-BR.md | 4 ++-- README.zh-CN.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.es.md b/README.es.md index 70ccd5eb1..ecdd93c41 100644 --- a/README.es.md +++ b/README.es.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift
-# SurfSense: Dale inteligencia competitiva a tus agentes de IA +# SurfSense: NotebookLM para investigación de inteligencia competitiva -SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. +SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. > [!NOTE] > **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM** diff --git a/README.hi.md b/README.hi.md index cbf9ae8fa..3cb769318 100644 --- a/README.hi.md +++ b/README.hi.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift -# SurfSense: अपने AI एजेंट्स को दें कॉम्पिटिटिव इंटेलिजेंस +# SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM -SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। +SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। > [!NOTE] > **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना** diff --git a/README.md b/README.md index 8f2e65118..eac392ebd 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift -# SurfSense: Give Your AI Agents Competitive Intelligence +# SurfSense: NotebookLM for Competitive Intelligence Research -SurfSense is the **open-source competitive intelligence platform for AI agents**. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. +SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. > [!NOTE] > **📢 A note for our NotebookLM-alternative users** diff --git a/README.pt-BR.md b/README.pt-BR.md index 076736bcf..e32882831 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift -# SurfSense: Dê Inteligência Competitiva aos Seus Agentes de IA +# SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva -O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. +O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. > [!NOTE] > **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM** diff --git a/README.zh-CN.md b/README.zh-CN.md index 7d2ae35c3..11bace243 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -20,9 +20,9 @@ MODSetter%2FSurfSense | Trendshift -# SurfSense：为你的 AI 智能体注入竞争情报能力 +# SurfSense：面向竞争情报研究的 NotebookLM -SurfSense 是**面向 AI 智能体的开源竞争情报平台**。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 +SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 > [!NOTE] > **📢 致我们的 NotebookLM 替代品用户** From ab5970fab11939ec0e4d9b258a9e9b10ccbbe3c1 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 14:42:11 -0700 Subject: [PATCH 24/28] feat: updating messaging of instagram and tiktok pages --- surfsense_web/lib/auth-utils.ts | 1 + .../lib/connectors-marketing/instagram.tsx | 47 +++++++----- .../lib/connectors-marketing/tiktok.tsx | 76 +++++++++++-------- 3 files changed, 72 insertions(+), 52 deletions(-) diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 8c8d919c5..7d9320fb0 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -40,6 +40,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/external-mcp-connectors", "/reddit", "/instagram", + "/tiktok", "/youtube", "/google-maps", "/google-search", diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index d22400b7a..5422a0c90 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -6,28 +6,32 @@ export const instagram: ConnectorPageContent = { name: "Instagram", icon: IconBrandInstagram, - metaTitle: "Instagram Scraper API for Creator Research | SurfSense", + metaTitle: "Instagram Scraper API for Profiles and Reels | SurfSense", metaDescription: - "Scrape public Instagram posts, reels, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", + "Instagram scraper API for public profiles, posts, and reels. No login or Graph API review. Structured data for AI agents, plus a free tier. Start now.", keywords: [ "instagram scraper", "instagram scraper api", "instagram api", "instagram api alternative", "scrape instagram", + "instagram scraping", "instagram graph api alternative", "instagram profile scraper", "instagram post scraper", + "instagram posts scraper", "instagram reel scraper", + "instagram reels scraper", + "instagram data scraper", "instagram data api", "instagram mcp server", "creator research", "social listening", ], - h1: "Instagram Scraper API for Creator Research and Social Listening", + h1: "Instagram Scraper API for Profiles, Posts, and Reels", heroLede: - "The SurfSense Instagram API extracts public posts, reels, and profile details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.", + "The SurfSense Instagram scraper extracts public profiles, posts, and reels without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.", transcript: { prompt: "Pull recent reels from @competitor and summarize what they're posting", @@ -54,48 +58,48 @@ export const instagram: ConnectorPageContent = { }, extractIntro: - "Every call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.", + "Every Instagram scraper call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.", extractFields: [ { - label: "Posts & Reels", + label: "Posts and reels", description: - "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.", + "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp from any public post or reel.", }, { - label: "Profiles", + label: "Profile data", description: - "Follower, following, and post counts, bio, external URL, verified and business flags.", + "Follower, following, and post counts, bio, external URL, verified and business flags for public Instagram profiles.", }, { - label: "Owner & Media", + label: "Owner and media", description: "Owner username and id on every item, plus image and video URLs, alt text, and view counts.", }, ], - useCasesHeading: "What teams do with the Instagram API", + useCasesHeading: "What teams do with the Instagram scraper API", useCases: [ { title: "Creator and competitor monitoring", description: - "Track what your competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.", + "Scrape Instagram profiles and feeds to track what competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.", }, { title: "Content and format research", description: - "Study a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.", + "Pull a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.", }, { title: "Influencer vetting and outreach", description: - "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership.", + "Use the Instagram profile scraper to verify follower count, post cadence, and real engagement from public data before you pay for a partnership.", }, ], comparison: { - heading: "An Instagram API alternative built for agents", + heading: "An Instagram Graph API alternative built for agents", intro: - "The official Instagram Graph API requires a Business account, app review, and access tokens, and it can't read arbitrary public profiles. Here is how SurfSense compares.", + "The official Instagram Graph API requires a Business account, app review, and access tokens, and it cannot read arbitrary public profiles. SurfSense is an Instagram API alternative for public data. Here is how it compares.", columnLabel: "Instagram Graph API", rows: [ { @@ -259,7 +263,12 @@ export const instagram: ConnectorPageContent = { { question: "Do I need an Instagram account or the Graph API?", answer: - "No. This is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured data back.", + "No. This Instagram scraper API is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call SurfSense with one key, or add the MCP server to your agent, and get structured profile, post, and reel data back.", + }, + { + question: "What Instagram data can I scrape?", + answer: + "Public profiles, posts, and reels. Each item includes captions, hashtags, mentions, engagement counts, media URLs, and owner metadata. Point the Instagram profile scraper at a handle, or pass post and reel URLs directly. Discover creators with search queries when you do not have a URL yet.", }, { question: "What are the rate limits?", @@ -274,11 +283,11 @@ export const instagram: ConnectorPageContent = { ], related: [ - { label: "Reddit API", href: "/reddit" }, + { label: "TikTok API", href: "/tiktok" }, { label: "YouTube API", href: "/youtube" }, + { label: "Reddit API", href: "/reddit" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, - { label: "Read the docs", href: "/docs" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index d77865633..f245de92b 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -6,30 +6,33 @@ export const tiktok: ConnectorPageContent = { name: "TikTok", icon: IconBrandTiktok, - metaTitle: "TikTok Scraper API for Trend and Creator Research | SurfSense", + metaTitle: "TikTok Scraper API for Videos and Comments | SurfSense", metaDescription: - "Scrape public TikTok videos, comments, accounts, and trending feeds by hashtag, profile, or URL with the SurfSense TikTok Scraper API. No approval process or research-API gatekeeping, plus a free tier. Start now.", + "TikTok scraper API for public videos, comments, hashtags, and profiles. No Research API approval. Structured data for AI agents, plus a free tier. Start now.", keywords: [ "tiktok scraper", "tiktok scraper api", "tiktok api", "tiktok api alternative", "scrape tiktok", + "tiktok scraping", + "tiktok research api alternative", "tiktok data api", - "tiktok hashtag scraper", "tiktok comments scraper", + "tiktok comment scraper", + "tiktok hashtag scraper", + "tiktok profile scraper", + "tiktok video scraper", "tiktok trending scraper", "tiktok user search", - "tiktok trend tracking", "tiktok mcp", "social listening", "influencer research tool", - "short-form video analytics", ], - h1: "TikTok Scraper API for Trend and Creator Research", + h1: "TikTok Scraper API for Videos, Comments, and Trend Research", heroLede: - "The SurfSense TikTok API extracts public videos by hashtag, creator profile, or URL without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", + "The SurfSense TikTok scraper extracts public videos by hashtag, creator profile, or URL, plus comment threads and trending feeds, without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.", transcript: { prompt: "Find trending TikToks about meal prep this week", @@ -55,46 +58,47 @@ export const tiktok: ConnectorPageContent = { }, extractIntro: - "Every call returns structured items. Scrape videos from a hashtag, creator profile, or video URL — or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", + "Every TikTok scraper call returns structured items. Scrape videos from a hashtag, creator profile, or video URL, or switch verbs to pull a video's comments, discover accounts by keyword, or fetch the current trending feed.", extractFields: [ { label: "Videos", - description: "Caption text, canonical web URL, duration, and cover image for each video.", + description: "Caption text, canonical web URL, duration, and cover image for each TikTok video.", }, { label: "Engagement", description: - "Play, like, comment, share, and save counts — the signal for what is breaking out.", + "Play, like, comment, share, and save counts, the signal for what is breaking out.", }, { - label: "Authors", + label: "Authors and profiles", description: "Creator handle, nickname, follower and heart counts, and verified status.", }, + { + label: "Comments", + description: + "Public comment threads on any video URL, so you can read audience reaction beyond vanity views.", + }, { label: "Music", - description: "Track name, artist, and whether the sound is original — the seed of a trend.", + description: "Track name, artist, and whether the sound is original, the seed of a trend.", }, { label: "Hashtags", description: "Every hashtag on a video, so you can map a topic cluster or campaign.", }, - { - label: "Timestamps", - description: "Created and scraped times so you can track a video's momentum over runs.", - }, ], - useCasesHeading: "What teams do with the TikTok API", + useCasesHeading: "What teams do with the TikTok scraper API", useCases: [ { title: "Trend and hashtag monitoring", description: - "Track a hashtag and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", + "Use the TikTok hashtag scraper to track a topic and feed the stream to an agent that flags breakout videos, rising sounds, and formats before they saturate. Catch the wave while it is still rising, not after.", }, { title: "Creator and influencer discovery", description: - "Surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.", + "Scrape TikTok profiles and search users by keyword to surface the creators driving a topic, ranked by real engagement, so your team shortlists partners from data instead of a manager's pitch deck.", }, { title: "Competitor content analysis", @@ -102,16 +106,16 @@ export const tiktok: ConnectorPageContent = { "Watch what your category posts and what actually lands. Turn a competitor's best-performing formats and hooks into your own content brief.", }, { - title: "Campaign and sentiment tracking", + title: "Campaign and comment sentiment", description: - "Measure how a launch or branded hashtag spreads across TikTok — video count, reach, and engagement over time — then pull the comments on top videos to read how the audience actually reacts, not just a vanity view count.", + "Measure how a launch or branded hashtag spreads across TikTok, then use the TikTok comments scraper on top videos to read how the audience actually reacts, not just a vanity view count.", }, ], comparison: { - heading: "A TikTok API alternative built for agents", + heading: "A TikTok Research API alternative built for agents", intro: - "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need it for commercial research, here is how SurfSense compares.", + "TikTok's official Research API is approval-gated and largely limited to academic and nonprofit use. If you cannot get access or need commercial TikTok scraping, SurfSense is a TikTok API alternative. Here is how it compares.", columnLabel: "Official TikTok API", rows: [ { @@ -180,7 +184,7 @@ export const tiktok: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "Keyword search terms. Keyword video search is login-walled and returns no videos — use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", + "Keyword search terms. Keyword video search is login-walled and returns no videos; use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.", }, { name: "results_per_page", @@ -253,9 +257,19 @@ export const tiktok: ConnectorPageContent = { "SurfSense reads only public TikTok data, the same videos any logged-out visitor can see. It never logs in and cannot access private or deleted content. As always, review TikTok's terms and your own compliance needs before you run at scale.", }, { - question: "Does this need the official TikTok API?", + question: "Does this need the official TikTok Research API?", answer: - "No. It is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured videos back.", + "No. This TikTok scraper API is an independent alternative, not a wrapper on the Research API, so there is no application or approval process. You call SurfSense with one key, or add the MCP server to your agent, and get structured videos, comments, and profile data back.", + }, + { + question: "What TikTok data can I scrape?", + answer: + "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + }, + { + question: "Can I scrape TikTok comments and hashtags?", + answer: + "Yes. Pass a video URL to the comments endpoint for the public comment thread. Pass hashtag names or /tag/ URLs to the TikTok hashtag scraper to pull videos under that tag. Keyword video search is login-walled, so hashtags and direct URLs are the reliable discovery paths.", }, { question: "What are the rate limits?", @@ -265,18 +279,14 @@ export const tiktok: ConnectorPageContent = { { question: "Can I scrape a specific creator's videos?", answer: - "Pass a profile or profile URL and you always get the account's metadata — name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account; for reliable video results, scrape by hashtag or by a direct video URL.", - }, - { - question: "What TikTok data can I scrape?", - answer: - "Four verbs: scrape (videos by hashtag, profile, or URL), comments (a video's public comment thread), user search (find accounts by keyword — the reliable discovery path, since keyword video search is login-walled), and trending (the current Explore feed). Each returns structured items and is billed per item returned.", + "Pass a profile or profile URL and you always get the account's metadata: name, followers, bio, verification. TikTok often withholds the profile video list from automated clients, so that list can come back empty even for a public account. For reliable video results, scrape by hashtag or by a direct video URL.", }, ], related: [ - { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "YouTube API", href: "/youtube" }, + { label: "Reddit API", href: "/reddit" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, From 1bc7d9f51c7015917a0a3af9e584a6eb03a6faaa Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 15:03:24 -0700 Subject: [PATCH 25/28] Update README files in multiple languages to include Instagram and TikTok as new data sources for live scraping, enhancing the platform's competitive intelligence capabilities. --- .github/workflows/desktop-release.yml | 15 +++++++++++++++ README.es.md | 6 ++++-- README.hi.md | 6 ++++-- README.md | 6 ++++-- README.pt-BR.md | 6 ++++-- README.zh-CN.md | 6 ++++-- 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 3ad529671..7955adcbe 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -75,6 +75,21 @@ jobs: echo "Windows signing: skipped" fi + - name: Verify Apple notarization credentials + if: matrix.os == 'macos-latest' + shell: bash + # Fails in seconds with the same 401 notarytool would throw after the + # full build, instead of wasting ~15 min of build time on bad secrets. + run: | + xcrun notarytool history \ + --apple-id "$APPLE_ID" \ + --password "$APPLE_APP_SPECIFIC_PASSWORD" \ + --team-id "$APPLE_TEAM_ID" > /dev/null + env: + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + - name: Setup pnpm uses: pnpm/action-setup@v5 diff --git a/README.es.md b/README.es.md index ecdd93c41..08231b5ab 100644 --- a/README.es.md +++ b/README.es.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para investigación de inteligencia competitiva -SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. +SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. > [!NOTE] > **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM** @@ -103,6 +103,8 @@ Las automatizaciones ejecutan turnos completos de agente según un horario o en |---|---|---| | **Reddit** | Publicaciones, comentarios y flujos de subreddits sin los límites de tasa de la API oficial | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | Videos, transcripciones e hilos de comentarios para escuchar a tu marca y productos | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Perfiles, publicaciones y reels públicos sin la Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Videos, comentarios, hashtags y perfiles sin aprobación de la Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Lugares, calificaciones y reseñas para investigar competidores locales y prospectos | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs en vivo para seguimiento de posiciones y monitoreo de mercado | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** (rastreo web) | Cualquier página de la web abierta como contenido limpio y estructurado | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | Característica | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Google Maps, Google Search y rastreo web vía API REST y MCP | +| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y rastreo web vía API REST y MCP | | **Servidor MCP** | No | Cada conector expuesto como herramienta nativa de agente, más servidores MCP propios con aplicaciones OAuth de un clic | | **Fuentes por Notebook** | 50 (gratis) a 600 (Ultra, $249.99/mes) | Ilimitadas | | **Número de Notebooks** | 100 (gratis) a 500 (niveles de pago) | Ilimitado | diff --git a/README.hi.md b/README.hi.md index 3cb769318..eb65ff489 100644 --- a/README.hi.md +++ b/README.hi.md @@ -22,7 +22,7 @@ # SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM -SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। +SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। > [!NOTE] > **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना** @@ -103,6 +103,8 @@ SurfSense **AI एजेंट्स के लिए ओपन सोर्स |---|---|---| | **Reddit** | आधिकारिक API की रेट लिमिट के बिना पोस्ट, कमेंट और सबरेडिट स्ट्रीम | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | ब्रांड और प्रोडक्ट लिसनिंग के लिए वीडियो, ट्रांसक्रिप्ट और कमेंट थ्रेड | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Graph API के बिना सार्वजनिक प्रोफ़ाइल, पोस्ट और रील्स | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Research API अप्रूवल के बिना वीडियो, कमेंट, हैशटैग और प्रोफ़ाइल | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | स्थानीय प्रतिस्पर्धी और लीड रिसर्च के लिए स्थान, रेटिंग और रिव्यू | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | रैंक ट्रैकिंग और मार्केट मॉनिटरिंग के लिए लाइव SERP | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** | ओपन वेब का कोई भी पेज साफ़-सुथरे, स्ट्रक्चर्ड कंटेंट के रूप में | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | फ़ीचर | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Google Maps, Google Search और वेब क्रॉल कनेक्टर | +| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और वेब क्रॉल कनेक्टर | | **MCP सर्वर** | नहीं | हर कनेक्टर नेटिव एजेंट टूल के रूप में उपलब्ध, साथ ही वन-क्लिक OAuth ऐप्स के साथ अपने MCP सर्वर लाने की सुविधा | | **प्रति नोटबुक स्रोत** | 50 (Free) से 600 (Ultra, $249.99/माह) | असीमित | | **नोटबुक की संख्या** | 100 (Free) से 500 (सशुल्क टियर) | असीमित | diff --git a/README.md b/README.md index eac392ebd..ba289ada4 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM for Competitive Intelligence Research -SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. +SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. > [!NOTE] > **📢 A note for our NotebookLM-alternative users** @@ -103,6 +103,8 @@ Automations run full agent turns on a schedule or in response to events, then wr |---|---|---| | **Reddit** | Posts, comments, and subreddit streams without the official API's rate limits | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | Videos, transcripts, and comment threads for brand and product listening | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Public profiles, posts, and reels without the Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Videos, comments, hashtags, and profiles without Research API approval | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Places, ratings, and reviews for local competitor and lead research | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | Live SERPs for rank tracking and market monitoring | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** | Any page on the open web as clean, structured content | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -243,7 +245,7 @@ Still comparing us as a NotebookLM alternative? Here is the honest breakdown. | Feature | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Live market data for agents** | No | Reddit, YouTube, Google Maps, Google Search, and web crawl connectors via REST API and MCP | +| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and web crawl connectors via REST API and MCP | | **MCP server** | No | Every connector exposed as a native agent tool, plus bring-your-own MCP servers with one-click OAuth apps | | **Sources per Notebook** | 50 (Free) to 600 (Ultra, $249.99/mo) | Unlimited | | **Number of Notebooks** | 100 (Free) to 500 (paid tiers) | Unlimited | diff --git a/README.pt-BR.md b/README.pt-BR.md index e32882831..327e3373c 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva -O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. +O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. > [!NOTE] > **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM** @@ -103,6 +103,8 @@ As automações executam turnos completos de agente de forma agendada ou em resp |---|---|---| | **Reddit** | Posts, comentários e fluxos de subreddits sem os limites de requisição da API oficial | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | Vídeos, transcrições e threads de comentários para monitoramento de marca e produto | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | Perfis, posts e reels públicos sem a Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | Vídeos, comentários, hashtags e perfis sem aprovação da Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Estabelecimentos, avaliações e reviews para pesquisa local de concorrentes e leads | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs ao vivo para acompanhamento de rankings e monitoramento de mercado | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** (rastreamento web) | Qualquer página da web aberta como conteúdo limpo e estruturado | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ Ainda nos comparando como alternativa ao NotebookLM? Aqui está o comparativo ho | Recurso | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Google Maps, Google Search e rastreamento web via API REST e MCP | +| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e rastreamento web via API REST e MCP | | **Servidor MCP** | Não | Cada conector exposto como ferramenta nativa de agente, além de servidores MCP próprios com apps OAuth em um clique | | **Fontes por Notebook** | 50 (gratuito) a 600 (Ultra, US$ 249,99/mês) | Ilimitadas | | **Número de Notebooks** | 100 (gratuito) a 500 (planos pagos) | Ilimitado | diff --git a/README.zh-CN.md b/README.zh-CN.md index 11bace243..53a95968b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -22,7 +22,7 @@ # SurfSense：面向竞争情报研究的 NotebookLM -SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Google Maps、Google Search 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 +SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 > [!NOTE] > **📢 致我们的 NotebookLM 替代品用户** @@ -103,6 +103,8 @@ SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 Noteboo |---|---|---| | **Reddit** | 帖子、评论和子版块信息流，不受官方 API 速率限制 | [Reddit Scraper API](https://www.surfsense.com/reddit) | | **YouTube** | 视频、字幕转录和评论串，用于品牌和产品舆情监听 | [YouTube Scraper API](https://www.surfsense.com/youtube) | +| **Instagram** | 公开主页、帖子和 Reels，无需 Graph API | [Instagram Scraper API](https://www.surfsense.com/instagram) | +| **TikTok** | 视频、评论、话题标签和主页，无需 Research API 审批 | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | 地点、评分和评论，用于本地竞争对手和潜在客户调研 | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | 实时搜索结果页，用于排名追踪和市场监控 | [Google Search API](https://www.surfsense.com/google-search) | | **Web Crawl** | 把开放网络上的任意页面转为干净、结构化的内容 | [Web Crawling API](https://www.surfsense.com/web-crawl) | @@ -244,7 +246,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | 功能 | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Google Maps、Google Search 和网页爬取连接器 | +| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和网页爬取连接器 | | **MCP 服务器** | 无 | 每个连接器都作为原生智能体工具暴露，还可自带 MCP 服务器并使用一键 OAuth 应用 | | **每个笔记本的来源数** | 50 个（免费版）至 600 个（Ultra 版，249.99 美元/月） | 无限制 | | **笔记本数量** | 100 个（免费版）至 500 个（付费档位） | 无限制 | From 1131da5ed71bbffe3f729311da38fe1c3c1eaa0f Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 16:29:39 -0700 Subject: [PATCH 26/28] feat: bumped version to 0.0.32 --- VERSION | 2 +- .../builtins/instagram/system_prompt.md | 2 +- .../builtins/tiktok/system_prompt.md | 4 +- .../capabilities/tiktok/scrape/definition.py | 3 +- .../app/capabilities/tiktok/scrape/schemas.py | 7 +- .../platforms/instagram/scraper.py | 4 +- .../platforms/tiktok/orchestrator.py | 86 +++++++++- surfsense_backend/pyproject.toml | 2 +- .../unit/platforms/tiktok/test_discovery.py | 149 ++++++++++++++++++ surfsense_backend/uv.lock | 134 +++++++++++++++- surfsense_browser_extension/package.json | 2 +- surfsense_desktop/package.json | 2 +- surfsense_evals/scripts/analyze_failures.py | 2 - .../scripts/check_uploaded_status.py | 1 - .../scripts/compute_post_retry_accuracy.py | 4 +- .../patch_manifest_for_parallel_ingest.py | 1 - surfsense_evals/scripts/peek_crag_run.py | 7 +- surfsense_evals/scripts/peek_disagreements.py | 3 +- .../scripts/retry_failed_questions.py | 6 +- surfsense_evals/scripts/summarise_crag_run.py | 3 +- .../scripts/summarise_parser_compare_run.py | 3 +- .../src/surfsense_evals/core/cli.py | 7 +- .../surfsense_evals/core/clients/documents.py | 5 +- .../core/metrics/comparison.py | 4 +- .../surfsense_evals/core/parse/__init__.py | 2 +- .../surfsense_evals/core/parse/citations.py | 4 +- .../surfsense_evals/core/parsers/azure_di.py | 15 +- .../core/parsers/llamacloud.py | 2 +- .../core/providers/openrouter_chat.py | 2 +- .../core/providers/openrouter_pdf.py | 4 +- .../src/surfsense_evals/suites/__init__.py | 2 +- .../suites/_demo/hello/__init__.py | 1 - .../suites/medical/cure/__init__.py | 2 +- .../suites/medical/cure/ingest.py | 6 +- .../suites/medical/cure/runner.py | 5 +- .../suites/medical/mirage/__init__.py | 2 +- .../suites/medical/mirage/ingest.py | 96 +++++------ .../suites/medical/mirage/prompt.py | 1 - .../suites/medical/mirage/runner.py | 5 +- .../multimodal_doc/parser_compare/ingest.py | 4 +- .../multimodal_doc/parser_compare/runner.py | 4 +- .../suites/research/crag/html_extract.py | 5 +- .../suites/research/crag/prompt.py | 1 - .../suites/research/crag/runner.py | 4 +- .../suites/research/frames/dataset.py | 2 +- .../suites/research/frames/ingest.py | 1 - .../suites/research/frames/prompt.py | 1 - .../tests/suites/test_crag_dataset.py | 2 - .../tests/suites/test_crag_grader.py | 2 - .../tests/suites/test_crag_html_extract.py | 3 - .../tests/suites/test_frames_dataset.py | 3 - .../tests/suites/test_frames_grader.py | 3 - .../features/scrapers/platforms/instagram.py | 8 +- .../features/scrapers/platforms/tiktok.py | 18 ++- surfsense_web/package.json | 2 +- 55 files changed, 496 insertions(+), 159 deletions(-) create mode 100644 surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py diff --git a/VERSION b/VERSION index d788d4335..78bae5bb6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.31 +0.0.32 diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md index 42713cc38..ce9182771 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -13,7 +13,7 @@ Answer the delegated question from live Instagram data gathered with your verbs, - Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled). -- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). +- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). Google-backed discovery is slow (~30-60s per query), so start with **at most 3** distinct queries per task and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent. - Profile metadata (follower counts, bio, post count): call `instagram_details`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md index 03114cd12..4300e9313 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/tiktok/system_prompt.md @@ -14,11 +14,11 @@ Answer the delegated question from live TikTok data gathered with your verb, com -- Finding videos on a topic: call `tiktok_scrape` with `hashtags` (no leading '#'), or pass a TikTok URL in `urls`. +- Finding videos on a topic: prefer `tiktok_scrape` with `hashtags` (no leading '#') or a direct TikTok URL in `urls` (fastest). `search_queries` also finds videos on a topic, but it is Google-backed and slow, so start with **at most 3** distinct queries and only add more if the first round returns nothing significant — never batch many phrasing variants of the same intent. - Scraping a specific video, profile, hashtag, or search page: pass its TikTok URL in `urls`. - Profiles: a creator's `profiles` feed returns the account's metadata (name, followers, bio, verification) reliably, but its video list is often withheld by TikTok — treat an empty video list as a known limit, not a failure to retry endlessly. Prefer `hashtags` or a direct video URL for videos. - Comments on a video: call `tiktok_comments` with the video URL(s) in `video_urls`. -- Finding accounts by keyword: call `tiktok_user_search` with `queries`. Keyword search returns no videos, so do not use `search_queries` for videos — use `hashtags` or a video URL. +- Finding accounts by keyword: call `tiktok_user_search` with `queries` — that is the path for accounts. Use `search_queries` on `tiktok_scrape` only when you want videos, not accounts. - "What's trending now": call `tiktok_trending` (no query needed); set `max_items` for how many. - Controlling volume: use `max_items` for the total cap and `results_per_page` per target (per-verb equivalents: `comments_per_video`, `results_per_query`). - Requested counts: `max_items` defaults low — when the task asks for N items, set `max_items` (and the per-target count) above N. A call that caps below the target can never satisfy it. diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py index 9f8632f3b..e2706a2ef 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/definition.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/definition.py @@ -11,7 +11,8 @@ TIKTOK_SCRAPE = Capability( name="tiktok.scrape", description=( "Scrape public TikTok videos. Use urls, profiles, hashtags, or " - "search_queries." + "search_queries (search_queries are resolved via Google to public " + "videos; for accounts by keyword use tiktok.user_search)." ), input_schema=ScrapeInput, output_schema=ScrapeOutput, diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index 2f5005353..853b70bd9 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -43,7 +43,12 @@ class ScrapeInput(BaseModel): search_queries: list[str] = Field( default_factory=list, max_length=MAX_TIKTOK_SOURCES, - description="Search terms to run on TikTok.", + description=( + "Search terms resolved via Google (site:tiktok.com) to public TikTok " + "videos, since TikTok's own keyword search is login-walled. Slower " + "than hashtags/urls. To find accounts by keyword, use " + "tiktok.user_search instead." + ), ) results_per_page: int = Field( default=10, diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index a4c22bde7..9f4f6e2eb 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -306,9 +306,9 @@ async def _discover_via_google( """ serps = await scrape_serps( GoogleSearchScrapeInput( - queries=query, site="instagram.com", maxPagesPerQuery=2 + queries=query, site="instagram.com", maxPagesPerQuery=1 ), - limit=2, + limit=1, ) resolved: list[ResolvedUrl] = [] seen: set[tuple[str, str]] = set() diff --git a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py index 5d6a24545..01fae6caf 100644 --- a/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py +++ b/surfsense_backend/app/proprietary/platforms/tiktok/orchestrator.py @@ -12,6 +12,9 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import quote +from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput +from app.proprietary.platforms.google_search.scraper import scrape_serps + from .extraction.timestamps import now_iso from .flows import FetchCommentsFn, FetchFn, FetchListingFn, FetchUsersFn from .flows.comments import iter_comments @@ -35,9 +38,24 @@ _HASHTAG_URL = "https://www.tiktok.com/tag/{tag}" _SEARCH_URL = "https://www.tiktok.com/search?q={query}" _EXPLORE_URL = "https://www.tiktok.com/explore" +# A ``searchQueries`` term whose Google discovery surfaced no scrapable video +# URLs degrades to one honest ErrorItem (mirrors the listing flow's contract: +# never vanish silently). +_EMPTY_DISCOVERY_MESSAGE = ( + "No public TikTok videos found for this query via Google discovery. Try a " + "narrower phrasing, a hashtag, or a direct video URL." +) + def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: - """Build the target list from every input source, dropping unresolved URLs.""" + """Build the target list from the URL/profile/hashtag sources. + + ``searchQueries`` is deliberately excluded: TikTok's own keyword search is + login-walled for anonymous sessions, so it is routed through Google video + discovery in :func:`iter_tiktok` instead. A raw ``tiktok.com/search?...`` + URL passed explicitly in ``startUrls``/``postURLs`` still resolves here and + keeps its native listing routing. + """ targets: list[TikTokTarget] = [] for entry in input_model.startUrls: resolved = resolve_target(entry.url) @@ -52,13 +70,42 @@ def _resolve_targets(input_model: TikTokScrapeInput) -> list[TikTokTarget]: targets.append(TikTokTarget("profile", name, _PROFILE_URL.format(name=name))) for tag in input_model.hashtags: targets.append(TikTokTarget("hashtag", tag, _HASHTAG_URL.format(tag=tag))) - for query in input_model.searchQueries: - targets.append( - TikTokTarget("search", query, _SEARCH_URL.format(query=quote(query))) - ) return targets +async def _discover_via_google(query: str, *, limit: int) -> list[TikTokTarget]: + """Discover public TikTok video targets via Google ``site:tiktok.com``. + + TikTok's anonymous keyword search is login-walled, so we reuse the existing + ``google_search`` platform, classify each organic URL with ``resolve_target``, + and keep only video hits (``/@user/video/``) — the one kind that scrapes + reliably over plain HTTP. Profile/hashtag/search/photo/non-tiktok results are + dropped (accounts belong to the ``user_search`` verb). De-duped, capped at + ``limit``. + """ + serps = await scrape_serps( + GoogleSearchScrapeInput( + queries=query, site="tiktok.com", maxPagesPerQuery=1 + ), + limit=1, + ) + resolved: list[TikTokTarget] = [] + seen: set[str] = set() + for serp in serps: + for org in serp.get("organicResults") or []: + url = org.get("url", "") if isinstance(org, dict) else "" + target = resolve_target(url) + if target is None or target.kind != "video": + continue + if target.value in seen: + continue + seen.add(target.value) + resolved.append(target) + if len(resolved) >= limit: + return resolved + return resolved + + def _dispatch( target: TikTokTarget, *, @@ -81,9 +128,11 @@ async def iter_tiktok( ) -> AsyncIterator[dict[str, Any]]: """Yield normalized items for every resolved target, in order. - The video flow's ``fetch_html`` opens its own warmed proxy session per call - when none is bound; the listing flow drives its own browser. Neither binds a - ContextVar across these ``yield``s, so the generator stays context-safe. + Direct sources (URLs, profiles, hashtags) resolve up front; ``searchQueries`` + then run through Google video discovery. The video flow's ``fetch_html`` + opens its own warmed proxy session per call when none is bound; the listing + flow drives its own browser. Neither binds a ContextVar across these + ``yield``s, so the generator stays context-safe. """ cap = input_model.resultsPerPage for target in _resolve_targets(input_model): @@ -92,6 +141,27 @@ async def iter_tiktok( ): yield item + # searchQueries -> Google-discovered public video URLs, de-duped across + # queries so the same video surfacing under two terms is scraped once. + seen_videos: set[str] = set() + for query in input_model.searchQueries: + discovered = await _discover_via_google(query, limit=cap) + if not discovered: + yield ErrorItem( + url=_SEARCH_URL.format(query=quote(query)), + input=query, + error=_EMPTY_DISCOVERY_MESSAGE, + errorCode="no_items", + scrapedAt=now_iso(), + ).to_output() + continue + for target in discovered: + if target.value in seen_videos: + continue + seen_videos.add(target.value) + async for item in iter_video(target, fetch=fetch): + yield item + async def scrape_tiktok( input_model: TikTokScrapeInput, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 2faa2be17..30bb7ea5c 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "surf-new-backend" -version = "0.0.31" +version = "0.0.32" description = "SurfSense Backend" requires-python = ">=3.12" dependencies = [ diff --git a/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py new file mode 100644 index 000000000..4664f85e0 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/tiktok/test_discovery.py @@ -0,0 +1,149 @@ +"""Offline tests for Google-backed TikTok video discovery. + +``searchQueries`` are login-walled on TikTok's native search, so they route +through the ``google_search`` platform (``site:tiktok.com``): each organic URL +is classified with ``resolve_target`` and only video hits (``/@user/video/``) +are kept — profiles/hashtags/search/photo/non-tiktok are dropped (accounts +belong to the user-search verb). These tests inject a fake ``scrape_serps`` so +there is no network: they pin the classification, cross-query de-dup, the limit +cap, the barren-query ErrorItem, and that no ``/search?q=`` listing target is +ever built. +""" + +from __future__ import annotations + +import json + +from app.proprietary.platforms.tiktok import ( + TikTokScrapeInput, + orchestrator, + scrape_tiktok, +) + + +def _fake_serps(*organic_urls: str): + async def _scrape_serps(input_model, *, limit=None): + assert input_model.site == "tiktok.com" + assert input_model.maxPagesPerQuery == 1 + return [{"organicResults": [{"url": u} for u in organic_urls]}] + + return _scrape_serps + + +def _video_page(url: str) -> str: + """Render a rehydration blob for a ``/@user/video/`` URL.""" + video_id = url.rsplit("/", 1)[1] + username = url.split("@")[1].split("/")[0] + blob = { + "__DEFAULT_SCOPE__": { + "webapp.video-detail": { + "itemInfo": { + "itemStruct": { + "id": video_id, + "desc": "hi", + "author": {"uniqueId": username}, + "stats": {"diggCount": 1}, + } + } + } + } + } + return ( + '' + ) + + +async def _fetch_video(url: str) -> str: + return _video_page(url) + + +async def test_search_discovery_keeps_only_videos(monkeypatch): + # Only the video URL survives; profile / hashtag / search / photo / + # non-tiktok organic results are dropped. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps( + "https://www.tiktok.com/@nasa/video/123", + "https://www.tiktok.com/@nasa", + "https://www.tiktok.com/tag/space", + "https://www.tiktok.com/search?q=space", + "https://www.tiktok.com/@nasa/photo/999", + "https://example.com/not-tiktok", + ), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10), + fetch=_fetch_video, + ) + assert [i["id"] for i in items] == ["123"] + + +async def test_search_discovery_dedupes_across_queries(monkeypatch): + # The same video surfacing under two queries is scraped once. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps("https://www.tiktok.com/@nasa/video/123"), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space", "rockets"], resultsPerPage=10), + fetch=_fetch_video, + ) + assert [i["id"] for i in items] == ["123"] + + +async def test_search_discovery_respects_per_target_limit(monkeypatch): + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps( + "https://www.tiktok.com/@a/video/1", + "https://www.tiktok.com/@b/video/2", + "https://www.tiktok.com/@c/video/3", + ), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["x"], resultsPerPage=2), + fetch=_fetch_video, + ) + assert [i["id"] for i in items] == ["1", "2"] + + +async def test_search_barren_query_emits_error_item(monkeypatch): + # A query whose discovery finds no video URLs degrades to one ErrorItem. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps( + "https://www.tiktok.com/@nasa", + "https://example.com/x", + ), + ) + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10), + fetch=_fetch_video, + ) + assert len(items) == 1 + assert items[0]["errorCode"] == "no_items" + assert items[0]["input"] == "space" + + +async def test_search_never_builds_listing_target(monkeypatch): + # searchQueries must never hit the (login-walled) native search listing flow. + monkeypatch.setattr( + orchestrator, + "scrape_serps", + _fake_serps("https://www.tiktok.com/@nasa/video/123"), + ) + + async def _boom_listing(_url: str, _count: int) -> list[dict]: + raise AssertionError("searchQueries must not build a listing target") + + items = await scrape_tiktok( + TikTokScrapeInput(searchQueries=["space"], resultsPerPage=10), + fetch=_fetch_video, + fetch_listing=_boom_listing, + ) + assert [i["id"] for i in items] == ["123"] diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index a5b492fc8..2b1272363 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -36,6 +36,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -70,6 +73,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -104,6 +110,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -138,6 +147,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -172,6 +184,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -206,6 +221,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -240,6 +258,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -274,6 +295,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -308,6 +332,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -342,6 +369,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -387,6 +417,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -421,6 +455,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -455,6 +492,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -489,6 +529,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -523,6 +566,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -557,6 +603,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -591,6 +640,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -625,6 +677,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -659,6 +714,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -693,6 +751,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -727,6 +788,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -772,6 +836,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -806,6 +874,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -840,6 +911,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -874,6 +948,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -908,6 +985,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -942,6 +1022,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -976,6 +1059,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1010,6 +1096,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1044,6 +1133,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1078,6 +1170,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1112,6 +1207,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1157,6 +1255,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1191,6 +1293,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1258,6 +1363,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1292,6 +1403,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1326,6 +1440,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1393,6 +1510,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1460,6 +1583,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1494,6 +1623,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -10353,7 +10485,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.31" +version = "0.0.32" source = { editable = "." } dependencies = [ { name = "alembic" }, diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index 8dda0d82f..a2c25a602 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.31", + "version": "0.0.32", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index f12b9722c..629fe801d 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -1,7 +1,7 @@ { "name": "surfsense-desktop", "productName": "SurfSense", - "version": "0.0.31", + "version": "0.0.32", "description": "SurfSense Desktop App", "main": "dist/main.js", "scripts": { diff --git a/surfsense_evals/scripts/analyze_failures.py b/surfsense_evals/scripts/analyze_failures.py index e7ace1e1b..ff5ec23f4 100644 --- a/surfsense_evals/scripts/analyze_failures.py +++ b/surfsense_evals/scripts/analyze_failures.py @@ -12,12 +12,10 @@ Outputs (printed to stdout + written to `failures_n171.json`): from __future__ import annotations import json -import re from collections import Counter, defaultdict from pathlib import Path from typing import Any - REPO = Path(__file__).resolve().parents[1] RUN = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare" RAW = RUN / "raw.jsonl" diff --git a/surfsense_evals/scripts/check_uploaded_status.py b/surfsense_evals/scripts/check_uploaded_status.py index 7021ba83d..1903502ed 100644 --- a/surfsense_evals/scripts/check_uploaded_status.py +++ b/surfsense_evals/scripts/check_uploaded_status.py @@ -15,7 +15,6 @@ from pathlib import Path import httpx from dotenv import load_dotenv - REPO = Path(__file__).resolve().parents[1] PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs" diff --git a/surfsense_evals/scripts/compute_post_retry_accuracy.py b/surfsense_evals/scripts/compute_post_retry_accuracy.py index 4c8c47672..f3a716219 100644 --- a/surfsense_evals/scripts/compute_post_retry_accuracy.py +++ b/surfsense_evals/scripts/compute_post_retry_accuracy.py @@ -47,9 +47,7 @@ def _row_key(row: dict) -> tuple[str, str]: def _is_failure(row: dict) -> bool: if row.get("error"): return True - if not (row.get("raw_text") or "").strip(): - return True - return False + return bool(not (row.get("raw_text") or "").strip()) def _summarise(rows_by_arm: dict[str, list[dict]]) -> dict[str, dict]: diff --git a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py index e1a2edc65..8cfb13eb7 100644 --- a/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py +++ b/surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py @@ -27,7 +27,6 @@ from __future__ import annotations import json from pathlib import Path - REPO = Path(__file__).resolve().parents[1] MAP_PATH = REPO / "data" / "multimodal_doc" / "maps" / "mmlongbench_doc_map.jsonl" PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs" diff --git a/surfsense_evals/scripts/peek_crag_run.py b/surfsense_evals/scripts/peek_crag_run.py index 225e5ec98..3a79d76c7 100644 --- a/surfsense_evals/scripts/peek_crag_run.py +++ b/surfsense_evals/scripts/peek_crag_run.py @@ -10,19 +10,20 @@ from collections import defaultdict def main() -> None: raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1] print(f"Reading: {raw_path}") - rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()] + with open(raw_path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] by_q: dict[str, dict[str, dict]] = defaultdict(dict) for r in rows: by_q[r["qid"]][r["arm"]] = r for qid, arms in list(by_q.items()): b = arms.get("bare_llm", {}) - l = arms.get("long_context", {}) + lc = arms.get("long_context", {}) s = arms.get("surfsense", {}) print(f"\n=== {qid} ({b.get('domain')}/{b.get('question_type')}) ===") print(f" question: {b.get('extra', {}).get('question', '?')!r}") print(f" gold: {b.get('gold')!r}") - for arm_name, a in (("bare_llm", b), ("long_context", l), ("surfsense", s)): + for arm_name, a in (("bare_llm", b), ("long_context", lc), ("surfsense", s)): grade = a.get("graded", {}) text = (a.get("raw_text") or "").strip() tail = text[-200:] if text else "" diff --git a/surfsense_evals/scripts/peek_disagreements.py b/surfsense_evals/scripts/peek_disagreements.py index c0fe0acd9..b6497570e 100644 --- a/surfsense_evals/scripts/peek_disagreements.py +++ b/surfsense_evals/scripts/peek_disagreements.py @@ -10,7 +10,8 @@ from collections import defaultdict def main() -> None: raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1] print(f"Reading: {raw_path}") - rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()] + with open(raw_path, encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] by_q: dict[str, dict[str, dict]] = defaultdict(dict) for r in rows: by_q[r["qid"]][r["arm"]] = r diff --git a/surfsense_evals/scripts/retry_failed_questions.py b/surfsense_evals/scripts/retry_failed_questions.py index 7cc9478e0..25facfe60 100644 --- a/surfsense_evals/scripts/retry_failed_questions.py +++ b/surfsense_evals/scripts/retry_failed_questions.py @@ -106,9 +106,7 @@ def _is_failure_row(row: dict[str, Any]) -> bool: if row.get("error"): return True - if not (row.get("raw_text") or "").strip(): - return True - return False + return bool(not (row.get("raw_text") or "").strip()) @dataclass @@ -428,7 +426,7 @@ async def _run(args: argparse.Namespace) -> int: if f.arm == "native_pdf": pdf_path = Path(map_row["pdf_path"]) - if not pdf_path.exists(): + if not await asyncio.to_thread(pdf_path.exists): logger.error("PDF missing on disk: %s — skipping", pdf_path) continue request = _build_native_request( diff --git a/surfsense_evals/scripts/summarise_crag_run.py b/surfsense_evals/scripts/summarise_crag_run.py index 646fb6a97..9722cd391 100644 --- a/surfsense_evals/scripts/summarise_crag_run.py +++ b/surfsense_evals/scripts/summarise_crag_run.py @@ -11,7 +11,8 @@ def main() -> None: if not runs: print("(no CRAG runs found)") return - m = json.load(open(runs[-1], encoding="utf-8")) + with open(runs[-1], encoding="utf-8") as fh: + m = json.load(fh) metrics = m["metrics"] print(f"Reading: {runs[-1]}") diff --git a/surfsense_evals/scripts/summarise_parser_compare_run.py b/surfsense_evals/scripts/summarise_parser_compare_run.py index c54d82784..c091043aa 100644 --- a/surfsense_evals/scripts/summarise_parser_compare_run.py +++ b/surfsense_evals/scripts/summarise_parser_compare_run.py @@ -16,7 +16,6 @@ import statistics from collections import defaultdict from pathlib import Path - REPO = Path(__file__).resolve().parents[1] RUN_DIR = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare" RAW = RUN_DIR / "raw.jsonl" @@ -91,7 +90,7 @@ def main() -> None: print() print("by answer_format (accuracy):") - formats = sorted({f for m in arm_metrics.values() for f in m["by_format"].keys()}) + formats = sorted({f for m in arm_metrics.values() for f in m["by_format"]}) header = f"{'arm':<25} " + " ".join(f"{f:>10}" for f in formats) print(header) print("-" * len(header)) diff --git a/surfsense_evals/src/surfsense_evals/core/cli.py b/surfsense_evals/src/surfsense_evals/core/cli.py index 17979fba0..2dcbe7327 100644 --- a/surfsense_evals/src/surfsense_evals/core/cli.py +++ b/surfsense_evals/src/surfsense_evals/core/cli.py @@ -32,14 +32,13 @@ from __future__ import annotations import argparse import asyncio +import contextlib import json import logging import sys from dataclasses import dataclass from typing import Any -import sys - import httpx from rich.console import Console from rich.table import Table @@ -51,10 +50,8 @@ from rich.table import Table # Terminal, PowerShell, cmd) all interpret ANSI escapes natively. if sys.platform == "win32": for _stream in (sys.stdout, sys.stderr): - try: + with contextlib.suppress(AttributeError, ValueError): _stream.reconfigure(encoding="utf-8", errors="replace") - except (AttributeError, ValueError): - pass from . import registry from .auth import CredentialError, acquire_token, client_with_auth diff --git a/surfsense_evals/src/surfsense_evals/core/clients/documents.py b/surfsense_evals/src/surfsense_evals/core/clients/documents.py index 362aae53b..b96bef6d8 100644 --- a/surfsense_evals/src/surfsense_evals/core/clients/documents.py +++ b/surfsense_evals/src/surfsense_evals/core/clients/documents.py @@ -18,6 +18,7 @@ Document processing is asynchronous: from __future__ import annotations import asyncio +import contextlib import logging import mimetypes from collections.abc import Iterable, Sequence @@ -157,10 +158,8 @@ class DocumentsClient: ) finally: for _, (_, file_obj, _) in opened: - try: + with contextlib.suppress(Exception): file_obj.close() - except Exception: # noqa: BLE001 - pass response.raise_for_status() return FileUploadResult.from_payload(response.json()) diff --git a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py index 579576f4f..9e40dc5ca 100644 --- a/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py +++ b/surfsense_evals/src/surfsense_evals/core/metrics/comparison.py @@ -71,8 +71,8 @@ def mcnemar_test( f"Length mismatch: arm_a={len(arm_a_correct)}, arm_b={len(arm_b_correct)}" ) n = len(arm_a_correct) - b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct) if a and not c) - c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct) if (not a) and cc) + b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct, strict=False) if a and not c) + c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct, strict=False) if (not a) and cc) discordant = b + c if discordant == 0: return McnemarResult( diff --git a/surfsense_evals/src/surfsense_evals/core/parse/__init__.py b/surfsense_evals/src/surfsense_evals/core/parse/__init__.py index 208c2d374..8b90de78b 100644 --- a/surfsense_evals/src/surfsense_evals/core/parse/__init__.py +++ b/surfsense_evals/src/surfsense_evals/core/parse/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations from .answer_letter import AnswerLetterResult, extract_answer_letter -from .citations import CITATION_REGEX, CitationToken, ChunkCitation, UrlCitation, parse_citations +from .citations import CITATION_REGEX, ChunkCitation, CitationToken, UrlCitation, parse_citations from .freeform_answer import extract_freeform_answer from .sse import SseEvent, iter_sse_events diff --git a/surfsense_evals/src/surfsense_evals/core/parse/citations.py b/surfsense_evals/src/surfsense_evals/core/parse/citations.py index 1fcd35434..c57ffeb0b 100644 --- a/surfsense_evals/src/surfsense_evals/core/parse/citations.py +++ b/surfsense_evals/src/surfsense_evals/core/parse/citations.py @@ -15,7 +15,7 @@ from __future__ import annotations import re from dataclasses import dataclass -from typing import Any, Union +from typing import Any # Pattern preserves the TS source verbatim: # /[\[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g @@ -64,7 +64,7 @@ class UrlCitation: return {"kind": "url", "url": self.url} -CitationToken = Union[ChunkCitation, UrlCitation] +CitationToken = ChunkCitation | UrlCitation def parse_citations(text: str, *, url_map: dict[str, str] | None = None) -> list[CitationToken]: diff --git a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py index eebad906a..7d796ee99 100644 --- a/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py +++ b/surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py @@ -25,6 +25,7 @@ import asyncio import logging import os import random +from pathlib import Path logger = logging.getLogger(__name__) @@ -82,7 +83,7 @@ async def parse_with_azure_di( ServiceResponseError, ) - file_size_mb = os.path.getsize(file_path) / (1024 * 1024) + file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024) logger.info( "Azure DI parsing %s (mode=%s, model=%s, size=%.1fMB)", file_path, processing_mode, model_id, file_size_mb, @@ -96,12 +97,12 @@ async def parse_with_azure_di( credential=AzureKeyCredential(api_key), ) async with client: - with open(file_path, "rb") as fh: - poller = await client.begin_analyze_document( - model_id, - body=fh, - output_content_format=DocumentContentFormat.MARKDOWN, - ) + body = await asyncio.to_thread(Path(file_path).read_bytes) + poller = await client.begin_analyze_document( + model_id, + body=body, + output_content_format=DocumentContentFormat.MARKDOWN, + ) result = await poller.result() content = (result.content or "").strip() if not content: diff --git a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py index ba3d787ef..300b4ec87 100644 --- a/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py +++ b/surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py @@ -98,7 +98,7 @@ async def parse_with_llamacloud( from llama_cloud_services.parse.base import JobFailedException from llama_cloud_services.parse.utils import ResultType - file_size_mb = os.path.getsize(file_path) / (1024 * 1024) + file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024) # Match backend's per-page timeout heuristic so big PDFs don't drop # mid-job: 60s baseline + 30s/page (premium agent runs longer than # basic; both fit comfortably here). diff --git a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py index 2494434be..208fbb865 100644 --- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py +++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py @@ -29,8 +29,8 @@ from typing import Any import httpx from .openrouter_pdf import ( - OpenRouterResponse, _DEFAULT_HEADERS, + OpenRouterResponse, _parse_chat_completion, ) diff --git a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py index e98590cbf..985d88a68 100644 --- a/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py +++ b/surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py @@ -34,7 +34,7 @@ import base64 import logging import time from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from pathlib import Path from typing import Any @@ -43,7 +43,7 @@ import httpx logger = logging.getLogger(__name__) -class PdfEngine(str, Enum): +class PdfEngine(StrEnum): NATIVE = "native" MISTRAL_OCR = "mistral-ocr" CLOUDFLARE_AI = "cloudflare-ai" diff --git a/surfsense_evals/src/surfsense_evals/suites/__init__.py b/surfsense_evals/src/surfsense_evals/suites/__init__.py index 95ed958ca..a0a01223c 100644 --- a/surfsense_evals/src/surfsense_evals/suites/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/__init__.py @@ -20,7 +20,7 @@ from __future__ import annotations import importlib import logging import pkgutil -from typing import Iterable +from collections.abc import Iterable logger = logging.getLogger(__name__) diff --git a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py index 1da33926c..43dc51ac5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py @@ -6,7 +6,6 @@ import argparse from typing import Any from ....core.registry import ( - Benchmark, ReportSection, RunArtifact, RunContext, diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py index e13224be7..7e9d9a07b 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py @@ -12,7 +12,7 @@ Recall@k / MRR / nDCG@10 against qrels. from __future__ import annotations -from .runner import CureBenchmark from ....core import registry as _registry +from .runner import CureBenchmark _registry.register(CureBenchmark()) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py index 275e28ce5..0c32a38a1 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py @@ -227,12 +227,10 @@ async def run_ingest( def _take(it: Iterable, n: int) -> Iterable: - yielded = 0 - for x in it: - if yielded >= n: + for i, x in enumerate(it): + if i >= n: return yield x - yielded += 1 __all__ = ["DISCIPLINES", "CorpusPassage", "PassageBatch", "run_ingest"] diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py index 041e0e8b5..4a85b6ba5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py @@ -34,7 +34,6 @@ from ....core.ingest_settings import ( ) from ....core.metrics.retrieval import score_run from ....core.registry import ( - Benchmark, ReportSection, RunArtifact, RunContext, @@ -276,7 +275,7 @@ class CureBenchmark: ) per_query_retrieved: dict[str, list[str]] = {} - for q, res in zip(queries, results): + for q, res in zip(queries, results, strict=False): chunk_ids: list[int] = [] seen: set[int] = set() for citation in res.citations: @@ -311,7 +310,7 @@ class CureBenchmark: run_dir = ctx.runs_dir(run_timestamp=run_timestamp) raw_path = run_dir / "raw.jsonl" with raw_path.open("w", encoding="utf-8") as fh: - for q, res in zip(queries, results): + for q, res in zip(queries, results, strict=False): fh.write( json.dumps( { diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py index e527b37f4..265dd62f7 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py @@ -11,7 +11,7 @@ document — the corpus is millions of biomedical snippets. from __future__ import annotations -from .runner import MirageBenchmark from ....core import registry as _registry +from .runner import MirageBenchmark _registry.register(MirageBenchmark()) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py index 59006b6c0..8e891aabf 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py @@ -93,6 +93,25 @@ class SnippetRow: # --------------------------------------------------------------------------- +def _reuse_cached_dest(dest: Path, *, expect_zip: bool, label: str) -> Path | None: + """Return ``dest`` if a usable cache hit, else ``None`` (and delete corrupt zips).""" + + if not dest.exists(): + return None + if expect_zip and not _is_valid_zip(dest): + logger.warning( + "Cached %s at %s failed ZIP validation (size=%d B); deleting " + "and re-downloading.", + label, + dest, + dest.stat().st_size, + ) + dest.unlink(missing_ok=True) + return None + logger.info("Using cached %s at %s", label, dest) + return dest + + async def _fetch_to_path( url: str, *, @@ -127,19 +146,9 @@ async def _fetch_to_path( surprise-grabbing 16 GB on a slow link. """ - if dest.exists(): - if expect_zip and not _is_valid_zip(dest): - logger.warning( - "Cached %s at %s failed ZIP validation (size=%d B); deleting " - "and re-downloading.", - label, - dest, - dest.stat().st_size, - ) - dest.unlink(missing_ok=True) - else: - logger.info("Using cached %s at %s", label, dest) - return dest + cached = _reuse_cached_dest(dest, expect_zip=expect_zip, label=label) + if cached is not None: + return cached dest.parent.mkdir(parents=True, exist_ok=True) partial = dest.with_suffix(dest.suffix + ".partial") @@ -170,39 +179,38 @@ async def _fetch_to_path( async with httpx.AsyncClient( timeout=httpx.Timeout(timeout_s, connect=20.0), follow_redirects=True, - ) as client: - async with client.stream("GET", url, headers=headers) as response: - if existing_bytes and response.status_code == 200: - logger.warning( - "Server ignored Range header for %s; restarting from 0.", - label, - ) - partial.unlink(missing_ok=True) - existing_bytes = 0 - elif response.status_code == 416: - # Range not satisfiable — the .partial is at or - # past the end. Treat as "already downloaded"; - # validate by closing and re-opening for atomic - # rename below. - logger.info( - "Server reports %s already complete (HTTP 416).", - label, - ) - elif response.status_code not in (200, 206): - response.raise_for_status() + ) as client, client.stream("GET", url, headers=headers) as response: + if existing_bytes and response.status_code == 200: + logger.warning( + "Server ignored Range header for %s; restarting from 0.", + label, + ) + partial.unlink(missing_ok=True) + existing_bytes = 0 + elif response.status_code == 416: + # Range not satisfiable — the .partial is at or + # past the end. Treat as "already downloaded"; + # validate by closing and re-opening for atomic + # rename below. + logger.info( + "Server reports %s already complete (HTTP 416).", + label, + ) + elif response.status_code not in (200, 206): + response.raise_for_status() - total_size = _planned_total_size(response, existing_bytes) - if ( - total_size is not None - and total_size > _LARGE_DOWNLOAD_BYTES - and not allow_large_download - ): - raise _LargeDownloadAbort(label, total_size) + total_size = _planned_total_size(response, existing_bytes) + if ( + total_size is not None + and total_size > _LARGE_DOWNLOAD_BYTES + and not allow_large_download + ): + raise _LargeDownloadAbort(label, total_size) - mode = "ab" if existing_bytes else "wb" - with partial.open(mode) as fh: - async for chunk in response.aiter_bytes(chunk_size=1 << 18): - fh.write(chunk) + mode = "ab" if existing_bytes else "wb" + with partial.open(mode) as fh: + async for chunk in response.aiter_bytes(chunk_size=1 << 18): + fh.write(chunk) # Optional content sanity check before promoting to dest. if expect_zip and not _is_valid_zip(partial): raise zipfile.BadZipFile( diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py index 9e5b1c618..3e5192aaa 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py @@ -8,7 +8,6 @@ from __future__ import annotations from collections.abc import Mapping - _PROMPT_TEMPLATE = """\ You are a helpful medical expert. Answer the following multiple-choice question using the relevant medical knowledge available to you (and any diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py index b01b645a9..8a6b04ae0 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py @@ -29,7 +29,6 @@ from ....core.ingest_settings import ( ) from ....core.metrics.mc_accuracy import accuracy_with_wilson_ci, macro_accuracy from ....core.registry import ( - Benchmark, ReportSection, RunArtifact, RunContext, @@ -229,7 +228,7 @@ class MirageBenchmark: run_dir = ctx.runs_dir(run_timestamp=run_timestamp) raw_path = run_dir / "raw.jsonl" with raw_path.open("w", encoding="utf-8") as fh: - for q, res in zip(questions, results): + for q, res in zip(questions, results, strict=False): fh.write( json.dumps( { @@ -246,7 +245,7 @@ class MirageBenchmark: for task in tasks: n_correct = 0 n_total = 0 - for q, res in zip(questions, results): + for q, res in zip(questions, results, strict=False): if q.task != task: continue n_total += 1 diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py index 93c8db4ab..1360ec89e 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py @@ -41,8 +41,6 @@ from typing import Any from ....core.config import set_suite_state from ....core.parsers import ( - AzureDIError, - LlamaCloudError, count_pdf_pages, parse_with_azure_di, parse_with_llamacloud, @@ -131,7 +129,7 @@ async def _run_one_extraction( else: raise ValueError(f"Unknown parser {parser!r}") out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(markdown, encoding="utf-8") + await asyncio.to_thread(out_path.write_text, markdown, encoding="utf-8") return markdown, time.monotonic() - started diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py index 2c4a0ffe4..297bd2fa0 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py @@ -603,8 +603,8 @@ class ParserCompareBenchmark: f"engine: `{extra.get('pdf_engine', 'native')}`)." ) body.append( - f"- Preprocess tariff: basic = $1 / 1k pages, " - f"premium = $10 / 1k pages." + "- Preprocess tariff: basic = $1 / 1k pages, " + "premium = $10 / 1k pages." ) body.append("") body.append("### Per-arm summary") diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py index 1b00aedc2..dd618d7e3 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py @@ -177,10 +177,7 @@ def extract_main_content( # Prefer trafilatura output even if short, but only if it # contained any prose at all — empty trafilatura fall-through # to the stripped form. - if body and stripped and len(stripped) > len(body) * 1.5: - body = stripped - method = "fallback_strip" - elif not body and stripped: + if body and stripped and len(stripped) > len(body) * 1.5 or not body and stripped: body = stripped method = "fallback_strip" elif body: diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py index 626834505..0d4327774 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py @@ -28,7 +28,6 @@ in the runner, so the format is mandatory. from __future__ import annotations - _BASE_INSTRUCTIONS = ( "You are a careful question-answering assistant. The question is a " "real-world factual question that may be about finance, music, " diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py index 654c261a2..efb7b4474 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py @@ -830,11 +830,11 @@ def _per_facet_truthfulness( """Bucket truthfulness scores by ``key_fn(q)``.""" buckets: dict[str, dict[str, list[CragGradeResult]]] = {} - for q, b, l, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False): + for q, b, lc, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False): key = key_fn(q) bucket = buckets.setdefault(key, {"bare_llm": [], "long_context": [], "surfsense": []}) bucket["bare_llm"].append(b) - bucket["long_context"].append(l) + bucket["long_context"].append(lc) bucket["surfsense"].append(s) out: dict[str, Any] = {} for key, arms in buckets.items(): diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py index c3b6b878e..629874102 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py @@ -102,7 +102,7 @@ def _parse_wiki_links(raw: Any) -> list[str]: except (SyntaxError, ValueError): # Fall back: maybe it's a comma-separated string with no quotes. return [tok.strip() for tok in text.strip("[]").split(",") if tok.strip()] - if isinstance(parsed, (list, tuple)): + if isinstance(parsed, list | tuple): return [str(x).strip() for x in parsed if str(x).strip()] return [str(parsed).strip()] diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py index 98e035f28..0288e192e 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py @@ -34,7 +34,6 @@ filename (without extension), so we round-trip via from __future__ import annotations -import asyncio import json import logging from dataclasses import dataclass diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py index 16bb06da4..8f8b6e3b6 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py @@ -17,7 +17,6 @@ Format expectations (mirrors the FRAMES paper, section 4): from __future__ import annotations - _BASE_INSTRUCTIONS = ( "You are a careful question-answering assistant. The question may " "require combining facts from multiple sources, doing arithmetic, " diff --git a/surfsense_evals/tests/suites/test_crag_dataset.py b/surfsense_evals/tests/suites/test_crag_dataset.py index 36114b52e..6221467fd 100644 --- a/surfsense_evals/tests/suites/test_crag_dataset.py +++ b/surfsense_evals/tests/suites/test_crag_dataset.py @@ -11,8 +11,6 @@ import bz2 import json from pathlib import Path -import pytest - from surfsense_evals.suites.research.crag.dataset import ( CragPage, CragQuestion, diff --git a/surfsense_evals/tests/suites/test_crag_grader.py b/surfsense_evals/tests/suites/test_crag_grader.py index 93bf6f478..74960afa6 100644 --- a/surfsense_evals/tests/suites/test_crag_grader.py +++ b/surfsense_evals/tests/suites/test_crag_grader.py @@ -7,8 +7,6 @@ exercise the deterministic shortcut + the special-case routing for from __future__ import annotations -import pytest - from surfsense_evals.suites.research.crag.grader import ( CragGradeResult, _flags_false_premise, diff --git a/surfsense_evals/tests/suites/test_crag_html_extract.py b/surfsense_evals/tests/suites/test_crag_html_extract.py index a2b47aa45..3bf757dbd 100644 --- a/surfsense_evals/tests/suites/test_crag_html_extract.py +++ b/surfsense_evals/tests/suites/test_crag_html_extract.py @@ -11,13 +11,10 @@ We don't network-fetch trafilatura; we just verify the wrapper: from __future__ import annotations -import pytest - from surfsense_evals.suites.research.crag.html_extract import ( extract_main_content, ) - _RICH_HTML = """\ diff --git a/surfsense_evals/tests/suites/test_frames_dataset.py b/surfsense_evals/tests/suites/test_frames_dataset.py index e79e7db89..f76dc0b6f 100644 --- a/surfsense_evals/tests/suites/test_frames_dataset.py +++ b/surfsense_evals/tests/suites/test_frames_dataset.py @@ -16,8 +16,6 @@ from __future__ import annotations import textwrap from pathlib import Path -import pytest - from surfsense_evals.suites.research.frames.dataset import ( FramesQuestion, _parse_reasoning_types, @@ -25,7 +23,6 @@ from surfsense_evals.suites.research.frames.dataset import ( load_questions, ) - # --------------------------------------------------------------------------- # Pure-function tests # --------------------------------------------------------------------------- diff --git a/surfsense_evals/tests/suites/test_frames_grader.py b/surfsense_evals/tests/suites/test_frames_grader.py index e6e38ff8a..d4bbad79a 100644 --- a/surfsense_evals/tests/suites/test_frames_grader.py +++ b/surfsense_evals/tests/suites/test_frames_grader.py @@ -8,10 +8,7 @@ runner knows to consult the judge. from __future__ import annotations -import pytest - from surfsense_evals.suites.research.frames.grader import ( - GradeResult, _maybe_number, _normalise, _whole_word_substring, diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py index 28ad34e66..1792bf7dd 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -41,7 +41,9 @@ def register( list[str] | None, Field( description="Terms to discover public profiles for (resolved via " - "Google). Provide search_queries OR urls." + "Google). Google-backed discovery is slow (~30-60s per query), so " + "start with at most 3 queries and expand only if nothing " + "significant is found. Provide search_queries OR urls." ), ] = None, search_type: Annotated[ @@ -100,7 +102,9 @@ def register( search_queries: Annotated[ list[str] | None, Field( - description="Terms to discover public profiles for. Provide " + description="Terms to discover public profiles for. Google-backed " + "discovery is slow (~30-60s per query), so start with at most 3 " + "queries and expand only if nothing significant is found. Provide " "search_queries OR urls." ), ] = None, diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py index c295ec3b6..7e2bff509 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py @@ -52,9 +52,11 @@ def register( search_queries: Annotated[ list[str] | None, Field( - description="Keyword search terms. Returns no videos — use " - "hashtags/profiles/urls for videos, or the user-search tool for " - "accounts." + description="Keyword search terms, resolved via Google to public " + "TikTok videos (TikTok's own keyword search is login-walled). " + "Slower than hashtags/urls, so start with at most 3 queries and " + "expand only if nothing significant is found. For accounts by " + "keyword use surfsense_tiktok_user_search." ), ] = None, results_per_page: Annotated[ @@ -67,13 +69,15 @@ def register( workspace: WorkspaceParam = None, response_format: ResponseFormatParam = "markdown", ) -> str: - """Scrape public TikTok videos by hashtag, profile, or URL. + """Scrape public TikTok videos by hashtag, profile, URL, or keyword. Use for TikTok video research — a creator's videos, a hashtag feed, or a specific video/profile/hashtag URL — instead of a generic web search. - Returns videos with text, author, stats, music, and the web URL. For - accounts by keyword use the user-search tool; keyword search returns no - videos. Example: hashtags=['food'], max_items=20. + search_queries also finds videos on a topic (resolved via Google), but is + slower: start with at most 3 queries and expand only if nothing + significant is found. Returns videos with text, author, stats, music, and + the web URL. For accounts by keyword use surfsense_tiktok_user_search. + Example: hashtags=['food'], max_items=20. """ return await run_scraper( client, diff --git a/surfsense_web/package.json b/surfsense_web/package.json index efa04b4f6..989dde480 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,6 +1,6 @@ { "name": "surfsense_web", - "version": "0.0.31", + "version": "0.0.32", "private": true, "packageManager": "pnpm@10.26.0", "description": "SurfSense Frontend", From e8b3692b546c234964b0fd3a32b39b535a3c796d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Jul 2026 16:30:37 -0700 Subject: [PATCH 27/28] chore: linting --- .../playground/components/output-viewer.tsx | 10 ++------- .../components/playground-index.tsx | 3 ++- .../playground/components/run-detail.tsx | 8 +------ .../components/run-progress-panel.tsx | 4 +++- .../components/run-status-badge.tsx | 5 ++++- .../playground/components/runs-table.tsx | 3 ++- .../playground/components/schema-form.tsx | 19 +++-------------- .../components/assistant-ui/thread.tsx | 5 +---- .../components/homepage/hero-chat-demo.tsx | 3 ++- .../components/homepage/hero-section.tsx | 8 +++---- .../ui/sidebar/NotificationsDropdown.tsx | 4 +--- .../settings/auto-reload-settings.tsx | 10 ++++++--- .../settings/workspace-api-access-control.tsx | 21 ++++++++++++++++--- .../content/docs/connectors/native/meta.json | 10 ++++++++- surfsense_web/hooks/use-run-stream.ts | 16 +++----------- .../lib/apis/scrapers-api.service.ts | 6 +++--- .../lib/connectors-marketing/tiktok.tsx | 3 ++- .../lib/playground/code-snippets.selfcheck.ts | 6 +++++- surfsense_web/lib/playground/format.ts | 4 +--- surfsense_web/lib/playground/json-schema.ts | 8 +------ 20 files changed, 74 insertions(+), 82 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx index e692734d3..72262d440 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx @@ -3,7 +3,6 @@ import { Check, Copy, Download } from "lucide-react"; import { useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Table, TableBody, @@ -12,6 +11,7 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { downloadCsv, rowsToCsv } from "@/lib/playground/csv"; const MAX_TABLE_ROWS = 200; @@ -117,13 +117,7 @@ export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBa
{items && items.length > 0 && ( - diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx index 58d56e146..732e1105c 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx @@ -25,7 +25,8 @@ export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) {

- Manually run SurfSense's platform-native APIs and inspect their output. To use these APIs outside SurfSense,{" "} + Manually run SurfSense's platform-native APIs and inspect their output. To use these + APIs outside SurfSense,{" "} parseJsonl(run?.output_text ?? null), [run?.output_text]); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx index 33a88ae76..a4e9f0023 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-progress-panel.tsx @@ -9,7 +9,9 @@ import { formatDuration } from "@/lib/playground/format"; function eventLabel(event: ScraperRunEvent): string { const base = event.message || - (event.phase ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase()) : "Working"); + (event.phase + ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase()) + : "Working"); if (event.current !== undefined && event.current !== null) { const counter = event.total !== undefined && event.total !== null diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx index cfc75ca8a..9627fb287 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-status-badge.tsx @@ -14,7 +14,10 @@ export function RunStatusBadge({ status }: { status: string }) { } if (normalized === "success") { return ( - + Success ); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx index 6368a1f76..57b7a547c 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx @@ -71,7 +71,8 @@ export function RunsTable({ workspaceId }: { workspaceId: number }) { - View all API runs for this workspace, including runs from the playground, API keys, and agents. + View all API runs for this workspace, including runs from the playground, API keys, and + agents. diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx index 36aee26bb..1d7eec8cf 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx @@ -43,12 +43,7 @@ function FieldControl({ if (field.kind === "boolean") { return ( - + ); } @@ -141,9 +136,7 @@ function FieldRow({ {field.required ? "required" : "optional"}

- {field.description && ( -

{field.description}

- )} + {field.description &&

{field.description}

} { diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 0cf6ef296..4837c0b99 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -1018,10 +1018,7 @@ const ConnectedScraperIcons: FC<{ workspaceId: number }> = ({ workspaceId }) => return ( - + diff --git a/surfsense_web/components/homepage/hero-chat-demo.tsx b/surfsense_web/components/homepage/hero-chat-demo.tsx index e617cd51e..a4a5ebc2a 100644 --- a/surfsense_web/components/homepage/hero-chat-demo.tsx +++ b/surfsense_web/components/homepage/hero-chat-demo.tsx @@ -27,7 +27,8 @@ export type HeroChatDemoScript = { type Stage = "typing" | "steps" | "answer" | "done"; -const PLACEHOLDER = "Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs"; +const PLACEHOLDER = + "Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs"; /** Blinking caret for the typewriter (overlay only, never inside the real input). */ function Caret() { diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 05db2781f..6e31891bd 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -717,10 +717,10 @@ export function HeroSection() { "relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400" )} > - SurfSense is an open-source competitive intelligence platform, like NotebookLM but with - live scraping connectors. Your AI agents monitor competitors, track rankings, and listen - to your market with live data from platforms like Reddit, YouTube, Instagram, TikTok, - Google Maps, Google Search, and the open web. + SurfSense is an open-source competitive intelligence platform, like NotebookLM but + with live scraping connectors. Your AI agents monitor competitors, track rankings, and + listen to your market with live data from platforms like Reddit, YouTube, Instagram, + TikTok, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx b/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx index 4fe035ec2..1e9b1a33e 100644 --- a/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx +++ b/surfsense_web/components/layout/ui/sidebar/NotificationsDropdown.tsx @@ -324,9 +324,7 @@ export function NotificationsDropdown({ )} > {tab.label} - + {formatNotificationCount(tab.count)} diff --git a/surfsense_web/components/settings/auto-reload-settings.tsx b/surfsense_web/components/settings/auto-reload-settings.tsx index ede826dd0..316c5975d 100644 --- a/surfsense_web/components/settings/auto-reload-settings.tsx +++ b/surfsense_web/components/settings/auto-reload-settings.tsx @@ -198,8 +198,8 @@ export function AutoReloadSettings() { Last top-up failed - Your saved card was declined and top-ups were turned off. Update your card and - re-enable top-ups below. + Your saved card was declined and top-ups were turned off. Update your card and re-enable + top-ups below. )} @@ -290,7 +290,11 @@ export function AutoReloadSettings() {
-