From 797dd2df47bcd279a6c4e866cdeb0b2041caac88 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:32 +0200 Subject: [PATCH 01/20] feat: add HttpUrlStr shared URL validator --- .../app/capabilities/core/validation.py | 24 ++++++++++++++ .../tests/unit/capabilities/core/__init__.py | 0 .../unit/capabilities/core/test_validation.py | 32 +++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 surfsense_backend/app/capabilities/core/validation.py create mode 100644 surfsense_backend/tests/unit/capabilities/core/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/core/test_validation.py diff --git a/surfsense_backend/app/capabilities/core/validation.py b/surfsense_backend/app/capabilities/core/validation.py new file mode 100644 index 000000000..3e0885488 --- /dev/null +++ b/surfsense_backend/app/capabilities/core/validation.py @@ -0,0 +1,24 @@ +"""Shared Pydantic field types for capability I/O schemas.""" + +from __future__ import annotations + +from typing import Annotated +from urllib.parse import urlsplit + +import validators +from pydantic import AfterValidator +from pydantic_core import PydanticCustomError + +_HTTP_SCHEMES = frozenset({"http", "https"}) + + +def _validate_http_url(value: str) -> str: + """Accept only well-formed http(s) URLs, returned trimmed and unchanged.""" + url = value.strip() + if not validators.url(url) or urlsplit(url).scheme.lower() not in _HTTP_SCHEMES: + raise PydanticCustomError("http_url", "must be a valid http(s) URL") + return url + + +HttpUrlStr = Annotated[str, AfterValidator(_validate_http_url)] +"""A request URL validated as http(s) and kept as ``str`` (no normalization).""" diff --git a/surfsense_backend/tests/unit/capabilities/core/__init__.py b/surfsense_backend/tests/unit/capabilities/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/core/test_validation.py b/surfsense_backend/tests/unit/capabilities/core/test_validation.py new file mode 100644 index 000000000..03db512ba --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/core/test_validation.py @@ -0,0 +1,32 @@ +"""``HttpUrlStr`` boundary type: accept well-formed http(s) URLs, reject the rest.""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel, ValidationError + +from app.capabilities.core.validation import HttpUrlStr + +pytestmark = pytest.mark.unit + + +class _Model(BaseModel): + urls: list[HttpUrlStr] + + +def test_accepts_http_and_https_urls_unchanged() -> None: + model = _Model(urls=["https://example.com/path?q=1", "http://a.co"]) + assert model.urls == ["https://example.com/path?q=1", "http://a.co"] + + +def test_trims_surrounding_whitespace() -> None: + assert _Model(urls=[" https://example.com "]).urls == ["https://example.com"] + + +@pytest.mark.parametrize( + "bad", + ["not-a-url", "example.com", "ftp://example.com", "http://", "javascript:alert(1)"], +) +def test_rejects_malformed_or_non_http_urls(bad: str) -> None: + with pytest.raises(ValidationError): + _Model(urls=[bad]) From a123c95703b29f5ebfdf35d3ce0b187fa4367b6d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:32 +0200 Subject: [PATCH 02/20] feat: validate web.crawl start URLs --- surfsense_backend/app/capabilities/web/crawl/schemas.py | 4 +++- .../tests/unit/capabilities/web/crawl/test_schemas.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/capabilities/web/crawl/schemas.py b/surfsense_backend/app/capabilities/web/crawl/schemas.py index d6c9c38a3..30e8ff6ae 100644 --- a/surfsense_backend/app/capabilities/web/crawl/schemas.py +++ b/surfsense_backend/app/capabilities/web/crawl/schemas.py @@ -18,6 +18,8 @@ from typing import Literal from pydantic import BaseModel, Field +from app.capabilities.core.validation import HttpUrlStr + MAX_START_URLS = 20 """Per-call cap on seed URLs: bounds a synchronous request's fan-out (05).""" @@ -29,7 +31,7 @@ MAX_CRAWL_PAGES = 200 class CrawlInput(BaseModel): - startUrls: list[str] = Field( + startUrls: list[HttpUrlStr] = Field( min_length=1, max_length=MAX_START_URLS, description=( diff --git a/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py b/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py index f7b9a2b39..080dda2b2 100644 --- a/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py @@ -20,6 +20,11 @@ def test_requires_at_least_one_start_url() -> None: CrawlInput(startUrls=[]) +def test_rejects_malformed_start_url() -> None: + with pytest.raises(ValidationError): + CrawlInput(startUrls=["not-a-url"]) + + def test_camelcase_fields_and_defaults() -> None: model = CrawlInput(startUrls=["https://e.com"]) assert model.startUrls == ["https://e.com"] From 1996d3e54d194fee50ddb0d22f64e8b5e91703bb Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:32 +0200 Subject: [PATCH 03/20] feat: validate amazon scrape URLs --- surfsense_backend/app/capabilities/amazon/scrape/schemas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/capabilities/amazon/scrape/schemas.py b/surfsense_backend/app/capabilities/amazon/scrape/schemas.py index e510e2eea..b2a847058 100644 --- a/surfsense_backend/app/capabilities/amazon/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/amazon/scrape/schemas.py @@ -4,6 +4,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.amazon import ProductItem MAX_AMAZON_SOURCES = 20 @@ -13,7 +14,7 @@ MAX_AMAZON_RESULTS = 1000 class ScrapeInput(BaseModel): """Agent-facing controls for public product discovery and enrichment.""" - urls: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES) + urls: list[HttpUrlStr] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES) search_terms: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES) max_items: int = Field(default=10, ge=1, le=100) domain: str = Field( From 4e4b9236c3f02f658c5b852011a7fdde469bfcf3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:32 +0200 Subject: [PATCH 04/20] feat: validate walmart scrape and reviews URLs --- surfsense_backend/app/capabilities/walmart/reviews/schemas.py | 3 ++- surfsense_backend/app/capabilities/walmart/scrape/schemas.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py index faf56ed5b..bd1dc73d9 100644 --- a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py +++ b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py @@ -11,13 +11,14 @@ from typing import Literal from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.walmart import ReviewItem MAX_WALMART_REVIEW_SOURCES = 20 class ReviewsInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_WALMART_REVIEW_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py index 2d42dc371..7faaabd1d 100644 --- a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py @@ -6,6 +6,7 @@ from urllib.parse import quote_plus from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.walmart import ProductItem MAX_WALMART_SOURCES = 20 @@ -15,7 +16,7 @@ MAX_WALMART_RESULTS = 1000 class ScrapeInput(BaseModel): """Agent-facing controls for public Walmart product discovery and enrichment.""" - urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) + urls: list[HttpUrlStr] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) search_terms: list[str] = Field( default_factory=list, max_length=MAX_WALMART_SOURCES ) From babefecee788b2580ece89ae0e3e824921ef5d43 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:32 +0200 Subject: [PATCH 05/20] feat: validate youtube scrape and comments URLs --- surfsense_backend/app/capabilities/youtube/comments/schemas.py | 3 ++- surfsense_backend/app/capabilities/youtube/scrape/schemas.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/capabilities/youtube/comments/schemas.py b/surfsense_backend/app/capabilities/youtube/comments/schemas.py index a9da5b7a2..ecfa80026 100644 --- a/surfsense_backend/app/capabilities/youtube/comments/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/comments/schemas.py @@ -10,6 +10,7 @@ from typing import Literal from pydantic import BaseModel, Field +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.youtube import CommentItem MAX_COMMENT_VIDEOS = 20 @@ -17,7 +18,7 @@ MAX_COMMENT_VIDEOS = 20 class CommentsInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( min_length=1, max_length=MAX_COMMENT_VIDEOS, description="YouTube video URLs to fetch comments (and replies) for (1-20).", diff --git a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py index 4d80769b6..63a1df060 100644 --- a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.youtube import VideoItem MAX_YOUTUBE_SOURCES = 20 @@ -17,7 +18,7 @@ MAX_YOUTUBE_SOURCES = 20 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_YOUTUBE_SOURCES, description=( From 96f05991eb73f861d519fbfa0c4abef2eb22fc48 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:38 +0200 Subject: [PATCH 06/20] feat: validate reddit scrape URLs --- surfsense_backend/app/capabilities/reddit/scrape/schemas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/capabilities/reddit/scrape/schemas.py b/surfsense_backend/app/capabilities/reddit/scrape/schemas.py index 0c2da536e..9e0143ba7 100644 --- a/surfsense_backend/app/capabilities/reddit/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/reddit/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.reddit import RedditItem from app.proprietary.platforms.reddit.schemas import RedditSort, RedditTime @@ -21,7 +22,7 @@ MAX_REDDIT_ITEMS = 100 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_REDDIT_SOURCES, description=( From a5ccd403879eaa23c91680e9604caf0166005758 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:39 +0200 Subject: [PATCH 07/20] feat: validate tiktok scrape and comments URLs --- surfsense_backend/app/capabilities/tiktok/comments/schemas.py | 3 ++- surfsense_backend/app/capabilities/tiktok/scrape/schemas.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py index f3c68d445..61ae0c756 100644 --- a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field +from app.capabilities.core.validation import HttpUrlStr from app.capabilities.tiktok.scrape.schemas import ( MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES, @@ -18,7 +19,7 @@ from app.proprietary.platforms.tiktok import CommentItem class CommentsInput(BaseModel): - video_urls: list[str] = Field( + video_urls: list[HttpUrlStr] = Field( min_length=1, max_length=MAX_TIKTOK_SOURCES, description="TikTok video URLs (/@/video/) to pull comments from.", diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index ac3792712..b139e6e49 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -12,6 +12,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.tiktok import TikTokVideoItem MAX_TIKTOK_SOURCES = 20 @@ -22,7 +23,7 @@ MAX_TIKTOK_ITEMS = 100 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_TIKTOK_SOURCES, description=( From 9e04eb2b892ef621f60d8d1389b2eeb1ebed3cca Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:39 +0200 Subject: [PATCH 08/20] feat: validate google maps scrape and reviews URLs --- .../app/capabilities/google_maps/reviews/schemas.py | 3 ++- .../app/capabilities/google_maps/scrape/schemas.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py b/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py index 1868765e9..26cdad29e 100644 --- a/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py +++ b/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py @@ -10,6 +10,7 @@ from typing import Literal from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.google_maps import ReviewItem MAX_MAPS_REVIEW_SOURCES = 20 @@ -17,7 +18,7 @@ MAX_MAPS_REVIEW_SOURCES = 20 class ReviewsInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_MAPS_REVIEW_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py b/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py index 95833c8f7..7cf1ca6a1 100644 --- a/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.google_maps import PlaceItem MAX_MAPS_SOURCES = 20 @@ -26,7 +27,7 @@ class ScrapeInput(BaseModel): "(at least one is required). Pair with location to scope a search." ), ) - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_MAPS_SOURCES, description=( From 574b40be40282e3fc8bb49209ca89bb903f7081a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:39 +0200 Subject: [PATCH 09/20] feat: validate indeed scrape URLs --- surfsense_backend/app/capabilities/indeed/scrape/schemas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py index 02e29100d..7c5dacb59 100644 --- a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.indeed_jobs import IndeedItem from app.proprietary.platforms.indeed_jobs.schemas import ( IndeedJobType, @@ -26,7 +27,7 @@ MAX_INDEED_ITEMS = 100 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_INDEED_SOURCES, description=( From 4dddd55512aba32525287f0d90c2448f87823750 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:39 +0200 Subject: [PATCH 10/20] feat: return structured field errors on 422 --- surfsense_backend/app/app.py | 51 +++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index dde2ce7fa..de3b066f6 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -8,6 +8,7 @@ from collections import defaultdict from contextlib import asynccontextmanager from datetime import UTC, datetime from threading import Lock +from typing import Any import redis from fastapi import Depends, FastAPI, HTTPException, Request, Response, status @@ -94,17 +95,21 @@ def _build_error_response( code: str = "INTERNAL_ERROR", request_id: str = "", extra_headers: dict[str, str] | None = None, + fields: list[dict[str, Any]] | None = None, ) -> JSONResponse: """Build the standardized error envelope (new ``error`` + legacy ``detail``).""" + error: dict[str, Any] = { + "code": code, + "message": message, + "status": status_code, + "request_id": request_id, + "timestamp": datetime.now(UTC).isoformat(), + "report_url": ISSUES_URL, + } + if fields: + error["fields"] = fields body = { - "error": { - "code": code, - "message": message, - "status": status_code, - "request_id": request_id, - "timestamp": datetime.now(UTC).isoformat(), - "report_url": ISSUES_URL, - }, + "error": error, "detail": message, } headers = {"X-Request-ID": request_id} @@ -222,16 +227,30 @@ def _http_exception_handler(request: Request, exc: HTTPException) -> JSONRespons def _validation_error_handler( request: Request, exc: RequestValidationError ) -> JSONResponse: - """Return 422 with field-level detail in the standard envelope.""" + """Return 422 with field-level detail in the standard envelope. + + ``error.fields`` carries each failure's location path and message so clients + can attach errors to the offending input; ``message`` is the flat summary. + """ rid = _get_request_id(request) - fields = [] - for err in exc.errors(): - loc = " -> ".join(str(part) for part in err.get("loc", [])) - fields.append(f"{loc}: {err.get('msg', 'invalid')}") - message = ( - f"Validation failed: {'; '.join(fields)}" if fields else "Validation failed." + fields = [ + { + "loc": [str(part) for part in err.get("loc", ())], + # Drop pydantic's "Value error, " prefix so messages read for humans. + "msg": str(err.get("msg", "invalid")).removeprefix("Value error, "), + } + for err in exc.errors() + ] + + def _segment(field: dict[str, Any]) -> str: + loc = " -> ".join(field["loc"]) + return f"{loc}: {field['msg']}" if loc else field["msg"] + + summary = "; ".join(_segment(f) for f in fields) + message = f"Validation failed: {summary}" if fields else "Validation failed." + return _build_error_response( + 422, message, code="VALIDATION_ERROR", request_id=rid, fields=fields ) - return _build_error_response(422, message, code="VALIDATION_ERROR", request_id=rid) def _unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: From 6330d5a789d53b53540f167273f93d384206631d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:50 +0200 Subject: [PATCH 11/20] fix: reject spoofed non-YouTube URLs --- .../platforms/youtube/url_resolver.py | 19 +++++++++++++------ .../unit/platforms/youtube/test_parsers.py | 18 ++++++++++++++++-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py b/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py index 0e68425a9..aa7df36e9 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py @@ -15,6 +15,9 @@ ResolvedKind = Literal["video", "channel", "playlist", "hashtag", "search"] _PLAYLIST_ID_RE = re.compile(r"[?&]list=([\w-]+)") +_YOUTUBE_HOSTS = frozenset({"www.youtube.com", "youtube.com", "m.youtube.com"}) +"""Hosts whose page paths (/@, /channel, /playlist, ...) are trusted as YouTube.""" + @dataclass(frozen=True) class ResolvedUrl: @@ -29,7 +32,7 @@ def get_youtube_video_id(url: str) -> str | None: hostname = parsed.hostname or "" if hostname == "youtu.be": return parsed.path[1:] or None - if hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"): + if hostname in _YOUTUBE_HOSTS: if parsed.path == "/watch": return parse_qs(parsed.query).get("v", [None])[0] for prefix in ("/embed/", "/v/", "/shorts/"): @@ -41,17 +44,21 @@ def get_youtube_video_id(url: str) -> str | None: def resolve_url(url: str) -> ResolvedUrl | None: """Classify a YouTube URL into a scrape job, or ``None`` if unrecognized.""" parsed = urlparse(url) + hostname = (parsed.hostname or "").lower() path = parsed.path or "" - # Shorts are videos with their own path. - if "/shorts/" in path: - vid = path.split("/shorts/")[1].split("/")[0] - return ResolvedUrl("video", vid, url) if vid else None - + # Videos: watch / youtu.be / embed / shorts. get_youtube_video_id validates + # the host itself, so this also covers youtu.be short links. video_id = get_youtube_video_id(url) if video_id: return ResolvedUrl("video", video_id, url) + # Every remaining shape is a youtube.com page keyed off its path. Require a + # YouTube host so a well-formed non-YouTube URL (e.g. https://evil.com/@x) + # is rejected instead of misclassified as a channel by its path alone. + if hostname not in _YOUTUBE_HOSTS: + return None + # Playlist (either a /playlist page or any URL carrying ?list=). playlist_match = _PLAYLIST_ID_RE.search(url) if path.startswith("/playlist") and playlist_match: diff --git a/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py index 55f591c63..6b7b460a9 100644 --- a/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py @@ -946,8 +946,22 @@ def test_resolve_url(url, kind, value): assert resolved.value == value -def test_resolve_url_unrecognized(): - assert resolve_url("https://example.com/foo") is None +@pytest.mark.parametrize( + "url", + [ + "https://example.com/foo", + # Well-formed non-YouTube URLs whose path mimics a YouTube page must be + # rejected, not misclassified by path alone (host-spoof guard). + "https://evil.com/@Apify", + "https://evil.com/channel/UC123456789abc", + "https://evil.com/shorts/abc123", + "https://evil.com/playlist?list=PL123", + "https://evil.com/hashtag/tech", + "https://evil.com/results?search_query=web+scraping", + ], +) +def test_resolve_url_unrecognized(url): + assert resolve_url(url) is None # --- optional: exercise captured real fixtures if present -------------------- From 18f5d30e6105b29b04474bb1dbe95062320558e5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:50 +0200 Subject: [PATCH 12/20] feat: carry validation field errors in AppError --- surfsense_web/lib/apis/base-api.service.ts | 11 +++++++++-- surfsense_web/lib/error.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 31341a9e6..872c885d9 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -10,6 +10,7 @@ import { AuthorizationError, NetworkError, NotFoundError, + type ValidationFieldError, } from "../error"; enum ResponseType { @@ -165,6 +166,9 @@ class BaseApiService { const requestId: string | undefined = envelope?.request_id ?? response.headers.get("X-Request-ID") ?? undefined; const reportUrl: string | undefined = envelope?.report_url; + const validationFields: ValidationFieldError[] | undefined = Array.isArray(envelope?.fields) + ? envelope.fields + : undefined; // Handle 401 - try to refresh token first (only once) if (response.status === 401) { @@ -209,8 +213,8 @@ class BaseApiService { response.status, response.statusText ); - default: - throw new AppError( + default: { + const appError = new AppError( errorMessage || "Something went wrong", response.status, response.statusText, @@ -218,6 +222,9 @@ class BaseApiService { requestId, reportUrl ); + appError.fields = validationFields; + throw appError; + } } } refreshRetryBlockedUntil.delete(getRefreshRetryKey(mergedOptions.method, url)); diff --git a/surfsense_web/lib/error.ts b/surfsense_web/lib/error.ts index 10cb3706e..8c87007b8 100644 --- a/surfsense_web/lib/error.ts +++ b/surfsense_web/lib/error.ts @@ -1,11 +1,19 @@ export const SURFSENSE_ISSUES_URL = "https://github.com/MODSetter/SurfSense/issues"; +/** One field-level failure from a 422 validation response. */ +export interface ValidationFieldError { + loc: string[]; + msg: string; +} + export class AppError extends Error { status?: number; statusText?: string; code?: string; requestId?: string; reportUrl?: string; + /** Per-field failures from a 422 response, keyed by their location path. */ + fields?: ValidationFieldError[]; constructor( message: string, status?: number, From d7f89bb3fd0f3b32e4f3a6906540848901cf5d48 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:50 +0200 Subject: [PATCH 13/20] feat: map 422 field errors for playground forms --- .../lib/playground/field-errors.test.ts | 34 +++++++++++++++++++ surfsense_web/lib/playground/field-errors.ts | 18 ++++++++++ 2 files changed, 52 insertions(+) create mode 100644 surfsense_web/lib/playground/field-errors.test.ts create mode 100644 surfsense_web/lib/playground/field-errors.ts diff --git a/surfsense_web/lib/playground/field-errors.test.ts b/surfsense_web/lib/playground/field-errors.test.ts new file mode 100644 index 000000000..273cf1cc3 --- /dev/null +++ b/surfsense_web/lib/playground/field-errors.test.ts @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { AppError } from "@/lib/error"; +import { fieldErrorsFromError } from "./field-errors"; + +// Run with: pnpm exec tsx --test lib/playground/field-errors.test.ts + +function validationError(fields: { loc: string[]; msg: string }[]): AppError { + const error = new AppError("Validation failed", 422, "Unprocessable Entity", "VALIDATION_ERROR"); + error.fields = fields; + return error; +} + +test("maps a body field failure to its top-level field name", () => { + const error = validationError([ + { loc: ["body", "startUrls", "0"], msg: "must be a valid http(s) URL" }, + ]); + assert.deepEqual(fieldErrorsFromError(error), { + startUrls: "must be a valid http(s) URL", + }); +}); + +test("keeps the first failure per field", () => { + const error = validationError([ + { loc: ["body", "urls", "0"], msg: "first" }, + { loc: ["body", "urls", "3"], msg: "second" }, + ]); + assert.deepEqual(fieldErrorsFromError(error), { urls: "first" }); +}); + +test("returns nothing for non-AppError or errors without fields", () => { + assert.deepEqual(fieldErrorsFromError(new Error("boom")), {}); + assert.deepEqual(fieldErrorsFromError(new AppError("no fields", 500)), {}); +}); diff --git a/surfsense_web/lib/playground/field-errors.ts b/surfsense_web/lib/playground/field-errors.ts new file mode 100644 index 000000000..668d2d9c1 --- /dev/null +++ b/surfsense_web/lib/playground/field-errors.ts @@ -0,0 +1,18 @@ +import { AppError } from "@/lib/error"; + +/** Map a 422 response's field failures to ``{ fieldName: message }`` for the form. */ +export function fieldErrorsFromError(error: unknown): Record { + if (!(error instanceof AppError) || !error.fields) return {}; + const errors: Record = {}; + for (const { loc, msg } of error.fields) { + const name = topLevelField(loc); + if (name && !errors[name]) errors[name] = msg; + } + return errors; +} + +/** The input field a location path points at, dropping the ``body`` request root. */ +function topLevelField(loc: string[]): string | undefined { + const path = loc[0] === "body" ? loc.slice(1) : loc; + return path[0]; +} From 1ad8efb6587b64fe2716b1d1c4e13379bd891d49 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:50 +0200 Subject: [PATCH 14/20] feat: add per-platform URL hints --- .../lib/playground/url-hints.test.ts | 52 +++++++++++ surfsense_web/lib/playground/url-hints.ts | 90 +++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 surfsense_web/lib/playground/url-hints.test.ts create mode 100644 surfsense_web/lib/playground/url-hints.ts diff --git a/surfsense_web/lib/playground/url-hints.test.ts b/surfsense_web/lib/playground/url-hints.test.ts new file mode 100644 index 000000000..84570c8e9 --- /dev/null +++ b/surfsense_web/lib/playground/url-hints.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { urlFieldWarning, urlFieldWarnings } from "./url-hints"; + +// Run with: pnpm exec tsx --test lib/playground/url-hints.test.ts + +test("accepts a matching platform URL (no warning)", () => { + assert.equal(urlFieldWarning("youtube", "urls", "https://www.youtube.com/watch?v=x"), undefined); + assert.equal(urlFieldWarning("youtube", "urls", "https://youtu.be/x"), undefined); +}); + +test("flags a well-formed URL for the wrong platform", () => { + assert.equal( + urlFieldWarning("youtube", "urls", "https://google.com"), + "Not a YouTube URL: https://google.com" + ); +}); + +test("flags a malformed / scheme-less URL", () => { + assert.equal( + urlFieldWarning("reddit", "urls", "reddit.com/r/python"), + "Not a Reddit URL: reddit.com/r/python" + ); +}); + +test("lists every offending line, ignoring blanks", () => { + const value = "https://www.tiktok.com/@a\n\n https://google.com \nnot-a-url"; + assert.equal( + urlFieldWarning("tiktok", "urls", value), + "Not a TikTok URL: https://google.com, not-a-url" + ); +}); + +test("matches marketplace/shortlink host variants", () => { + assert.equal(urlFieldWarning("amazon", "urls", "https://www.amazon.fr/dp/B0"), undefined); + assert.equal(urlFieldWarning("amazon", "urls", "https://a.co/d/xyz"), undefined); + assert.equal(urlFieldWarning("google_maps", "urls", "https://maps.app.goo.gl/xyz"), undefined); +}); + +test("only inspects known URL fields, and skips platforms without a rule", () => { + // search_terms is not a URL field — never warned. + assert.equal(urlFieldWarning("amazon", "search_terms", "laptop"), undefined); + // instagram has no rule (its urls accept bare @handles). + assert.equal(urlFieldWarning("instagram", "urls", "natgeo"), undefined); +}); + +test("collects per-field warnings across the form values", () => { + assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://google.com", country: "us" }), { + urls: "Not an Indeed URL: https://google.com", + }); + assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://www.indeed.com/jobs?q=x" }), {}); +}); diff --git a/surfsense_web/lib/playground/url-hints.ts b/surfsense_web/lib/playground/url-hints.ts new file mode 100644 index 000000000..e45a5d8ea --- /dev/null +++ b/surfsense_web/lib/playground/url-hints.ts @@ -0,0 +1,90 @@ +/** + * Client-side, per-platform URL hints for the playground (UX only). + * + * The scraper API stays authoritative: it rejects malformed URLs (422), and + * each platform decides what it can actually scrape. These hints only *warn*, + * before a run, when a line in a platform's ``urls`` field is not a URL for + * that platform — so a typo (wrong site, missing scheme) is caught without a + * round-trip. They never block the run. + */ + +interface PlatformUrlRule { + /** Host suffixes (``youtube.com``) or ``.``-terminated prefixes (``amazon.``). */ + hosts: string[]; + /** Human platform name used in the warning message. */ + label: string; +} + +/** + * Keyed by platform slug. Instagram is intentionally absent: its ``urls`` field + * also accepts bare ``@handles``, so a non-URL line there is not a mistake. + */ +const PLATFORM_URL_RULES: Record = { + amazon: { hosts: ["amazon.", "amzn.to", "a.co"], label: "Amazon" }, + walmart: { hosts: ["walmart.com"], label: "Walmart" }, + reddit: { hosts: ["reddit.com", "redd.it"], label: "Reddit" }, + youtube: { hosts: ["youtube.com", "youtu.be"], label: "YouTube" }, + tiktok: { hosts: ["tiktok.com"], label: "TikTok" }, + google_maps: { hosts: ["google.", "goo.gl"], label: "Google Maps" }, + indeed: { hosts: ["indeed.com"], label: "Indeed" }, +}; + +/** The array fields that carry platform URLs at the capability layer. */ +const URL_FIELD_NAMES = new Set(["urls", "video_urls", "startUrls"]); + +function hostMatches(host: string, patterns: string[]): boolean { + return patterns.some((pattern) => + pattern.endsWith(".") + ? host.includes(pattern) + : host === pattern || host.endsWith(`.${pattern}`) + ); +} + +function article(word: string): "a" | "an" { + return /^[aeiou]/i.test(word) ? "an" : "a"; +} + +function isPlatformUrl(line: string, rule: PlatformUrlRule): boolean { + try { + const { protocol, hostname } = new URL(line); + if (protocol !== "http:" && protocol !== "https:") return false; + return hostMatches(hostname.toLowerCase(), rule.hosts); + } catch { + return false; + } +} + +/** + * Warn if any line in a platform ``urls`` field is not a URL for that platform. + * Returns ``undefined`` when the field/platform has no rule or every line is OK. + */ +export function urlFieldWarning( + platform: string, + fieldName: string, + value: unknown +): string | undefined { + const rule = PLATFORM_URL_RULES[platform]; + if (!rule || !URL_FIELD_NAMES.has(fieldName)) return undefined; + + const badLines = String(value ?? "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .filter((line) => !isPlatformUrl(line, rule)); + + if (badLines.length === 0) return undefined; + return `Not ${article(rule.label)} ${rule.label} URL: ${badLines.join(", ")}`; +} + +/** Per-field warnings for the current form values (empty when every URL looks valid). */ +export function urlFieldWarnings( + platform: string, + values: Record +): Record { + const warnings: Record = {}; + for (const [name, value] of Object.entries(values)) { + const warning = urlFieldWarning(platform, name, value); + if (warning) warnings[name] = warning; + } + return warnings; +} From 15962d205f8bbd09d92014850b99223fe113d27d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:50 +0200 Subject: [PATCH 15/20] feat: surface validation feedback in playground UI --- .../components/playground-runner.tsx | 20 ++++++++++++- .../playground/components/schema-form.tsx | 8 +++++ .../components/webcrawler-config.tsx | 30 ++++++++++++++++--- surfsense_web/lib/url.ts | 10 +++++++ 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx index 32624ca55..6fbf7b98b 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx @@ -12,6 +12,7 @@ import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; import { scrapersApiService } from "@/lib/apis/scrapers-api.service"; import { AppError } from "@/lib/error"; import { findVerb } from "@/lib/playground/catalog"; +import { fieldErrorsFromError } from "@/lib/playground/field-errors"; import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format"; import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema"; import { @@ -19,6 +20,7 @@ import { getAmazonFieldOptions, hasAmazonFranceValue, } from "@/lib/playground/platform-overrides/amazon"; +import { urlFieldWarnings } from "@/lib/playground/url-hints"; import { ApiReference } from "./api-reference"; import { OutputViewer } from "./output-viewer"; import { RunProgressPanel } from "./run-progress-panel"; @@ -73,7 +75,12 @@ function getRunErrorMessage(error: unknown): string { } if (status === 422) { - return "Invalid input. Check the fields above and try again."; + if (Object.keys(fieldErrorsFromError(error)).length > 0) { + return "Invalid input. Check the highlighted fields."; + } + return error instanceof Error && error.message + ? error.message + : "Invalid input. Check the fields above and try again."; } return error instanceof Error && error.message @@ -123,6 +130,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn const fields = useMemo(() => parseSchemaFields(capability?.input_schema), [capability]); const [values, setValues] = useState>({}); + const [fieldErrors, setFieldErrors] = useState>({}); const run = useRunStream(workspaceId); const isRunning = run.status === "running"; const previousStatusRef = useRef(run.status); @@ -164,9 +172,15 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn const handleChange = (name: string, value: unknown) => { setValues((prev) => ({ ...prev, [name]: value })); + setFieldErrors((prev) => { + if (!(name in prev)) return prev; + const { [name]: _cleared, ...rest } = prev; + return rest; + }); }; const handleRun = useCallback(() => { + setFieldErrors({}); const payload = buildPayload(fields, values); void run.start(platform, verb, payload); }, [fields, values, platform, verb, run]); @@ -178,6 +192,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn const endpoint = `POST /workspaces/${workspaceId}/scrapers/${platform}/${verb}`; const isAmazonScrape = platform === "amazon" && verb === "scrape"; const hasAmazonFranceUrl = useMemo(() => hasAmazonFranceValue(values), [values]); + const fieldWarnings = useMemo(() => urlFieldWarnings(platform, values), [platform, values]); useEffect(() => { const previousStatus = previousStatusRef.current; @@ -194,6 +209,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn } if (run.status === "error") { + setFieldErrors(fieldErrorsFromError(run.error)); const key = `${run.runId ?? "run"}:error`; if (notifiedRunRef.current === key) return; notifiedRunRef.current = key; @@ -268,6 +284,8 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn onChange={handleChange} disabled={isRunning} getFieldOptions={isAmazonScrape ? getAmazonFieldOptions : undefined} + fieldErrors={fieldErrors} + fieldWarnings={fieldWarnings} /> {isAmazonScrape ? : null} diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx index cdd425925..c86dd9839 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx @@ -32,6 +32,8 @@ interface SchemaFormProps { getFieldOptions?: FieldOptionsResolver; /** Field names flagged by a 422 response, shown with error styling. */ fieldErrors?: Record; + /** Client-side, non-blocking URL warnings keyed by field name. */ + fieldWarnings?: Record; } function FieldControl({ @@ -147,6 +149,7 @@ function FieldRow({ onChange, disabled, error, + warning, options, }: { field: FormField; @@ -154,6 +157,7 @@ function FieldRow({ onChange: (value: unknown) => void; disabled?: boolean; error?: string; + warning?: string; options?: FieldOption[]; }) { return ( @@ -179,6 +183,7 @@ function FieldRow({ options={options} /> {error &&

{error}

} + {!error && warning &&

{warning}

} ); } @@ -190,6 +195,7 @@ export function SchemaForm({ disabled, getFieldOptions, fieldErrors, + fieldWarnings, }: SchemaFormProps) { const [showAdvanced, setShowAdvanced] = useState(false); @@ -209,6 +215,7 @@ export function SchemaForm({ onChange={(value) => onChange(field.name, value)} disabled={disabled} error={fieldErrors?.[field.name]} + warning={fieldWarnings?.[field.name]} options={getFieldOptions?.(field)} /> ))} @@ -239,6 +246,7 @@ export function SchemaForm({ onChange={(value) => onChange(field.name, value)} disabled={disabled} error={fieldErrors?.[field.name]} + warning={fieldWarnings?.[field.name]} options={getFieldOptions?.(field)} /> ))} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx index b7fbae91c..5a4540bcf 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx @@ -8,8 +8,18 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; +import { isHttpUrl } from "@/lib/url"; +import { cn } from "@/lib/utils"; import type { ConnectorConfigProps } from "../index"; +/** Return the malformed (non-http/https) lines from a newline-separated URL list. */ +function invalidUrlLines(value: string): string[] { + return value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !isHttpUrl(line)); +} + export const WebcrawlerConfig: FC = ({ connector, onConfigChange }) => { // Initialize with existing config values const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; @@ -19,6 +29,8 @@ export const WebcrawlerConfig: FC = ({ connector, onConfig const [initialUrls, setInitialUrls] = useState(existingUrls); const [showApiKey, setShowApiKey] = useState(false); + const invalidUrls = invalidUrlLines(initialUrls); + const handleApiKeyChange = (value: string) => { setApiKey(value); if (onConfigChange) { @@ -108,11 +120,21 @@ export const WebcrawlerConfig: FC = ({ connector, onConfig placeholder="https://example.com https://docs.example.com https://blog.example.com" value={initialUrls} onChange={(e) => handleUrlsChange(e.target.value)} - className="min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none" + aria-invalid={invalidUrls.length > 0} + className={cn( + "min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none", + invalidUrls.length > 0 && "border-destructive" + )} /> -

- Enter URLs to crawl (one per line). You can add more URLs later. -

+ {invalidUrls.length > 0 ? ( +

+ Not valid http(s) URLs: {invalidUrls.join(", ")} +

+ ) : ( +

+ Enter URLs to crawl (one per line). You can add more URLs later. +

+ )} {/* Info Alert */} diff --git a/surfsense_web/lib/url.ts b/surfsense_web/lib/url.ts index 7bb0488d0..c351d1b18 100644 --- a/surfsense_web/lib/url.ts +++ b/surfsense_web/lib/url.ts @@ -12,3 +12,13 @@ export function tryGetHostname(url: string): string | undefined { return undefined; } } + +/** True when the value parses as an http(s) URL — mirrors the backend's boundary rule. */ +export function isHttpUrl(value: string): boolean { + try { + const { protocol } = new URL(value); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} From 0118744c20380e0efc1cb82c83572b73df21fb55 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 19:39:15 +0200 Subject: [PATCH 16/20] fix: strip 'body' root from validation error summary --- surfsense_backend/app/app.py | 7 ++- .../tests/unit/test_error_contract.py | 44 ++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index de3b066f6..9f88fe02a 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -243,7 +243,12 @@ def _validation_error_handler( ] def _segment(field: dict[str, Any]) -> str: - loc = " -> ".join(field["loc"]) + # Drop the "body" request root so model-level errors read as a plain + # sentence and field errors read as "field -> sub", not "body -> field". + path = field["loc"] + if path and path[0] == "body": + path = path[1:] + loc = " -> ".join(path) return f"{loc}: {field['msg']}" if loc else field["msg"] summary = "; ".join(_segment(f) for f in fields) diff --git a/surfsense_backend/tests/unit/test_error_contract.py b/surfsense_backend/tests/unit/test_error_contract.py index ec8021290..e1e160232 100644 --- a/surfsense_backend/tests/unit/test_error_contract.py +++ b/surfsense_backend/tests/unit/test_error_contract.py @@ -14,6 +14,7 @@ import json import pytest from fastapi import HTTPException +from pydantic import BaseModel, model_validator from starlette.testclient import TestClient from app.exceptions import ( @@ -32,6 +33,23 @@ from app.exceptions import ( pytestmark = pytest.mark.unit +class _RequireOne(BaseModel): + """Body model with a cross-field rule, used to test the 422 summary. + + Defined at module scope (not inside the app factory) so FastAPI's ``Body`` + ``TypeAdapter`` can resolve the forward reference. + """ + + a: str | None = None + b: str | None = None + + @model_validator(mode="after") + def _at_least_one(self): + if not self.a and not self.b: + raise ValueError("Provide at least one of 'a' or 'b'.") + return self + + # --------------------------------------------------------------------------- # Helpers - lightweight FastAPI app that re-uses the real global handlers # --------------------------------------------------------------------------- @@ -39,7 +57,7 @@ pytestmark = pytest.mark.unit def _make_test_app(): """Build a minimal FastAPI app with the same handlers as the real one.""" - from fastapi import FastAPI + from fastapi import Body, FastAPI from fastapi.exceptions import RequestValidationError from pydantic import BaseModel @@ -124,6 +142,10 @@ def _make_test_app(): async def validated(item: Item): return item.model_dump() + @app.post("/require-one") + async def require_one(payload: _RequireOne = Body(...)): + return payload.model_dump() + return app @@ -277,6 +299,26 @@ class TestValidationErrorHandler: body = _assert_envelope(resp, 422) assert body["error"]["code"] == "VALIDATION_ERROR" + def test_field_error_drops_body_root(self, client): + # The "body" request root is noise in the human summary; the field name + # is kept so clients can still attach the error inline. + resp = client.post("/require-one", json={"a": 123}) + body = _assert_envelope(resp, 422) + msg = body["error"]["message"] + assert "body" not in msg + assert "a:" in msg + assert body["error"]["fields"][0]["loc"] == ["body", "a"] + + def test_model_level_error_reads_as_sentence(self, client): + # A cross-field rule (loc == ["body"]) must read as a plain sentence, + # not "body: ...", while fields still carry the location for clients. + resp = client.post("/require-one", json={}) + body = _assert_envelope(resp, 422) + assert body["error"]["message"] == ( + "Validation failed: Provide at least one of 'a' or 'b'." + ) + assert body["error"]["fields"][0]["loc"] == ["body"] + # --------------------------------------------------------------------------- # SurfSenseError class hierarchy unit tests From 55676cfe942f1b84b1735a6c8f268d0e1d3362df Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 19:39:15 +0200 Subject: [PATCH 17/20] fix: skip logging and telemetry for handled 4xx errors --- surfsense_web/lib/apis/base-api.service.ts | 46 ++++++++++++++-------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 872c885d9..867dea93e 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -293,24 +293,36 @@ class BaseApiService { ); } - console.error("Request failed:", JSON.stringify(error)); - if (!(error instanceof AuthenticationError)) { - import("posthog-js") - .then(({ default: posthog }) => { - posthog.captureException(error, { - api_url: url, - api_method: options?.method ?? "GET", - ...(error instanceof AppError && { - status_code: error.status, - status_text: error.statusText, - error_code: error.code, - request_id: error.requestId, - }), + // Handled client errors (validation, credits, auth, not-found, ...) are + // expected outcomes surfaced to the user via typed errors + toasts — not + // app faults. Skip logging/telemetry for them so they don't spam the + // console, PostHog, or Next.js's dev error overlay. + const isHandledClientError = + error instanceof AppError && + typeof error.status === "number" && + error.status >= 400 && + error.status < 500; + + if (!isHandledClientError) { + console.error("Request failed:", JSON.stringify(error)); + if (!(error instanceof AuthenticationError)) { + import("posthog-js") + .then(({ default: posthog }) => { + posthog.captureException(error, { + api_url: url, + api_method: options?.method ?? "GET", + ...(error instanceof AppError && { + status_code: error.status, + status_text: error.statusText, + error_code: error.code, + request_id: error.requestId, + }), + }); + }) + .catch(() => { + console.error("Failed to capture exception in PostHog"); }); - }) - .catch(() => { - console.error("Failed to capture exception in PostHog"); - }); + } } throw error; } From 44149323466b0ad46f0cae6da3e66dc4c755d2f4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 19:39:15 +0200 Subject: [PATCH 18/20] feat: surface playground run errors inline and persist toasts --- .../components/playground-runner.tsx | 14 +++++++++++-- .../playground/components/schema-form.tsx | 21 ++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx index 6fbf7b98b..6bee3e4e5 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx @@ -209,11 +209,21 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn } if (run.status === "error") { - setFieldErrors(fieldErrorsFromError(run.error)); + const nextFieldErrors = fieldErrorsFromError(run.error); + setFieldErrors(nextFieldErrors); const key = `${run.runId ?? "run"}:error`; if (notifiedRunRef.current === key) return; notifiedRunRef.current = key; - toast.error(getRunErrorMessage(run.error)); + // Field-level failures are shown inline (the form reveals + focuses the + // first one), so only toast global failures that have no field to anchor to. + // Keep run errors until dismissed — they're actionable and shouldn't + // vanish before the user reads them. + if (Object.keys(nextFieldErrors).length === 0) { + toast.error(getRunErrorMessage(run.error), { + duration: Number.POSITIVE_INFINITY, + closeButton: true, + }); + } } }, [run.status, run.runId, run.error]); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx index c86dd9839..b8eb72c66 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx @@ -1,7 +1,7 @@ "use client"; import { ChevronDown } from "lucide-react"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -205,6 +205,25 @@ export function SchemaForm({ return { primary: primaryFields, advanced: advancedFields }; }, [fields]); + // First invalid field in display order; drives reveal + focus below. + const firstErrorName = useMemo( + () => (fieldErrors ? fields.find((f) => fieldErrors[f.name])?.name : undefined), + [fields, fieldErrors] + ); + + // Reveal the section holding an invalid field, then move focus to it, so an + // error is never left hidden inside the collapsed "Advanced" group. + useEffect(() => { + if (!firstErrorName) return; + if (advanced.some((f) => f.name === firstErrorName)) setShowAdvanced(true); + const raf = requestAnimationFrame(() => { + const el = document.getElementById(`field-${firstErrorName}`); + el?.focus(); + el?.scrollIntoView({ block: "center", behavior: "smooth" }); + }); + return () => cancelAnimationFrame(raf); + }, [firstErrorName, advanced]); + return (
{primary.map((field) => ( From 1e801b3ae93ff35e58a6f313c162e0f026dcade2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 20:47:06 +0200 Subject: [PATCH 19/20] feat: render api reference schemas with JsonView --- .../[workspace_id]/playground/components/api-reference.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx index 05434b953..ba7312d7b 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx @@ -2,6 +2,7 @@ import { Check, ChevronRight, Copy } from "lucide-react"; import { useMemo, useState } from "react"; +import { JsonView } from "@/components/json-view"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { BACKEND_URL } from "@/lib/env-config"; @@ -51,9 +52,9 @@ function SchemaBlock({ title, schema }: { title: string; schema: Record
- 
-					{json}
-
+
+ +
); From cbf2334fb0d2ecbff578d1f6d3ae359d8572e96f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 20:47:10 +0200 Subject: [PATCH 20/20] fix: scope runs detail scrolling to the output table --- .../playground/components/output-viewer.tsx | 25 ++-- .../playground/components/run-detail.tsx | 9 +- .../playground/components/runs-table.tsx | 127 +++++++++--------- 3 files changed, 76 insertions(+), 85 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx index 72262d440..106200a70 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx @@ -2,15 +2,9 @@ import { Check, Copy, Download } from "lucide-react"; import { useMemo, useState } from "react"; +import { JsonView } from "@/components/json-view"; import { Button } from "@/components/ui/button"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; +import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { downloadCsv, rowsToCsv } from "@/lib/playground/csv"; @@ -48,10 +42,11 @@ function ResultTable({ items }: { items: Record[] }) { const rows = items.slice(0, MAX_TABLE_ROWS); + // overscroll-contain keeps table scrolling from chaining to the page. return ( -
- - +
+
+ {columns.map((col) => ( @@ -72,7 +67,7 @@ function ResultTable({ items }: { items: Record[] }) { ))} -
+
); } @@ -152,9 +147,9 @@ export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBa )} ) : ( - 
-					{json}
-
+
+ +
)}
); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx index 4a128f5d8..fdf41729c 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx @@ -1,6 +1,7 @@ "use client"; import { useMemo } from "react"; +import { JsonView } from "@/components/json-view"; import { Spinner } from "@/components/ui/spinner"; import { useScraperRun } from "@/hooks/use-scraper-runs"; import { OutputViewer } from "./output-viewer"; @@ -69,7 +70,7 @@ export function RunDetail({ workspaceId, runId }: { workspaceId: number; runId: {run.progress && run.progress.length > 0 && (

Progress

-
+
{run.progress.map((event, i) => (
{progressLine(event)} @@ -81,9 +82,9 @@ export function RunDetail({ workspaceId, runId }: { workspaceId: number; runId:

Input

- 
-					{JSON.stringify(run.input ?? {}, null, 2)}
-
+
+ +
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx index 57b7a547c..0d642ac3f 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx @@ -2,7 +2,7 @@ import { useInfiniteQuery } from "@tanstack/react-query"; import { ChevronDown, ChevronRight, History, Info } from "lucide-react"; -import { Fragment, useState } from "react"; +import { useState } from "react"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { @@ -13,14 +13,6 @@ import { SelectValue, } from "@/components/ui/select"; import { Spinner } from "@/components/ui/spinner"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; import { scrapersApiService } from "@/lib/apis/scrapers-api.service"; import { formatRelativeDate } from "@/lib/format-date"; import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog"; @@ -33,6 +25,12 @@ import { RunStatusBadge } from "./run-status-badge"; const PAGE_SIZE = 25; const ALL = "__all__"; +// A grid (not a ) keeps an expanded row's detail out of a table-cell, +// where overflow/max-height scroll containers get ignored. Tracks are fr/fixed +// (never content-based auto) so separate per-row grids stay column-aligned. +const ROW_GRID = + "grid grid-cols-[2rem_minmax(8rem,2fr)_minmax(4rem,1fr)_minmax(5rem,1fr)_4rem_5rem_5rem_minmax(6rem,1fr)] items-center gap-2 px-3"; + const CAPABILITY_OPTIONS = PLAYGROUND_PLATFORMS.flatMap((platform) => platform.verbs.map((verb) => verb.name) ); @@ -122,63 +120,60 @@ export function RunsTable({ workspaceId }: { workspaceId: number }) { ) : (
-
- - - - API - Origin - Status - Items - Duration - Cost - When - - - - {runs.map((run) => { - const isOpen = expanded === run.id; - return ( - - setExpanded(isOpen ? null : run.id)} - > - - {isOpen ? ( - - ) : ( - - )} - - {run.capability} - {run.origin} - - - - {run.item_count} - - {formatDuration(run.duration_ms)} - - - {formatCost(run.cost_micros)} - - - {formatRelativeDate(run.created_at)} - - - {isOpen && ( - - - - - - )} - - ); - })} - -
+
+ + API + Origin + Status + Items + Duration + Cost + When +
+ {runs.map((run) => { + const isOpen = expanded === run.id; + return ( +
+ + {isOpen && } +
+ ); + })}
)}