Merge pull request #1623 from CREDO23/fixes-playground-ui

[Fixes] Scraper API input validation & error handling

surfsense_backend/app/app.py

@@ -8,6 +8,7 @@ from collections import defaultdict
 from contextlib import asynccontextmanager
 from datetime import UTC, datetime
 from threading import Lock
+from typing import Any
 
 import redis
 from fastapi import Depends, FastAPI, HTTPException, Request, Response, status
@@ -95,17 +96,21 @@ def _build_error_response(
     code: str = "INTERNAL_ERROR",
     request_id: str = "",
     extra_headers: dict[str, str] | None = None,
+    fields: list[dict[str, Any]] | None = None,
 ) -> JSONResponse:
     """Build the standardized error envelope (new ``error`` + legacy ``detail``)."""
+    error: dict[str, Any] = {
+        "code": code,
+        "message": message,
+        "status": status_code,
+        "request_id": request_id,
+        "timestamp": datetime.now(UTC).isoformat(),
+        "report_url": ISSUES_URL,
+    }
+    if fields:
+        error["fields"] = fields
     body = {
-        "error": {
-            "code": code,
-            "message": message,
-            "status": status_code,
-            "request_id": request_id,
-            "timestamp": datetime.now(UTC).isoformat(),
-            "report_url": ISSUES_URL,
-        },
+        "error": error,
         "detail": message,
     }
     headers = {"X-Request-ID": request_id}
@@ -223,16 +228,35 @@ def _http_exception_handler(request: Request, exc: HTTPException) -> JSONRespons
 def _validation_error_handler(
     request: Request, exc: RequestValidationError
 ) -> JSONResponse:
-    """Return 422 with field-level detail in the standard envelope."""
+    """Return 422 with field-level detail in the standard envelope.
+
+    ``error.fields`` carries each failure's location path and message so clients
+    can attach errors to the offending input; ``message`` is the flat summary.
+    """
     rid = _get_request_id(request)
-    fields = []
-    for err in exc.errors():
-        loc = " -> ".join(str(part) for part in err.get("loc", []))
-        fields.append(f"{loc}: {err.get('msg', 'invalid')}")
-    message = (
-        f"Validation failed: {'; '.join(fields)}" if fields else "Validation failed."
+    fields = [
+        {
+            "loc": [str(part) for part in err.get("loc", ())],
+            # Drop pydantic's "Value error, " prefix so messages read for humans.
+            "msg": str(err.get("msg", "invalid")).removeprefix("Value error, "),
+        }
+        for err in exc.errors()
+    ]
+
+    def _segment(field: dict[str, Any]) -> str:
+        # Drop the "body" request root so model-level errors read as a plain
+        # sentence and field errors read as "field -> sub", not "body -> field".
+        path = field["loc"]
+        if path and path[0] == "body":
+            path = path[1:]
+        loc = " -> ".join(path)
+        return f"{loc}: {field['msg']}" if loc else field["msg"]
+
+    summary = "; ".join(_segment(f) for f in fields)
+    message = f"Validation failed: {summary}" if fields else "Validation failed."
+    return _build_error_response(
+        422, message, code="VALIDATION_ERROR", request_id=rid, fields=fields
     )
-    return _build_error_response(422, message, code="VALIDATION_ERROR", request_id=rid)
 
 
 def _unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse:

surfsense_backend/app/capabilities/amazon/scrape/schemas.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.amazon import ProductItem
 
 MAX_AMAZON_SOURCES = 20
@@ -13,7 +14,7 @@ MAX_AMAZON_RESULTS = 1000
 class ScrapeInput(BaseModel):
     """Agent-facing controls for public product discovery and enrichment."""
 
-    urls: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
+    urls: list[HttpUrlStr] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
     search_terms: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES)
     max_items: int = Field(default=10, ge=1, le=100)
     domain: str = Field(

surfsense_backend/app/capabilities/core/validation.py

@@ -0,0 +1,24 @@
+"""Shared Pydantic field types for capability I/O schemas."""
+
+from __future__ import annotations
+
+from typing import Annotated
+from urllib.parse import urlsplit
+
+import validators
+from pydantic import AfterValidator
+from pydantic_core import PydanticCustomError
+
+_HTTP_SCHEMES = frozenset({"http", "https"})
+
+
+def _validate_http_url(value: str) -> str:
+    """Accept only well-formed http(s) URLs, returned trimmed and unchanged."""
+    url = value.strip()
+    if not validators.url(url) or urlsplit(url).scheme.lower() not in _HTTP_SCHEMES:
+        raise PydanticCustomError("http_url", "must be a valid http(s) URL")
+    return url
+
+
+HttpUrlStr = Annotated[str, AfterValidator(_validate_http_url)]
+"""A request URL validated as http(s) and kept as ``str`` (no normalization)."""

surfsense_backend/app/capabilities/google_maps/reviews/schemas.py

@@ -10,6 +10,7 @@ from typing import Literal
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.google_maps import ReviewItem
 
 MAX_MAPS_REVIEW_SOURCES = 20
@@ -17,7 +18,7 @@ MAX_MAPS_REVIEW_SOURCES = 20
 
 
 class ReviewsInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_MAPS_REVIEW_SOURCES,
         description=(

surfsense_backend/app/capabilities/google_maps/scrape/schemas.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.google_maps import PlaceItem
 
 MAX_MAPS_SOURCES = 20
@@ -26,7 +27,7 @@ class ScrapeInput(BaseModel):
             "(at least one is required). Pair with location to scope a search."
         ),
     )
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_MAPS_SOURCES,
         description=(

surfsense_backend/app/capabilities/indeed/scrape/schemas.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.indeed_jobs import IndeedItem
 from app.proprietary.platforms.indeed_jobs.schemas import (
     IndeedJobType,
@@ -26,7 +27,7 @@ MAX_INDEED_ITEMS = 100
 
 
 class ScrapeInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_INDEED_SOURCES,
         description=(

surfsense_backend/app/capabilities/reddit/scrape/schemas.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.reddit import RedditItem
 from app.proprietary.platforms.reddit.schemas import RedditSort, RedditTime
 
@@ -21,7 +22,7 @@ MAX_REDDIT_ITEMS = 100
 
 
 class ScrapeInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_REDDIT_SOURCES,
         description=(

surfsense_backend/app/capabilities/tiktok/comments/schemas.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.capabilities.tiktok.scrape.schemas import (
     MAX_TIKTOK_ITEMS,
     MAX_TIKTOK_SOURCES,
@@ -18,7 +19,7 @@ from app.proprietary.platforms.tiktok import CommentItem
 
 
 class CommentsInput(BaseModel):
-    video_urls: list[str] = Field(
+    video_urls: list[HttpUrlStr] = Field(
         min_length=1,
         max_length=MAX_TIKTOK_SOURCES,
         description="TikTok video URLs (/@<user>/video/<id>) to pull comments from.",

surfsense_backend/app/capabilities/tiktok/scrape/schemas.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.tiktok import TikTokVideoItem
 
 MAX_TIKTOK_SOURCES = 20
@@ -22,7 +23,7 @@ MAX_TIKTOK_ITEMS = 100
 
 
 class ScrapeInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_TIKTOK_SOURCES,
         description=(

surfsense_backend/app/capabilities/walmart/reviews/schemas.py

@@ -11,13 +11,14 @@ from typing import Literal
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.walmart import ReviewItem
 
 MAX_WALMART_REVIEW_SOURCES = 20
 
 
 class ReviewsInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_WALMART_REVIEW_SOURCES,
         description=(

surfsense_backend/app/capabilities/walmart/scrape/schemas.py

@@ -6,6 +6,7 @@ from urllib.parse import quote_plus
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.walmart import ProductItem
 
 MAX_WALMART_SOURCES = 20
@@ -15,7 +16,7 @@ MAX_WALMART_RESULTS = 1000
 class ScrapeInput(BaseModel):
     """Agent-facing controls for public Walmart product discovery and enrichment."""
 
-    urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES)
+    urls: list[HttpUrlStr] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES)
     search_terms: list[str] = Field(
         default_factory=list, max_length=MAX_WALMART_SOURCES
     )

surfsense_backend/app/capabilities/web/crawl/schemas.py

@@ -18,6 +18,8 @@ from typing import Literal
 
 from pydantic import BaseModel, Field
 
+from app.capabilities.core.validation import HttpUrlStr
+
 MAX_START_URLS = 20
 """Per-call cap on seed URLs: bounds a synchronous request's fan-out (05)."""
 
@@ -29,7 +31,7 @@ MAX_CRAWL_PAGES = 200
 
 
 class CrawlInput(BaseModel):
-    startUrls: list[str] = Field(
+    startUrls: list[HttpUrlStr] = Field(
         min_length=1,
         max_length=MAX_START_URLS,
         description=(

surfsense_backend/app/capabilities/youtube/comments/schemas.py

@@ -10,6 +10,7 @@ from typing import Literal
 
 from pydantic import BaseModel, Field
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.youtube import CommentItem
 
 MAX_COMMENT_VIDEOS = 20
@@ -17,7 +18,7 @@ MAX_COMMENT_VIDEOS = 20
 
 
 class CommentsInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         min_length=1,
         max_length=MAX_COMMENT_VIDEOS,
         description="YouTube video URLs to fetch comments (and replies) for (1-20).",

surfsense_backend/app/capabilities/youtube/scrape/schemas.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
+from app.capabilities.core.validation import HttpUrlStr
 from app.proprietary.platforms.youtube import VideoItem
 
 MAX_YOUTUBE_SOURCES = 20
@@ -17,7 +18,7 @@ MAX_YOUTUBE_SOURCES = 20
 
 
 class ScrapeInput(BaseModel):
-    urls: list[str] = Field(
+    urls: list[HttpUrlStr] = Field(
         default_factory=list,
         max_length=MAX_YOUTUBE_SOURCES,
         description=(

surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py

@@ -15,6 +15,9 @@ ResolvedKind = Literal["video", "channel", "playlist", "hashtag", "search"]
 
 _PLAYLIST_ID_RE = re.compile(r"[?&]list=([\w-]+)")
 
+_YOUTUBE_HOSTS = frozenset({"www.youtube.com", "youtube.com", "m.youtube.com"})
+"""Hosts whose page paths (/@, /channel, /playlist, ...) are trusted as YouTube."""
+
 
 @dataclass(frozen=True)
 class ResolvedUrl:
@@ -29,7 +32,7 @@ def get_youtube_video_id(url: str) -> str | None:
     hostname = parsed.hostname or ""
     if hostname == "youtu.be":
         return parsed.path[1:] or None
-    if hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"):
+    if hostname in _YOUTUBE_HOSTS:
         if parsed.path == "/watch":
             return parse_qs(parsed.query).get("v", [None])[0]
         for prefix in ("/embed/", "/v/", "/shorts/"):
@@ -41,17 +44,21 @@ def get_youtube_video_id(url: str) -> str | None:
 def resolve_url(url: str) -> ResolvedUrl | None:
     """Classify a YouTube URL into a scrape job, or ``None`` if unrecognized."""
     parsed = urlparse(url)
+    hostname = (parsed.hostname or "").lower()
     path = parsed.path or ""
 
-    # Shorts are videos with their own path.
-    if "/shorts/" in path:
-        vid = path.split("/shorts/")[1].split("/")[0]
-        return ResolvedUrl("video", vid, url) if vid else None
-
+    # Videos: watch / youtu.be / embed / shorts. get_youtube_video_id validates
+    # the host itself, so this also covers youtu.be short links.
     video_id = get_youtube_video_id(url)
     if video_id:
         return ResolvedUrl("video", video_id, url)
 
+    # Every remaining shape is a youtube.com page keyed off its path. Require a
+    # YouTube host so a well-formed non-YouTube URL (e.g. https://evil.com/@x)
+    # is rejected instead of misclassified as a channel by its path alone.
+    if hostname not in _YOUTUBE_HOSTS:
+        return None
+
     # Playlist (either a /playlist page or any URL carrying ?list=).
     playlist_match = _PLAYLIST_ID_RE.search(url)
     if path.startswith("/playlist") and playlist_match:

surfsense_backend/tests/unit/capabilities/core/test_validation.py

@@ -0,0 +1,32 @@
+"""``HttpUrlStr`` boundary type: accept well-formed http(s) URLs, reject the rest."""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import BaseModel, ValidationError
+
+from app.capabilities.core.validation import HttpUrlStr
+
+pytestmark = pytest.mark.unit
+
+
+class _Model(BaseModel):
+    urls: list[HttpUrlStr]
+
+
+def test_accepts_http_and_https_urls_unchanged() -> None:
+    model = _Model(urls=["https://example.com/path?q=1", "http://a.co"])
+    assert model.urls == ["https://example.com/path?q=1", "http://a.co"]
+
+
+def test_trims_surrounding_whitespace() -> None:
+    assert _Model(urls=["  https://example.com  "]).urls == ["https://example.com"]
+
+
+@pytest.mark.parametrize(
+    "bad",
+    ["not-a-url", "example.com", "ftp://example.com", "http://", "javascript:alert(1)"],
+)
+def test_rejects_malformed_or_non_http_urls(bad: str) -> None:
+    with pytest.raises(ValidationError):
+        _Model(urls=[bad])

surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py

@@ -20,6 +20,11 @@ def test_requires_at_least_one_start_url() -> None:
         CrawlInput(startUrls=[])
 
 
+def test_rejects_malformed_start_url() -> None:
+    with pytest.raises(ValidationError):
+        CrawlInput(startUrls=["not-a-url"])
+
+
 def test_camelcase_fields_and_defaults() -> None:
     model = CrawlInput(startUrls=["https://e.com"])
     assert model.startUrls == ["https://e.com"]

surfsense_backend/tests/unit/platforms/youtube/test_parsers.py

@@ -946,8 +946,22 @@ def test_resolve_url(url, kind, value):
     assert resolved.value == value
 
 
-def test_resolve_url_unrecognized():
-    assert resolve_url("https://example.com/foo") is None
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://example.com/foo",
+        # Well-formed non-YouTube URLs whose path mimics a YouTube page must be
+        # rejected, not misclassified by path alone (host-spoof guard).
+        "https://evil.com/@Apify",
+        "https://evil.com/channel/UC123456789abc",
+        "https://evil.com/shorts/abc123",
+        "https://evil.com/playlist?list=PL123",
+        "https://evil.com/hashtag/tech",
+        "https://evil.com/results?search_query=web+scraping",
+    ],
+)
+def test_resolve_url_unrecognized(url):
+    assert resolve_url(url) is None
 
 
 # --- optional: exercise captured real fixtures if present --------------------

surfsense_backend/tests/unit/test_error_contract.py

@@ -14,6 +14,7 @@ import json
 
 import pytest
 from fastapi import HTTPException
+from pydantic import BaseModel, model_validator
 from starlette.testclient import TestClient
 
 from app.exceptions import (
@@ -32,6 +33,23 @@ from app.exceptions import (
 pytestmark = pytest.mark.unit
 
 
+class _RequireOne(BaseModel):
+    """Body model with a cross-field rule, used to test the 422 summary.
+
+    Defined at module scope (not inside the app factory) so FastAPI's ``Body``
+    ``TypeAdapter`` can resolve the forward reference.
+    """
+
+    a: str | None = None
+    b: str | None = None
+
+    @model_validator(mode="after")
+    def _at_least_one(self):
+        if not self.a and not self.b:
+            raise ValueError("Provide at least one of 'a' or 'b'.")
+        return self
+
+
 # ---------------------------------------------------------------------------
 # Helpers - lightweight FastAPI app that re-uses the real global handlers
 # ---------------------------------------------------------------------------
@@ -39,7 +57,7 @@ pytestmark = pytest.mark.unit
 
 def _make_test_app():
     """Build a minimal FastAPI app with the same handlers as the real one."""
-    from fastapi import FastAPI
+    from fastapi import Body, FastAPI
     from fastapi.exceptions import RequestValidationError
     from pydantic import BaseModel
 
@@ -124,6 +142,10 @@ def _make_test_app():
     async def validated(item: Item):
         return item.model_dump()
 
+    @app.post("/require-one")
+    async def require_one(payload: _RequireOne = Body(...)):
+        return payload.model_dump()
+
     return app
 
 
@@ -277,6 +299,26 @@ class TestValidationErrorHandler:
         body = _assert_envelope(resp, 422)
         assert body["error"]["code"] == "VALIDATION_ERROR"
 
+    def test_field_error_drops_body_root(self, client):
+        # The "body" request root is noise in the human summary; the field name
+        # is kept so clients can still attach the error inline.
+        resp = client.post("/require-one", json={"a": 123})
+        body = _assert_envelope(resp, 422)
+        msg = body["error"]["message"]
+        assert "body" not in msg
+        assert "a:" in msg
+        assert body["error"]["fields"][0]["loc"] == ["body", "a"]
+
+    def test_model_level_error_reads_as_sentence(self, client):
+        # A cross-field rule (loc == ["body"]) must read as a plain sentence,
+        # not "body: ...", while fields still carry the location for clients.
+        resp = client.post("/require-one", json={})
+        body = _assert_envelope(resp, 422)
+        assert body["error"]["message"] == (
+            "Validation failed: Provide at least one of 'a' or 'b'."
+        )
+        assert body["error"]["fields"][0]["loc"] == ["body"]
+
 
 # ---------------------------------------------------------------------------
 # SurfSenseError class hierarchy unit tests

surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx

@@ -2,6 +2,7 @@
 
 import { Check, ChevronRight, Copy } from "lucide-react";
 import { useMemo, useState } from "react";
+import { JsonView } from "@/components/json-view";
 import { Button } from "@/components/ui/button";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { BACKEND_URL } from "@/lib/env-config";
@@ -51,9 +52,9 @@ function SchemaBlock({ title, schema }: { title: string; schema: Record<string,
 			</summary>
 			<div className="relative border-t border-border/60">
 				<CopyButton text={json} />
-				<pre className="max-h-[360px] overflow-auto p-3 pr-20 text-xs">
-					<code>{json}</code>
-				</pre>
+				<div className="max-h-[360px] overflow-auto p-3 pr-20">
+					<JsonView src={schema} collapsed={2} />
+				</div>
 			</div>
 		</details>
 	);

surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx

@@ -2,15 +2,9 @@
 
 import { Check, Copy, Download } from "lucide-react";
 import { useMemo, useState } from "react";
+import { JsonView } from "@/components/json-view";
 import { Button } from "@/components/ui/button";
-import {
-	Table,
-	TableBody,
-	TableCell,
-	TableHead,
-	TableHeader,
-	TableRow,
-} from "@/components/ui/table";
+import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { downloadCsv, rowsToCsv } from "@/lib/playground/csv";
 
@@ -48,10 +42,11 @@ function ResultTable({ items }: { items: Record<string, unknown>[] }) {
 
 	const rows = items.slice(0, MAX_TABLE_ROWS);
 
+	// overscroll-contain keeps table scrolling from chaining to the page.
 	return (
-		<div className="overflow-x-auto rounded-md border border-border/60">
-			<Table>
-				<TableHeader>
+		<div className="max-h-[480px] overflow-auto overscroll-contain rounded-md border border-border/60">
+			<table className="w-full caption-bottom text-sm">
+				<TableHeader className="sticky top-0 z-10 bg-background">
 					<TableRow>
 						{columns.map((col) => (
 							<TableHead key={col} className="whitespace-nowrap">
@@ -72,7 +67,7 @@ function ResultTable({ items }: { items: Record<string, unknown>[] }) {
 						</TableRow>
 					))}
 				</TableBody>
-			</Table>
+			</table>
 		</div>
 	);
 }
@@ -152,9 +147,9 @@ export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBa
 					)}
 				</>
 			) : (
-				<pre className="max-h-[480px] overflow-auto rounded-md border border-border/60 bg-muted/20 p-3 text-xs">
-					<code>{json}</code>
-				</pre>
+				<div className="max-h-[480px] overflow-auto overscroll-contain rounded-md border border-border/60 bg-muted/20 p-3">
+					<JsonView src={data} collapsed={2} />
+				</div>
 			)}
 		</div>
 	);

surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx

@@ -12,6 +12,7 @@ import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities";
 import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
 import { AppError } from "@/lib/error";
 import { findVerb } from "@/lib/playground/catalog";
+import { fieldErrorsFromError } from "@/lib/playground/field-errors";
 import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format";
 import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema";
 import {
@@ -19,6 +20,7 @@ import {
 	getAmazonFieldOptions,
 	hasAmazonFranceValue,
 } from "@/lib/playground/platform-overrides/amazon";
+import { urlFieldWarnings } from "@/lib/playground/url-hints";
 import { ApiReference } from "./api-reference";
 import { OutputViewer } from "./output-viewer";
 import { RunProgressPanel } from "./run-progress-panel";
@@ -73,7 +75,12 @@ function getRunErrorMessage(error: unknown): string {
 	}
 
 	if (status === 422) {
-		return "Invalid input. Check the fields above and try again.";
+		if (Object.keys(fieldErrorsFromError(error)).length > 0) {
+			return "Invalid input. Check the highlighted fields.";
+		}
+		return error instanceof Error && error.message
+			? error.message
+			: "Invalid input. Check the fields above and try again.";
 	}
 
 	return error instanceof Error && error.message
@@ -123,6 +130,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 	const fields = useMemo(() => parseSchemaFields(capability?.input_schema), [capability]);
 
 	const [values, setValues] = useState<Record<string, unknown>>({});
+	const [fieldErrors, setFieldErrors] = useState<Record<string, string>>({});
 	const run = useRunStream(workspaceId);
 	const isRunning = run.status === "running";
 	const previousStatusRef = useRef(run.status);
@@ -164,9 +172,15 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 
 	const handleChange = (name: string, value: unknown) => {
 		setValues((prev) => ({ ...prev, [name]: value }));
+		setFieldErrors((prev) => {
+			if (!(name in prev)) return prev;
+			const { [name]: _cleared, ...rest } = prev;
+			return rest;
+		});
 	};
 
 	const handleRun = useCallback(() => {
+		setFieldErrors({});
 		const payload = buildPayload(fields, values);
 		void run.start(platform, verb, payload);
 	}, [fields, values, platform, verb, run]);
@@ -178,6 +192,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 	const endpoint = `POST /workspaces/${workspaceId}/scrapers/${platform}/${verb}`;
 	const isAmazonScrape = platform === "amazon" && verb === "scrape";
 	const hasAmazonFranceUrl = useMemo(() => hasAmazonFranceValue(values), [values]);
+	const fieldWarnings = useMemo(() => urlFieldWarnings(platform, values), [platform, values]);
 
 	useEffect(() => {
 		const previousStatus = previousStatusRef.current;
@@ -194,10 +209,21 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 		}
 
 		if (run.status === "error") {
+			const nextFieldErrors = fieldErrorsFromError(run.error);
+			setFieldErrors(nextFieldErrors);
 			const key = `${run.runId ?? "run"}:error`;
 			if (notifiedRunRef.current === key) return;
 			notifiedRunRef.current = key;
-			toast.error(getRunErrorMessage(run.error));
+			// Field-level failures are shown inline (the form reveals + focuses the
+			// first one), so only toast global failures that have no field to anchor to.
+			// Keep run errors until dismissed — they're actionable and shouldn't
+			// vanish before the user reads them.
+			if (Object.keys(nextFieldErrors).length === 0) {
+				toast.error(getRunErrorMessage(run.error), {
+					duration: Number.POSITIVE_INFINITY,
+					closeButton: true,
+				});
+			}
 		}
 	}, [run.status, run.runId, run.error]);
 
@@ -268,6 +294,8 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
 						onChange={handleChange}
 						disabled={isRunning}
 						getFieldOptions={isAmazonScrape ? getAmazonFieldOptions : undefined}
+						fieldErrors={fieldErrors}
+						fieldWarnings={fieldWarnings}
 					/>
 					{isAmazonScrape ? <AmazonMarketplaceHint showFranceWarning={hasAmazonFranceUrl} /> : null}
 

surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import { useMemo } from "react";
+import { JsonView } from "@/components/json-view";
 import { Spinner } from "@/components/ui/spinner";
 import { useScraperRun } from "@/hooks/use-scraper-runs";
 import { OutputViewer } from "./output-viewer";
@@ -69,7 +70,7 @@ export function RunDetail({ workspaceId, runId }: { workspaceId: number; runId:
 			{run.progress && run.progress.length > 0 && (
 				<div>
 					<h4 className="mb-1.5 text-xs font-medium text-muted-foreground">Progress</h4>
-					<div className="max-h-48 space-y-1 overflow-y-auto rounded-md border border-border/60 bg-background p-3 font-mono text-xs text-muted-foreground">
+					<div className="space-y-1 rounded-md border border-border/60 bg-background p-3 font-mono text-xs text-muted-foreground">
 						{run.progress.map((event, i) => (
 							<div key={i} className="truncate">
 								{progressLine(event)}
@@ -81,9 +82,9 @@ export function RunDetail({ workspaceId, runId }: { workspaceId: number; runId:
 
 			<div>
 				<h4 className="mb-1.5 text-xs font-medium text-muted-foreground">Input</h4>
-				<pre className="max-h-64 overflow-auto rounded-md border border-border/60 bg-background p-3 text-xs">
-					<code>{JSON.stringify(run.input ?? {}, null, 2)}</code>
-				</pre>
+				<div className="rounded-md border border-border/60 bg-background p-3">
+					<JsonView src={run.input ?? {}} collapsed={2} />
+				</div>
 			</div>
 
 			<div>

surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx

@@ -2,7 +2,7 @@
 
 import { useInfiniteQuery } from "@tanstack/react-query";
 import { ChevronDown, ChevronRight, History, Info } from "lucide-react";
-import { Fragment, useState } from "react";
+import { useState } from "react";
 import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Button } from "@/components/ui/button";
 import {
@@ -13,14 +13,6 @@ import {
 	SelectValue,
 } from "@/components/ui/select";
 import { Spinner } from "@/components/ui/spinner";
-import {
-	Table,
-	TableBody,
-	TableCell,
-	TableHead,
-	TableHeader,
-	TableRow,
-} from "@/components/ui/table";
 import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
 import { formatRelativeDate } from "@/lib/format-date";
 import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog";
@@ -33,6 +25,12 @@ import { RunStatusBadge } from "./run-status-badge";
 const PAGE_SIZE = 25;
 const ALL = "__all__";
 
+// A grid (not a <table>) keeps an expanded row's detail out of a table-cell,
+// where overflow/max-height scroll containers get ignored. Tracks are fr/fixed
+// (never content-based auto) so separate per-row grids stay column-aligned.
+const ROW_GRID =
+	"grid grid-cols-[2rem_minmax(8rem,2fr)_minmax(4rem,1fr)_minmax(5rem,1fr)_4rem_5rem_5rem_minmax(6rem,1fr)] items-center gap-2 px-3";
+
 const CAPABILITY_OPTIONS = PLAYGROUND_PLATFORMS.flatMap((platform) =>
 	platform.verbs.map((verb) => verb.name)
 );
@@ -122,63 +120,60 @@ export function RunsTable({ workspaceId }: { workspaceId: number }) {
 				</div>
 			) : (
 				<div className="overflow-hidden rounded-md border border-border/60">
-					<Table>
-						<TableHeader>
-							<TableRow>
-								<TableHead className="w-8" />
-								<TableHead>API</TableHead>
-								<TableHead>Origin</TableHead>
-								<TableHead>Status</TableHead>
-								<TableHead className="text-right">Items</TableHead>
-								<TableHead className="text-right">Duration</TableHead>
-								<TableHead className="text-right">Cost</TableHead>
-								<TableHead className="text-right">When</TableHead>
-							</TableRow>
-						</TableHeader>
-						<TableBody>
-							{runs.map((run) => {
-								const isOpen = expanded === run.id;
-								return (
-									<Fragment key={run.id}>
-										<TableRow
-											className="cursor-pointer"
-											onClick={() => setExpanded(isOpen ? null : run.id)}
-										>
-											<TableCell>
-												{isOpen ? (
-													<ChevronDown className="h-4 w-4 text-muted-foreground" />
-												) : (
-													<ChevronRight className="h-4 w-4 text-muted-foreground" />
-												)}
-											</TableCell>
-											<TableCell className="font-mono text-xs">{run.capability}</TableCell>
-											<TableCell className="text-xs text-muted-foreground">{run.origin}</TableCell>
-											<TableCell>
-												<RunStatusBadge status={run.status} />
-											</TableCell>
-											<TableCell className="text-right tabular-nums">{run.item_count}</TableCell>
-											<TableCell className="text-right tabular-nums text-muted-foreground">
-												{formatDuration(run.duration_ms)}
-											</TableCell>
-											<TableCell className="text-right tabular-nums text-muted-foreground">
-												{formatCost(run.cost_micros)}
-											</TableCell>
-											<TableCell className="text-right text-xs text-muted-foreground">
-												{formatRelativeDate(run.created_at)}
-											</TableCell>
-										</TableRow>
-										{isOpen && (
-											<TableRow className="hover:bg-transparent">
-												<TableCell colSpan={8} className="p-0">
-													<RunDetail workspaceId={workspaceId} runId={run.id} />
-												</TableCell>
-											</TableRow>
-										)}
-									</Fragment>
-								);
-							})}
-						</TableBody>
-					</Table>
+					<div
+						className={cn(
+							ROW_GRID,
+							"border-b border-border/60 bg-muted/30 py-2 text-xs font-medium text-muted-foreground"
+						)}
+					>
+						<span />
+						<span>API</span>
+						<span>Origin</span>
+						<span>Status</span>
+						<span className="text-right">Items</span>
+						<span className="text-right">Duration</span>
+						<span className="text-right">Cost</span>
+						<span className="text-right">When</span>
+					</div>
+					{runs.map((run) => {
+						const isOpen = expanded === run.id;
+						return (
+							<div key={run.id} className="border-b border-border/60 last:border-b-0">
+								<button
+									type="button"
+									onClick={() => setExpanded(isOpen ? null : run.id)}
+									className={cn(
+										ROW_GRID,
+										"w-full py-2.5 text-left text-sm transition-colors hover:bg-muted/50"
+									)}
+								>
+									{isOpen ? (
+										<ChevronDown className="h-4 w-4 text-muted-foreground" />
+									) : (
+										<ChevronRight className="h-4 w-4 text-muted-foreground" />
+									)}
+									<span className="min-w-0 truncate font-mono text-xs">{run.capability}</span>
+									<span className="min-w-0 truncate text-xs text-muted-foreground">
+										{run.origin}
+									</span>
+									<span>
+										<RunStatusBadge status={run.status} />
+									</span>
+									<span className="text-right tabular-nums">{run.item_count}</span>
+									<span className="text-right tabular-nums text-muted-foreground">
+										{formatDuration(run.duration_ms)}
+									</span>
+									<span className="text-right tabular-nums text-muted-foreground">
+										{formatCost(run.cost_micros)}
+									</span>
+									<span className="text-right text-xs text-muted-foreground">
+										{formatRelativeDate(run.created_at)}
+									</span>
+								</button>
+								{isOpen && <RunDetail workspaceId={workspaceId} runId={run.id} />}
+							</div>
+						);
+					})}
 				</div>
 			)}
 

surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { ChevronDown } from "lucide-react";
-import { useMemo, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
 import { Badge } from "@/components/ui/badge";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
@@ -32,6 +32,8 @@ interface SchemaFormProps {
 	getFieldOptions?: FieldOptionsResolver;
 	/** Field names flagged by a 422 response, shown with error styling. */
 	fieldErrors?: Record<string, string>;
+	/** Client-side, non-blocking URL warnings keyed by field name. */
+	fieldWarnings?: Record<string, string>;
 }
 
 function FieldControl({
@@ -147,6 +149,7 @@ function FieldRow({
 	onChange,
 	disabled,
 	error,
+	warning,
 	options,
 }: {
 	field: FormField;
@@ -154,6 +157,7 @@ function FieldRow({
 	onChange: (value: unknown) => void;
 	disabled?: boolean;
 	error?: string;
+	warning?: string;
 	options?: FieldOption[];
 }) {
 	return (
@@ -179,6 +183,7 @@ function FieldRow({
 				options={options}
 			/>
 			{error && <p className="text-xs text-destructive">{error}</p>}
+			{!error && warning && <p className="text-xs text-amber-600 dark:text-amber-500">{warning}</p>}
 		</div>
 	);
 }
@@ -190,6 +195,7 @@ export function SchemaForm({
 	disabled,
 	getFieldOptions,
 	fieldErrors,
+	fieldWarnings,
 }: SchemaFormProps) {
 	const [showAdvanced, setShowAdvanced] = useState(false);
 
@@ -199,6 +205,25 @@ export function SchemaForm({
 		return { primary: primaryFields, advanced: advancedFields };
 	}, [fields]);
 
+	// First invalid field in display order; drives reveal + focus below.
+	const firstErrorName = useMemo(
+		() => (fieldErrors ? fields.find((f) => fieldErrors[f.name])?.name : undefined),
+		[fields, fieldErrors]
+	);
+
+	// Reveal the section holding an invalid field, then move focus to it, so an
+	// error is never left hidden inside the collapsed "Advanced" group.
+	useEffect(() => {
+		if (!firstErrorName) return;
+		if (advanced.some((f) => f.name === firstErrorName)) setShowAdvanced(true);
+		const raf = requestAnimationFrame(() => {
+			const el = document.getElementById(`field-${firstErrorName}`);
+			el?.focus();
+			el?.scrollIntoView({ block: "center", behavior: "smooth" });
+		});
+		return () => cancelAnimationFrame(raf);
+	}, [firstErrorName, advanced]);
+
 	return (
 		<div className="space-y-5">
 			{primary.map((field) => (
@@ -209,6 +234,7 @@ export function SchemaForm({
 					onChange={(value) => onChange(field.name, value)}
 					disabled={disabled}
 					error={fieldErrors?.[field.name]}
+					warning={fieldWarnings?.[field.name]}
 					options={getFieldOptions?.(field)}
 				/>
 			))}
@@ -239,6 +265,7 @@ export function SchemaForm({
 									onChange={(value) => onChange(field.name, value)}
 									disabled={disabled}
 									error={fieldErrors?.[field.name]}
+									warning={fieldWarnings?.[field.name]}
 									options={getFieldOptions?.(field)}
 								/>
 							))}

surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx

@@ -8,8 +8,18 @@ import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import { Textarea } from "@/components/ui/textarea";
+import { isHttpUrl } from "@/lib/url";
+import { cn } from "@/lib/utils";
 import type { ConnectorConfigProps } from "../index";
 
+/** Return the malformed (non-http/https) lines from a newline-separated URL list. */
+function invalidUrlLines(value: string): string[] {
+	return value
+		.split("\n")
+		.map((line) => line.trim())
+		.filter((line) => line && !isHttpUrl(line));
+}
+
 export const WebcrawlerConfig: FC<ConnectorConfigProps> = ({ connector, onConfigChange }) => {
 	// Initialize with existing config values
 	const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || "";
@@ -19,6 +29,8 @@ export const WebcrawlerConfig: FC<ConnectorConfigProps> = ({ connector, onConfig
 	const [initialUrls, setInitialUrls] = useState(existingUrls);
 	const [showApiKey, setShowApiKey] = useState(false);
 
+	const invalidUrls = invalidUrlLines(initialUrls);
+
 	const handleApiKeyChange = (value: string) => {
 		setApiKey(value);
 		if (onConfigChange) {
@@ -108,11 +120,21 @@ export const WebcrawlerConfig: FC<ConnectorConfigProps> = ({ connector, onConfig
 					placeholder="https://example.com&#10;https://docs.example.com&#10;https://blog.example.com"
 					value={initialUrls}
 					onChange={(e) => handleUrlsChange(e.target.value)}
-					className="min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none"
+					aria-invalid={invalidUrls.length > 0}
+					className={cn(
+						"min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none",
+						invalidUrls.length > 0 && "border-destructive"
+					)}
 				/>
-				<p className="text-[10px] sm:text-xs text-muted-foreground">
-					Enter URLs to crawl (one per line). You can add more URLs later.
-				</p>
+				{invalidUrls.length > 0 ? (
+					<p className="text-[10px] sm:text-xs text-destructive">
+						Not valid http(s) URLs: {invalidUrls.join(", ")}
+					</p>
+				) : (
+					<p className="text-[10px] sm:text-xs text-muted-foreground">
+						Enter URLs to crawl (one per line). You can add more URLs later.
+					</p>
+				)}
 			</div>
 
 			{/* Info Alert */}

surfsense_web/lib/apis/base-api.service.ts

@@ -10,6 +10,7 @@ import {
 	AuthorizationError,
 	NetworkError,
 	NotFoundError,
+	type ValidationFieldError,
 } from "../error";
 
 enum ResponseType {
@@ -189,6 +190,9 @@ class BaseApiService {
 				const requestId: string | undefined =
 					envelope?.request_id ?? response.headers.get("X-Request-ID") ?? undefined;
 				const reportUrl: string | undefined = envelope?.report_url;
+				const validationFields: ValidationFieldError[] | undefined = Array.isArray(envelope?.fields)
+					? envelope.fields
+					: undefined;
 
 				// Handle 401 - try to refresh token first (only once)
 				if (response.status === 401) {
@@ -233,8 +237,8 @@ class BaseApiService {
 							response.status,
 							response.statusText
 						);
-					default:
-						throw new AppError(
+					default: {
+						const appError = new AppError(
 							errorMessage || "Something went wrong",
 							response.status,
 							response.statusText,
@@ -242,6 +246,9 @@ class BaseApiService {
 							requestId,
 							reportUrl
 						);
+						appError.fields = validationFields;
+						throw appError;
+					}
 				}
 			}
 			refreshRetryBlockedUntil.delete(getRefreshRetryKey(mergedOptions.method, url));
@@ -313,12 +320,23 @@ class BaseApiService {
 				throw networkError;
 			}
 
-			console.error("Request failed:", JSON.stringify(error));
-			// Only 5xx server faults are unexpected. 4xx (validation, authz, 404)
-			// are expected behavior — capturing them was billable error-tracking
-			// noise. AuthenticationError (401) is a 4xx and stays excluded.
-			if (error instanceof AppError && error.status >= 500) {
-				captureApiException(error, url, options?.method);
+			// Handled client errors (validation, credits, auth, not-found, ...) are
+			// expected outcomes surfaced to the user via typed errors + toasts — not
+			// app faults. Skip logging/telemetry for them so they don't spam the
+			// console, PostHog, or Next.js's dev error overlay.
+			const isHandledClientError =
+				error instanceof AppError &&
+				typeof error.status === "number" &&
+				error.status >= 400 &&
+				error.status < 500;
+
+			if (!isHandledClientError) {
+				console.error("Request failed:", JSON.stringify(error));
+				// Only 5xx server faults are unexpected; 4xx are handled above and
+				// network outages were already captured before this point.
+				if (error instanceof AppError && error.status >= 500) {
+					captureApiException(error, url, options?.method);
+				}
 			}
 			throw error;
 		}

surfsense_web/lib/error.ts

@@ -1,11 +1,19 @@
 export const SURFSENSE_ISSUES_URL = "https://github.com/MODSetter/SurfSense/issues";
 
+/** One field-level failure from a 422 validation response. */
+export interface ValidationFieldError {
+	loc: string[];
+	msg: string;
+}
+
 export class AppError extends Error {
 	status?: number;
 	statusText?: string;
 	code?: string;
 	requestId?: string;
 	reportUrl?: string;
+	/** Per-field failures from a 422 response, keyed by their location path. */
+	fields?: ValidationFieldError[];
 	constructor(
 		message: string,
 		status?: number,

surfsense_web/lib/playground/field-errors.test.ts

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { AppError } from "@/lib/error";
+import { fieldErrorsFromError } from "./field-errors";
+
+// Run with: pnpm exec tsx --test lib/playground/field-errors.test.ts
+
+function validationError(fields: { loc: string[]; msg: string }[]): AppError {
+	const error = new AppError("Validation failed", 422, "Unprocessable Entity", "VALIDATION_ERROR");
+	error.fields = fields;
+	return error;
+}
+
+test("maps a body field failure to its top-level field name", () => {
+	const error = validationError([
+		{ loc: ["body", "startUrls", "0"], msg: "must be a valid http(s) URL" },
+	]);
+	assert.deepEqual(fieldErrorsFromError(error), {
+		startUrls: "must be a valid http(s) URL",
+	});
+});
+
+test("keeps the first failure per field", () => {
+	const error = validationError([
+		{ loc: ["body", "urls", "0"], msg: "first" },
+		{ loc: ["body", "urls", "3"], msg: "second" },
+	]);
+	assert.deepEqual(fieldErrorsFromError(error), { urls: "first" });
+});
+
+test("returns nothing for non-AppError or errors without fields", () => {
+	assert.deepEqual(fieldErrorsFromError(new Error("boom")), {});
+	assert.deepEqual(fieldErrorsFromError(new AppError("no fields", 500)), {});
+});

surfsense_web/lib/playground/field-errors.ts

@@ -0,0 +1,18 @@
+import { AppError } from "@/lib/error";
+
+/** Map a 422 response's field failures to ``{ fieldName: message }`` for the form. */
+export function fieldErrorsFromError(error: unknown): Record<string, string> {
+	if (!(error instanceof AppError) || !error.fields) return {};
+	const errors: Record<string, string> = {};
+	for (const { loc, msg } of error.fields) {
+		const name = topLevelField(loc);
+		if (name && !errors[name]) errors[name] = msg;
+	}
+	return errors;
+}
+
+/** The input field a location path points at, dropping the ``body`` request root. */
+function topLevelField(loc: string[]): string | undefined {
+	const path = loc[0] === "body" ? loc.slice(1) : loc;
+	return path[0];
+}

surfsense_web/lib/playground/url-hints.test.ts

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { urlFieldWarning, urlFieldWarnings } from "./url-hints";
+
+// Run with: pnpm exec tsx --test lib/playground/url-hints.test.ts
+
+test("accepts a matching platform URL (no warning)", () => {
+	assert.equal(urlFieldWarning("youtube", "urls", "https://www.youtube.com/watch?v=x"), undefined);
+	assert.equal(urlFieldWarning("youtube", "urls", "https://youtu.be/x"), undefined);
+});
+
+test("flags a well-formed URL for the wrong platform", () => {
+	assert.equal(
+		urlFieldWarning("youtube", "urls", "https://google.com"),
+		"Not a YouTube URL: https://google.com"
+	);
+});
+
+test("flags a malformed / scheme-less URL", () => {
+	assert.equal(
+		urlFieldWarning("reddit", "urls", "reddit.com/r/python"),
+		"Not a Reddit URL: reddit.com/r/python"
+	);
+});
+
+test("lists every offending line, ignoring blanks", () => {
+	const value = "https://www.tiktok.com/@a\n\n https://google.com \nnot-a-url";
+	assert.equal(
+		urlFieldWarning("tiktok", "urls", value),
+		"Not a TikTok URL: https://google.com, not-a-url"
+	);
+});
+
+test("matches marketplace/shortlink host variants", () => {
+	assert.equal(urlFieldWarning("amazon", "urls", "https://www.amazon.fr/dp/B0"), undefined);
+	assert.equal(urlFieldWarning("amazon", "urls", "https://a.co/d/xyz"), undefined);
+	assert.equal(urlFieldWarning("google_maps", "urls", "https://maps.app.goo.gl/xyz"), undefined);
+});
+
+test("only inspects known URL fields, and skips platforms without a rule", () => {
+	// search_terms is not a URL field — never warned.
+	assert.equal(urlFieldWarning("amazon", "search_terms", "laptop"), undefined);
+	// instagram has no rule (its urls accept bare @handles).
+	assert.equal(urlFieldWarning("instagram", "urls", "natgeo"), undefined);
+});
+
+test("collects per-field warnings across the form values", () => {
+	assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://google.com", country: "us" }), {
+		urls: "Not an Indeed URL: https://google.com",
+	});
+	assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://www.indeed.com/jobs?q=x" }), {});
+});

surfsense_web/lib/playground/url-hints.ts

@@ -0,0 +1,90 @@
+/**
+ * Client-side, per-platform URL hints for the playground (UX only).
+ *
+ * The scraper API stays authoritative: it rejects malformed URLs (422), and
+ * each platform decides what it can actually scrape. These hints only *warn*,
+ * before a run, when a line in a platform's ``urls`` field is not a URL for
+ * that platform — so a typo (wrong site, missing scheme) is caught without a
+ * round-trip. They never block the run.
+ */
+
+interface PlatformUrlRule {
+	/** Host suffixes (``youtube.com``) or ``.``-terminated prefixes (``amazon.``). */
+	hosts: string[];
+	/** Human platform name used in the warning message. */
+	label: string;
+}
+
+/**
+ * Keyed by platform slug. Instagram is intentionally absent: its ``urls`` field
+ * also accepts bare ``@handles``, so a non-URL line there is not a mistake.
+ */
+const PLATFORM_URL_RULES: Record<string, PlatformUrlRule> = {
+	amazon: { hosts: ["amazon.", "amzn.to", "a.co"], label: "Amazon" },
+	walmart: { hosts: ["walmart.com"], label: "Walmart" },
+	reddit: { hosts: ["reddit.com", "redd.it"], label: "Reddit" },
+	youtube: { hosts: ["youtube.com", "youtu.be"], label: "YouTube" },
+	tiktok: { hosts: ["tiktok.com"], label: "TikTok" },
+	google_maps: { hosts: ["google.", "goo.gl"], label: "Google Maps" },
+	indeed: { hosts: ["indeed.com"], label: "Indeed" },
+};
+
+/** The array fields that carry platform URLs at the capability layer. */
+const URL_FIELD_NAMES = new Set(["urls", "video_urls", "startUrls"]);
+
+function hostMatches(host: string, patterns: string[]): boolean {
+	return patterns.some((pattern) =>
+		pattern.endsWith(".")
+			? host.includes(pattern)
+			: host === pattern || host.endsWith(`.${pattern}`)
+	);
+}
+
+function article(word: string): "a" | "an" {
+	return /^[aeiou]/i.test(word) ? "an" : "a";
+}
+
+function isPlatformUrl(line: string, rule: PlatformUrlRule): boolean {
+	try {
+		const { protocol, hostname } = new URL(line);
+		if (protocol !== "http:" && protocol !== "https:") return false;
+		return hostMatches(hostname.toLowerCase(), rule.hosts);
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Warn if any line in a platform ``urls`` field is not a URL for that platform.
+ * Returns ``undefined`` when the field/platform has no rule or every line is OK.
+ */
+export function urlFieldWarning(
+	platform: string,
+	fieldName: string,
+	value: unknown
+): string | undefined {
+	const rule = PLATFORM_URL_RULES[platform];
+	if (!rule || !URL_FIELD_NAMES.has(fieldName)) return undefined;
+
+	const badLines = String(value ?? "")
+		.split("\n")
+		.map((line) => line.trim())
+		.filter(Boolean)
+		.filter((line) => !isPlatformUrl(line, rule));
+
+	if (badLines.length === 0) return undefined;
+	return `Not ${article(rule.label)} ${rule.label} URL: ${badLines.join(", ")}`;
+}
+
+/** Per-field warnings for the current form values (empty when every URL looks valid). */
+export function urlFieldWarnings(
+	platform: string,
+	values: Record<string, unknown>
+): Record<string, string> {
+	const warnings: Record<string, string> = {};
+	for (const [name, value] of Object.entries(values)) {
+		const warning = urlFieldWarning(platform, name, value);
+		if (warning) warnings[name] = warning;
+	}
+	return warnings;
+}

surfsense_web/lib/url.ts

@@ -12,3 +12,13 @@ export function tryGetHostname(url: string): string | undefined {
 		return undefined;
 	}
 }
+
+/** True when the value parses as an http(s) URL — mirrors the backend's boundary rule. */
+export function isHttpUrl(value: string): boolean {
+	try {
+		const { protocol } = new URL(value);
+		return protocol === "http:" || protocol === "https:";
+	} catch {
+		return false;
+	}
+}