diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index 42efe38a7..b308d3770 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -8,6 +8,7 @@ from collections import defaultdict from contextlib import asynccontextmanager from datetime import UTC, datetime from threading import Lock +from typing import Any import redis from fastapi import Depends, FastAPI, HTTPException, Request, Response, status @@ -95,17 +96,21 @@ def _build_error_response( code: str = "INTERNAL_ERROR", request_id: str = "", extra_headers: dict[str, str] | None = None, + fields: list[dict[str, Any]] | None = None, ) -> JSONResponse: """Build the standardized error envelope (new ``error`` + legacy ``detail``).""" + error: dict[str, Any] = { + "code": code, + "message": message, + "status": status_code, + "request_id": request_id, + "timestamp": datetime.now(UTC).isoformat(), + "report_url": ISSUES_URL, + } + if fields: + error["fields"] = fields body = { - "error": { - "code": code, - "message": message, - "status": status_code, - "request_id": request_id, - "timestamp": datetime.now(UTC).isoformat(), - "report_url": ISSUES_URL, - }, + "error": error, "detail": message, } headers = {"X-Request-ID": request_id} @@ -223,16 +228,35 @@ def _http_exception_handler(request: Request, exc: HTTPException) -> JSONRespons def _validation_error_handler( request: Request, exc: RequestValidationError ) -> JSONResponse: - """Return 422 with field-level detail in the standard envelope.""" + """Return 422 with field-level detail in the standard envelope. + + ``error.fields`` carries each failure's location path and message so clients + can attach errors to the offending input; ``message`` is the flat summary. + """ rid = _get_request_id(request) - fields = [] - for err in exc.errors(): - loc = " -> ".join(str(part) for part in err.get("loc", [])) - fields.append(f"{loc}: {err.get('msg', 'invalid')}") - message = ( - f"Validation failed: {'; '.join(fields)}" if fields else "Validation failed." + fields = [ + { + "loc": [str(part) for part in err.get("loc", ())], + # Drop pydantic's "Value error, " prefix so messages read for humans. + "msg": str(err.get("msg", "invalid")).removeprefix("Value error, "), + } + for err in exc.errors() + ] + + def _segment(field: dict[str, Any]) -> str: + # Drop the "body" request root so model-level errors read as a plain + # sentence and field errors read as "field -> sub", not "body -> field". + path = field["loc"] + if path and path[0] == "body": + path = path[1:] + loc = " -> ".join(path) + return f"{loc}: {field['msg']}" if loc else field["msg"] + + summary = "; ".join(_segment(f) for f in fields) + message = f"Validation failed: {summary}" if fields else "Validation failed." + return _build_error_response( + 422, message, code="VALIDATION_ERROR", request_id=rid, fields=fields ) - return _build_error_response(422, message, code="VALIDATION_ERROR", request_id=rid) def _unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: diff --git a/surfsense_backend/app/capabilities/amazon/scrape/schemas.py b/surfsense_backend/app/capabilities/amazon/scrape/schemas.py index e510e2eea..b2a847058 100644 --- a/surfsense_backend/app/capabilities/amazon/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/amazon/scrape/schemas.py @@ -4,6 +4,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.amazon import ProductItem MAX_AMAZON_SOURCES = 20 @@ -13,7 +14,7 @@ MAX_AMAZON_RESULTS = 1000 class ScrapeInput(BaseModel): """Agent-facing controls for public product discovery and enrichment.""" - urls: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES) + urls: list[HttpUrlStr] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES) search_terms: list[str] = Field(default_factory=list, max_length=MAX_AMAZON_SOURCES) max_items: int = Field(default=10, ge=1, le=100) domain: str = Field( diff --git a/surfsense_backend/app/capabilities/core/validation.py b/surfsense_backend/app/capabilities/core/validation.py new file mode 100644 index 000000000..3e0885488 --- /dev/null +++ b/surfsense_backend/app/capabilities/core/validation.py @@ -0,0 +1,24 @@ +"""Shared Pydantic field types for capability I/O schemas.""" + +from __future__ import annotations + +from typing import Annotated +from urllib.parse import urlsplit + +import validators +from pydantic import AfterValidator +from pydantic_core import PydanticCustomError + +_HTTP_SCHEMES = frozenset({"http", "https"}) + + +def _validate_http_url(value: str) -> str: + """Accept only well-formed http(s) URLs, returned trimmed and unchanged.""" + url = value.strip() + if not validators.url(url) or urlsplit(url).scheme.lower() not in _HTTP_SCHEMES: + raise PydanticCustomError("http_url", "must be a valid http(s) URL") + return url + + +HttpUrlStr = Annotated[str, AfterValidator(_validate_http_url)] +"""A request URL validated as http(s) and kept as ``str`` (no normalization).""" diff --git a/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py b/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py index 1868765e9..26cdad29e 100644 --- a/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py +++ b/surfsense_backend/app/capabilities/google_maps/reviews/schemas.py @@ -10,6 +10,7 @@ from typing import Literal from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.google_maps import ReviewItem MAX_MAPS_REVIEW_SOURCES = 20 @@ -17,7 +18,7 @@ MAX_MAPS_REVIEW_SOURCES = 20 class ReviewsInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_MAPS_REVIEW_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py b/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py index 95833c8f7..7cf1ca6a1 100644 --- a/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/google_maps/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.google_maps import PlaceItem MAX_MAPS_SOURCES = 20 @@ -26,7 +27,7 @@ class ScrapeInput(BaseModel): "(at least one is required). Pair with location to scope a search." ), ) - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_MAPS_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py index 02e29100d..7c5dacb59 100644 --- a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.indeed_jobs import IndeedItem from app.proprietary.platforms.indeed_jobs.schemas import ( IndeedJobType, @@ -26,7 +27,7 @@ MAX_INDEED_ITEMS = 100 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_INDEED_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/reddit/scrape/schemas.py b/surfsense_backend/app/capabilities/reddit/scrape/schemas.py index 0c2da536e..9e0143ba7 100644 --- a/surfsense_backend/app/capabilities/reddit/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/reddit/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.reddit import RedditItem from app.proprietary.platforms.reddit.schemas import RedditSort, RedditTime @@ -21,7 +22,7 @@ MAX_REDDIT_ITEMS = 100 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_REDDIT_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py index f3c68d445..61ae0c756 100644 --- a/surfsense_backend/app/capabilities/tiktok/comments/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/comments/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field +from app.capabilities.core.validation import HttpUrlStr from app.capabilities.tiktok.scrape.schemas import ( MAX_TIKTOK_ITEMS, MAX_TIKTOK_SOURCES, @@ -18,7 +19,7 @@ from app.proprietary.platforms.tiktok import CommentItem class CommentsInput(BaseModel): - video_urls: list[str] = Field( + video_urls: list[HttpUrlStr] = Field( min_length=1, max_length=MAX_TIKTOK_SOURCES, description="TikTok video URLs (/@/video/) to pull comments from.", diff --git a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py index ac3792712..b139e6e49 100644 --- a/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/tiktok/scrape/schemas.py @@ -12,6 +12,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.tiktok import TikTokVideoItem MAX_TIKTOK_SOURCES = 20 @@ -22,7 +23,7 @@ MAX_TIKTOK_ITEMS = 100 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_TIKTOK_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py index faf56ed5b..bd1dc73d9 100644 --- a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py +++ b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py @@ -11,13 +11,14 @@ from typing import Literal from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.walmart import ReviewItem MAX_WALMART_REVIEW_SOURCES = 20 class ReviewsInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_WALMART_REVIEW_SOURCES, description=( diff --git a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py index 2d42dc371..7faaabd1d 100644 --- a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py @@ -6,6 +6,7 @@ from urllib.parse import quote_plus from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.walmart import ProductItem MAX_WALMART_SOURCES = 20 @@ -15,7 +16,7 @@ MAX_WALMART_RESULTS = 1000 class ScrapeInput(BaseModel): """Agent-facing controls for public Walmart product discovery and enrichment.""" - urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) + urls: list[HttpUrlStr] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) search_terms: list[str] = Field( default_factory=list, max_length=MAX_WALMART_SOURCES ) diff --git a/surfsense_backend/app/capabilities/web/crawl/schemas.py b/surfsense_backend/app/capabilities/web/crawl/schemas.py index d6c9c38a3..30e8ff6ae 100644 --- a/surfsense_backend/app/capabilities/web/crawl/schemas.py +++ b/surfsense_backend/app/capabilities/web/crawl/schemas.py @@ -18,6 +18,8 @@ from typing import Literal from pydantic import BaseModel, Field +from app.capabilities.core.validation import HttpUrlStr + MAX_START_URLS = 20 """Per-call cap on seed URLs: bounds a synchronous request's fan-out (05).""" @@ -29,7 +31,7 @@ MAX_CRAWL_PAGES = 200 class CrawlInput(BaseModel): - startUrls: list[str] = Field( + startUrls: list[HttpUrlStr] = Field( min_length=1, max_length=MAX_START_URLS, description=( diff --git a/surfsense_backend/app/capabilities/youtube/comments/schemas.py b/surfsense_backend/app/capabilities/youtube/comments/schemas.py index a9da5b7a2..ecfa80026 100644 --- a/surfsense_backend/app/capabilities/youtube/comments/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/comments/schemas.py @@ -10,6 +10,7 @@ from typing import Literal from pydantic import BaseModel, Field +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.youtube import CommentItem MAX_COMMENT_VIDEOS = 20 @@ -17,7 +18,7 @@ MAX_COMMENT_VIDEOS = 20 class CommentsInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( min_length=1, max_length=MAX_COMMENT_VIDEOS, description="YouTube video URLs to fetch comments (and replies) for (1-20).", diff --git a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py index 4d80769b6..63a1df060 100644 --- a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py @@ -10,6 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator +from app.capabilities.core.validation import HttpUrlStr from app.proprietary.platforms.youtube import VideoItem MAX_YOUTUBE_SOURCES = 20 @@ -17,7 +18,7 @@ MAX_YOUTUBE_SOURCES = 20 class ScrapeInput(BaseModel): - urls: list[str] = Field( + urls: list[HttpUrlStr] = Field( default_factory=list, max_length=MAX_YOUTUBE_SOURCES, description=( diff --git a/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py b/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py index 0e68425a9..aa7df36e9 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py @@ -15,6 +15,9 @@ ResolvedKind = Literal["video", "channel", "playlist", "hashtag", "search"] _PLAYLIST_ID_RE = re.compile(r"[?&]list=([\w-]+)") +_YOUTUBE_HOSTS = frozenset({"www.youtube.com", "youtube.com", "m.youtube.com"}) +"""Hosts whose page paths (/@, /channel, /playlist, ...) are trusted as YouTube.""" + @dataclass(frozen=True) class ResolvedUrl: @@ -29,7 +32,7 @@ def get_youtube_video_id(url: str) -> str | None: hostname = parsed.hostname or "" if hostname == "youtu.be": return parsed.path[1:] or None - if hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"): + if hostname in _YOUTUBE_HOSTS: if parsed.path == "/watch": return parse_qs(parsed.query).get("v", [None])[0] for prefix in ("/embed/", "/v/", "/shorts/"): @@ -41,17 +44,21 @@ def get_youtube_video_id(url: str) -> str | None: def resolve_url(url: str) -> ResolvedUrl | None: """Classify a YouTube URL into a scrape job, or ``None`` if unrecognized.""" parsed = urlparse(url) + hostname = (parsed.hostname or "").lower() path = parsed.path or "" - # Shorts are videos with their own path. - if "/shorts/" in path: - vid = path.split("/shorts/")[1].split("/")[0] - return ResolvedUrl("video", vid, url) if vid else None - + # Videos: watch / youtu.be / embed / shorts. get_youtube_video_id validates + # the host itself, so this also covers youtu.be short links. video_id = get_youtube_video_id(url) if video_id: return ResolvedUrl("video", video_id, url) + # Every remaining shape is a youtube.com page keyed off its path. Require a + # YouTube host so a well-formed non-YouTube URL (e.g. https://evil.com/@x) + # is rejected instead of misclassified as a channel by its path alone. + if hostname not in _YOUTUBE_HOSTS: + return None + # Playlist (either a /playlist page or any URL carrying ?list=). playlist_match = _PLAYLIST_ID_RE.search(url) if path.startswith("/playlist") and playlist_match: diff --git a/surfsense_backend/tests/unit/capabilities/core/__init__.py b/surfsense_backend/tests/unit/capabilities/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/core/test_validation.py b/surfsense_backend/tests/unit/capabilities/core/test_validation.py new file mode 100644 index 000000000..03db512ba --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/core/test_validation.py @@ -0,0 +1,32 @@ +"""``HttpUrlStr`` boundary type: accept well-formed http(s) URLs, reject the rest.""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel, ValidationError + +from app.capabilities.core.validation import HttpUrlStr + +pytestmark = pytest.mark.unit + + +class _Model(BaseModel): + urls: list[HttpUrlStr] + + +def test_accepts_http_and_https_urls_unchanged() -> None: + model = _Model(urls=["https://example.com/path?q=1", "http://a.co"]) + assert model.urls == ["https://example.com/path?q=1", "http://a.co"] + + +def test_trims_surrounding_whitespace() -> None: + assert _Model(urls=[" https://example.com "]).urls == ["https://example.com"] + + +@pytest.mark.parametrize( + "bad", + ["not-a-url", "example.com", "ftp://example.com", "http://", "javascript:alert(1)"], +) +def test_rejects_malformed_or_non_http_urls(bad: str) -> None: + with pytest.raises(ValidationError): + _Model(urls=[bad]) diff --git a/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py b/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py index f7b9a2b39..080dda2b2 100644 --- a/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py @@ -20,6 +20,11 @@ def test_requires_at_least_one_start_url() -> None: CrawlInput(startUrls=[]) +def test_rejects_malformed_start_url() -> None: + with pytest.raises(ValidationError): + CrawlInput(startUrls=["not-a-url"]) + + def test_camelcase_fields_and_defaults() -> None: model = CrawlInput(startUrls=["https://e.com"]) assert model.startUrls == ["https://e.com"] diff --git a/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py index 55f591c63..6b7b460a9 100644 --- a/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py @@ -946,8 +946,22 @@ def test_resolve_url(url, kind, value): assert resolved.value == value -def test_resolve_url_unrecognized(): - assert resolve_url("https://example.com/foo") is None +@pytest.mark.parametrize( + "url", + [ + "https://example.com/foo", + # Well-formed non-YouTube URLs whose path mimics a YouTube page must be + # rejected, not misclassified by path alone (host-spoof guard). + "https://evil.com/@Apify", + "https://evil.com/channel/UC123456789abc", + "https://evil.com/shorts/abc123", + "https://evil.com/playlist?list=PL123", + "https://evil.com/hashtag/tech", + "https://evil.com/results?search_query=web+scraping", + ], +) +def test_resolve_url_unrecognized(url): + assert resolve_url(url) is None # --- optional: exercise captured real fixtures if present -------------------- diff --git a/surfsense_backend/tests/unit/test_error_contract.py b/surfsense_backend/tests/unit/test_error_contract.py index ec8021290..e1e160232 100644 --- a/surfsense_backend/tests/unit/test_error_contract.py +++ b/surfsense_backend/tests/unit/test_error_contract.py @@ -14,6 +14,7 @@ import json import pytest from fastapi import HTTPException +from pydantic import BaseModel, model_validator from starlette.testclient import TestClient from app.exceptions import ( @@ -32,6 +33,23 @@ from app.exceptions import ( pytestmark = pytest.mark.unit +class _RequireOne(BaseModel): + """Body model with a cross-field rule, used to test the 422 summary. + + Defined at module scope (not inside the app factory) so FastAPI's ``Body`` + ``TypeAdapter`` can resolve the forward reference. + """ + + a: str | None = None + b: str | None = None + + @model_validator(mode="after") + def _at_least_one(self): + if not self.a and not self.b: + raise ValueError("Provide at least one of 'a' or 'b'.") + return self + + # --------------------------------------------------------------------------- # Helpers - lightweight FastAPI app that re-uses the real global handlers # --------------------------------------------------------------------------- @@ -39,7 +57,7 @@ pytestmark = pytest.mark.unit def _make_test_app(): """Build a minimal FastAPI app with the same handlers as the real one.""" - from fastapi import FastAPI + from fastapi import Body, FastAPI from fastapi.exceptions import RequestValidationError from pydantic import BaseModel @@ -124,6 +142,10 @@ def _make_test_app(): async def validated(item: Item): return item.model_dump() + @app.post("/require-one") + async def require_one(payload: _RequireOne = Body(...)): + return payload.model_dump() + return app @@ -277,6 +299,26 @@ class TestValidationErrorHandler: body = _assert_envelope(resp, 422) assert body["error"]["code"] == "VALIDATION_ERROR" + def test_field_error_drops_body_root(self, client): + # The "body" request root is noise in the human summary; the field name + # is kept so clients can still attach the error inline. + resp = client.post("/require-one", json={"a": 123}) + body = _assert_envelope(resp, 422) + msg = body["error"]["message"] + assert "body" not in msg + assert "a:" in msg + assert body["error"]["fields"][0]["loc"] == ["body", "a"] + + def test_model_level_error_reads_as_sentence(self, client): + # A cross-field rule (loc == ["body"]) must read as a plain sentence, + # not "body: ...", while fields still carry the location for clients. + resp = client.post("/require-one", json={}) + body = _assert_envelope(resp, 422) + assert body["error"]["message"] == ( + "Validation failed: Provide at least one of 'a' or 'b'." + ) + assert body["error"]["fields"][0]["loc"] == ["body"] + # --------------------------------------------------------------------------- # SurfSenseError class hierarchy unit tests diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx index 05434b953..ba7312d7b 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/api-reference.tsx @@ -2,6 +2,7 @@ import { Check, ChevronRight, Copy } from "lucide-react"; import { useMemo, useState } from "react"; +import { JsonView } from "@/components/json-view"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { BACKEND_URL } from "@/lib/env-config"; @@ -51,9 +52,9 @@ function SchemaBlock({ title, schema }: { title: string; schema: Record
- 
-					{json}
-
+
+ +
); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx index 72262d440..106200a70 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/output-viewer.tsx @@ -2,15 +2,9 @@ import { Check, Copy, Download } from "lucide-react"; import { useMemo, useState } from "react"; +import { JsonView } from "@/components/json-view"; import { Button } from "@/components/ui/button"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; +import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { downloadCsv, rowsToCsv } from "@/lib/playground/csv"; @@ -48,10 +42,11 @@ function ResultTable({ items }: { items: Record[] }) { const rows = items.slice(0, MAX_TABLE_ROWS); + // overscroll-contain keeps table scrolling from chaining to the page. return ( -
- - +
+
+ {columns.map((col) => ( @@ -72,7 +67,7 @@ function ResultTable({ items }: { items: Record[] }) { ))} -
+
); } @@ -152,9 +147,9 @@ export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBa )} ) : ( - 
-					{json}
-
+
+ +
)} ); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx index 32624ca55..6bee3e4e5 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx @@ -12,6 +12,7 @@ import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; import { scrapersApiService } from "@/lib/apis/scrapers-api.service"; import { AppError } from "@/lib/error"; import { findVerb } from "@/lib/playground/catalog"; +import { fieldErrorsFromError } from "@/lib/playground/field-errors"; import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format"; import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema"; import { @@ -19,6 +20,7 @@ import { getAmazonFieldOptions, hasAmazonFranceValue, } from "@/lib/playground/platform-overrides/amazon"; +import { urlFieldWarnings } from "@/lib/playground/url-hints"; import { ApiReference } from "./api-reference"; import { OutputViewer } from "./output-viewer"; import { RunProgressPanel } from "./run-progress-panel"; @@ -73,7 +75,12 @@ function getRunErrorMessage(error: unknown): string { } if (status === 422) { - return "Invalid input. Check the fields above and try again."; + if (Object.keys(fieldErrorsFromError(error)).length > 0) { + return "Invalid input. Check the highlighted fields."; + } + return error instanceof Error && error.message + ? error.message + : "Invalid input. Check the fields above and try again."; } return error instanceof Error && error.message @@ -123,6 +130,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn const fields = useMemo(() => parseSchemaFields(capability?.input_schema), [capability]); const [values, setValues] = useState>({}); + const [fieldErrors, setFieldErrors] = useState>({}); const run = useRunStream(workspaceId); const isRunning = run.status === "running"; const previousStatusRef = useRef(run.status); @@ -164,9 +172,15 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn const handleChange = (name: string, value: unknown) => { setValues((prev) => ({ ...prev, [name]: value })); + setFieldErrors((prev) => { + if (!(name in prev)) return prev; + const { [name]: _cleared, ...rest } = prev; + return rest; + }); }; const handleRun = useCallback(() => { + setFieldErrors({}); const payload = buildPayload(fields, values); void run.start(platform, verb, payload); }, [fields, values, platform, verb, run]); @@ -178,6 +192,7 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn const endpoint = `POST /workspaces/${workspaceId}/scrapers/${platform}/${verb}`; const isAmazonScrape = platform === "amazon" && verb === "scrape"; const hasAmazonFranceUrl = useMemo(() => hasAmazonFranceValue(values), [values]); + const fieldWarnings = useMemo(() => urlFieldWarnings(platform, values), [platform, values]); useEffect(() => { const previousStatus = previousStatusRef.current; @@ -194,10 +209,21 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn } if (run.status === "error") { + const nextFieldErrors = fieldErrorsFromError(run.error); + setFieldErrors(nextFieldErrors); const key = `${run.runId ?? "run"}:error`; if (notifiedRunRef.current === key) return; notifiedRunRef.current = key; - toast.error(getRunErrorMessage(run.error)); + // Field-level failures are shown inline (the form reveals + focuses the + // first one), so only toast global failures that have no field to anchor to. + // Keep run errors until dismissed — they're actionable and shouldn't + // vanish before the user reads them. + if (Object.keys(nextFieldErrors).length === 0) { + toast.error(getRunErrorMessage(run.error), { + duration: Number.POSITIVE_INFINITY, + closeButton: true, + }); + } } }, [run.status, run.runId, run.error]); @@ -268,6 +294,8 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn onChange={handleChange} disabled={isRunning} getFieldOptions={isAmazonScrape ? getAmazonFieldOptions : undefined} + fieldErrors={fieldErrors} + fieldWarnings={fieldWarnings} /> {isAmazonScrape ? : null} diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx index 4a128f5d8..fdf41729c 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/run-detail.tsx @@ -1,6 +1,7 @@ "use client"; import { useMemo } from "react"; +import { JsonView } from "@/components/json-view"; import { Spinner } from "@/components/ui/spinner"; import { useScraperRun } from "@/hooks/use-scraper-runs"; import { OutputViewer } from "./output-viewer"; @@ -69,7 +70,7 @@ export function RunDetail({ workspaceId, runId }: { workspaceId: number; runId: {run.progress && run.progress.length > 0 && (

Progress

-
+
{run.progress.map((event, i) => (
{progressLine(event)} @@ -81,9 +82,9 @@ export function RunDetail({ workspaceId, runId }: { workspaceId: number; runId:

Input

- 
-					{JSON.stringify(run.input ?? {}, null, 2)}
-
+
+ +
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx index 57b7a547c..0d642ac3f 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/runs-table.tsx @@ -2,7 +2,7 @@ import { useInfiniteQuery } from "@tanstack/react-query"; import { ChevronDown, ChevronRight, History, Info } from "lucide-react"; -import { Fragment, useState } from "react"; +import { useState } from "react"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { @@ -13,14 +13,6 @@ import { SelectValue, } from "@/components/ui/select"; import { Spinner } from "@/components/ui/spinner"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; import { scrapersApiService } from "@/lib/apis/scrapers-api.service"; import { formatRelativeDate } from "@/lib/format-date"; import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog"; @@ -33,6 +25,12 @@ import { RunStatusBadge } from "./run-status-badge"; const PAGE_SIZE = 25; const ALL = "__all__"; +// A grid (not a ) keeps an expanded row's detail out of a table-cell, +// where overflow/max-height scroll containers get ignored. Tracks are fr/fixed +// (never content-based auto) so separate per-row grids stay column-aligned. +const ROW_GRID = + "grid grid-cols-[2rem_minmax(8rem,2fr)_minmax(4rem,1fr)_minmax(5rem,1fr)_4rem_5rem_5rem_minmax(6rem,1fr)] items-center gap-2 px-3"; + const CAPABILITY_OPTIONS = PLAYGROUND_PLATFORMS.flatMap((platform) => platform.verbs.map((verb) => verb.name) ); @@ -122,63 +120,60 @@ export function RunsTable({ workspaceId }: { workspaceId: number }) { ) : (
-
- - - - API - Origin - Status - Items - Duration - Cost - When - - - - {runs.map((run) => { - const isOpen = expanded === run.id; - return ( - - setExpanded(isOpen ? null : run.id)} - > - - {isOpen ? ( - - ) : ( - - )} - - {run.capability} - {run.origin} - - - - {run.item_count} - - {formatDuration(run.duration_ms)} - - - {formatCost(run.cost_micros)} - - - {formatRelativeDate(run.created_at)} - - - {isOpen && ( - - - - - - )} - - ); - })} - -
+
+ + API + Origin + Status + Items + Duration + Cost + When +
+ {runs.map((run) => { + const isOpen = expanded === run.id; + return ( +
+ + {isOpen && } +
+ ); + })}
)} diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx index cdd425925..b8eb72c66 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx @@ -1,7 +1,7 @@ "use client"; import { ChevronDown } from "lucide-react"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -32,6 +32,8 @@ interface SchemaFormProps { getFieldOptions?: FieldOptionsResolver; /** Field names flagged by a 422 response, shown with error styling. */ fieldErrors?: Record; + /** Client-side, non-blocking URL warnings keyed by field name. */ + fieldWarnings?: Record; } function FieldControl({ @@ -147,6 +149,7 @@ function FieldRow({ onChange, disabled, error, + warning, options, }: { field: FormField; @@ -154,6 +157,7 @@ function FieldRow({ onChange: (value: unknown) => void; disabled?: boolean; error?: string; + warning?: string; options?: FieldOption[]; }) { return ( @@ -179,6 +183,7 @@ function FieldRow({ options={options} /> {error &&

{error}

} + {!error && warning &&

{warning}

}
); } @@ -190,6 +195,7 @@ export function SchemaForm({ disabled, getFieldOptions, fieldErrors, + fieldWarnings, }: SchemaFormProps) { const [showAdvanced, setShowAdvanced] = useState(false); @@ -199,6 +205,25 @@ export function SchemaForm({ return { primary: primaryFields, advanced: advancedFields }; }, [fields]); + // First invalid field in display order; drives reveal + focus below. + const firstErrorName = useMemo( + () => (fieldErrors ? fields.find((f) => fieldErrors[f.name])?.name : undefined), + [fields, fieldErrors] + ); + + // Reveal the section holding an invalid field, then move focus to it, so an + // error is never left hidden inside the collapsed "Advanced" group. + useEffect(() => { + if (!firstErrorName) return; + if (advanced.some((f) => f.name === firstErrorName)) setShowAdvanced(true); + const raf = requestAnimationFrame(() => { + const el = document.getElementById(`field-${firstErrorName}`); + el?.focus(); + el?.scrollIntoView({ block: "center", behavior: "smooth" }); + }); + return () => cancelAnimationFrame(raf); + }, [firstErrorName, advanced]); + return (
{primary.map((field) => ( @@ -209,6 +234,7 @@ export function SchemaForm({ onChange={(value) => onChange(field.name, value)} disabled={disabled} error={fieldErrors?.[field.name]} + warning={fieldWarnings?.[field.name]} options={getFieldOptions?.(field)} /> ))} @@ -239,6 +265,7 @@ export function SchemaForm({ onChange={(value) => onChange(field.name, value)} disabled={disabled} error={fieldErrors?.[field.name]} + warning={fieldWarnings?.[field.name]} options={getFieldOptions?.(field)} /> ))} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx index b7fbae91c..5a4540bcf 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx @@ -8,8 +8,18 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; +import { isHttpUrl } from "@/lib/url"; +import { cn } from "@/lib/utils"; import type { ConnectorConfigProps } from "../index"; +/** Return the malformed (non-http/https) lines from a newline-separated URL list. */ +function invalidUrlLines(value: string): string[] { + return value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !isHttpUrl(line)); +} + export const WebcrawlerConfig: FC = ({ connector, onConfigChange }) => { // Initialize with existing config values const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; @@ -19,6 +29,8 @@ export const WebcrawlerConfig: FC = ({ connector, onConfig const [initialUrls, setInitialUrls] = useState(existingUrls); const [showApiKey, setShowApiKey] = useState(false); + const invalidUrls = invalidUrlLines(initialUrls); + const handleApiKeyChange = (value: string) => { setApiKey(value); if (onConfigChange) { @@ -108,11 +120,21 @@ export const WebcrawlerConfig: FC = ({ connector, onConfig placeholder="https://example.com https://docs.example.com https://blog.example.com" value={initialUrls} onChange={(e) => handleUrlsChange(e.target.value)} - className="min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none" + aria-invalid={invalidUrls.length > 0} + className={cn( + "min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none", + invalidUrls.length > 0 && "border-destructive" + )} /> -

- Enter URLs to crawl (one per line). You can add more URLs later. -

+ {invalidUrls.length > 0 ? ( +

+ Not valid http(s) URLs: {invalidUrls.join(", ")} +

+ ) : ( +

+ Enter URLs to crawl (one per line). You can add more URLs later. +

+ )}
{/* Info Alert */} diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 224f8bb53..8e5daa00a 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -10,6 +10,7 @@ import { AuthorizationError, NetworkError, NotFoundError, + type ValidationFieldError, } from "../error"; enum ResponseType { @@ -189,6 +190,9 @@ class BaseApiService { const requestId: string | undefined = envelope?.request_id ?? response.headers.get("X-Request-ID") ?? undefined; const reportUrl: string | undefined = envelope?.report_url; + const validationFields: ValidationFieldError[] | undefined = Array.isArray(envelope?.fields) + ? envelope.fields + : undefined; // Handle 401 - try to refresh token first (only once) if (response.status === 401) { @@ -233,8 +237,8 @@ class BaseApiService { response.status, response.statusText ); - default: - throw new AppError( + default: { + const appError = new AppError( errorMessage || "Something went wrong", response.status, response.statusText, @@ -242,6 +246,9 @@ class BaseApiService { requestId, reportUrl ); + appError.fields = validationFields; + throw appError; + } } } refreshRetryBlockedUntil.delete(getRefreshRetryKey(mergedOptions.method, url)); @@ -313,12 +320,23 @@ class BaseApiService { throw networkError; } - console.error("Request failed:", JSON.stringify(error)); - // Only 5xx server faults are unexpected. 4xx (validation, authz, 404) - // are expected behavior — capturing them was billable error-tracking - // noise. AuthenticationError (401) is a 4xx and stays excluded. - if (error instanceof AppError && error.status >= 500) { - captureApiException(error, url, options?.method); + // Handled client errors (validation, credits, auth, not-found, ...) are + // expected outcomes surfaced to the user via typed errors + toasts — not + // app faults. Skip logging/telemetry for them so they don't spam the + // console, PostHog, or Next.js's dev error overlay. + const isHandledClientError = + error instanceof AppError && + typeof error.status === "number" && + error.status >= 400 && + error.status < 500; + + if (!isHandledClientError) { + console.error("Request failed:", JSON.stringify(error)); + // Only 5xx server faults are unexpected; 4xx are handled above and + // network outages were already captured before this point. + if (error instanceof AppError && error.status >= 500) { + captureApiException(error, url, options?.method); + } } throw error; } diff --git a/surfsense_web/lib/error.ts b/surfsense_web/lib/error.ts index 10cb3706e..8c87007b8 100644 --- a/surfsense_web/lib/error.ts +++ b/surfsense_web/lib/error.ts @@ -1,11 +1,19 @@ export const SURFSENSE_ISSUES_URL = "https://github.com/MODSetter/SurfSense/issues"; +/** One field-level failure from a 422 validation response. */ +export interface ValidationFieldError { + loc: string[]; + msg: string; +} + export class AppError extends Error { status?: number; statusText?: string; code?: string; requestId?: string; reportUrl?: string; + /** Per-field failures from a 422 response, keyed by their location path. */ + fields?: ValidationFieldError[]; constructor( message: string, status?: number, diff --git a/surfsense_web/lib/playground/field-errors.test.ts b/surfsense_web/lib/playground/field-errors.test.ts new file mode 100644 index 000000000..273cf1cc3 --- /dev/null +++ b/surfsense_web/lib/playground/field-errors.test.ts @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { AppError } from "@/lib/error"; +import { fieldErrorsFromError } from "./field-errors"; + +// Run with: pnpm exec tsx --test lib/playground/field-errors.test.ts + +function validationError(fields: { loc: string[]; msg: string }[]): AppError { + const error = new AppError("Validation failed", 422, "Unprocessable Entity", "VALIDATION_ERROR"); + error.fields = fields; + return error; +} + +test("maps a body field failure to its top-level field name", () => { + const error = validationError([ + { loc: ["body", "startUrls", "0"], msg: "must be a valid http(s) URL" }, + ]); + assert.deepEqual(fieldErrorsFromError(error), { + startUrls: "must be a valid http(s) URL", + }); +}); + +test("keeps the first failure per field", () => { + const error = validationError([ + { loc: ["body", "urls", "0"], msg: "first" }, + { loc: ["body", "urls", "3"], msg: "second" }, + ]); + assert.deepEqual(fieldErrorsFromError(error), { urls: "first" }); +}); + +test("returns nothing for non-AppError or errors without fields", () => { + assert.deepEqual(fieldErrorsFromError(new Error("boom")), {}); + assert.deepEqual(fieldErrorsFromError(new AppError("no fields", 500)), {}); +}); diff --git a/surfsense_web/lib/playground/field-errors.ts b/surfsense_web/lib/playground/field-errors.ts new file mode 100644 index 000000000..668d2d9c1 --- /dev/null +++ b/surfsense_web/lib/playground/field-errors.ts @@ -0,0 +1,18 @@ +import { AppError } from "@/lib/error"; + +/** Map a 422 response's field failures to ``{ fieldName: message }`` for the form. */ +export function fieldErrorsFromError(error: unknown): Record { + if (!(error instanceof AppError) || !error.fields) return {}; + const errors: Record = {}; + for (const { loc, msg } of error.fields) { + const name = topLevelField(loc); + if (name && !errors[name]) errors[name] = msg; + } + return errors; +} + +/** The input field a location path points at, dropping the ``body`` request root. */ +function topLevelField(loc: string[]): string | undefined { + const path = loc[0] === "body" ? loc.slice(1) : loc; + return path[0]; +} diff --git a/surfsense_web/lib/playground/url-hints.test.ts b/surfsense_web/lib/playground/url-hints.test.ts new file mode 100644 index 000000000..84570c8e9 --- /dev/null +++ b/surfsense_web/lib/playground/url-hints.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { urlFieldWarning, urlFieldWarnings } from "./url-hints"; + +// Run with: pnpm exec tsx --test lib/playground/url-hints.test.ts + +test("accepts a matching platform URL (no warning)", () => { + assert.equal(urlFieldWarning("youtube", "urls", "https://www.youtube.com/watch?v=x"), undefined); + assert.equal(urlFieldWarning("youtube", "urls", "https://youtu.be/x"), undefined); +}); + +test("flags a well-formed URL for the wrong platform", () => { + assert.equal( + urlFieldWarning("youtube", "urls", "https://google.com"), + "Not a YouTube URL: https://google.com" + ); +}); + +test("flags a malformed / scheme-less URL", () => { + assert.equal( + urlFieldWarning("reddit", "urls", "reddit.com/r/python"), + "Not a Reddit URL: reddit.com/r/python" + ); +}); + +test("lists every offending line, ignoring blanks", () => { + const value = "https://www.tiktok.com/@a\n\n https://google.com \nnot-a-url"; + assert.equal( + urlFieldWarning("tiktok", "urls", value), + "Not a TikTok URL: https://google.com, not-a-url" + ); +}); + +test("matches marketplace/shortlink host variants", () => { + assert.equal(urlFieldWarning("amazon", "urls", "https://www.amazon.fr/dp/B0"), undefined); + assert.equal(urlFieldWarning("amazon", "urls", "https://a.co/d/xyz"), undefined); + assert.equal(urlFieldWarning("google_maps", "urls", "https://maps.app.goo.gl/xyz"), undefined); +}); + +test("only inspects known URL fields, and skips platforms without a rule", () => { + // search_terms is not a URL field — never warned. + assert.equal(urlFieldWarning("amazon", "search_terms", "laptop"), undefined); + // instagram has no rule (its urls accept bare @handles). + assert.equal(urlFieldWarning("instagram", "urls", "natgeo"), undefined); +}); + +test("collects per-field warnings across the form values", () => { + assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://google.com", country: "us" }), { + urls: "Not an Indeed URL: https://google.com", + }); + assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://www.indeed.com/jobs?q=x" }), {}); +}); diff --git a/surfsense_web/lib/playground/url-hints.ts b/surfsense_web/lib/playground/url-hints.ts new file mode 100644 index 000000000..e45a5d8ea --- /dev/null +++ b/surfsense_web/lib/playground/url-hints.ts @@ -0,0 +1,90 @@ +/** + * Client-side, per-platform URL hints for the playground (UX only). + * + * The scraper API stays authoritative: it rejects malformed URLs (422), and + * each platform decides what it can actually scrape. These hints only *warn*, + * before a run, when a line in a platform's ``urls`` field is not a URL for + * that platform — so a typo (wrong site, missing scheme) is caught without a + * round-trip. They never block the run. + */ + +interface PlatformUrlRule { + /** Host suffixes (``youtube.com``) or ``.``-terminated prefixes (``amazon.``). */ + hosts: string[]; + /** Human platform name used in the warning message. */ + label: string; +} + +/** + * Keyed by platform slug. Instagram is intentionally absent: its ``urls`` field + * also accepts bare ``@handles``, so a non-URL line there is not a mistake. + */ +const PLATFORM_URL_RULES: Record = { + amazon: { hosts: ["amazon.", "amzn.to", "a.co"], label: "Amazon" }, + walmart: { hosts: ["walmart.com"], label: "Walmart" }, + reddit: { hosts: ["reddit.com", "redd.it"], label: "Reddit" }, + youtube: { hosts: ["youtube.com", "youtu.be"], label: "YouTube" }, + tiktok: { hosts: ["tiktok.com"], label: "TikTok" }, + google_maps: { hosts: ["google.", "goo.gl"], label: "Google Maps" }, + indeed: { hosts: ["indeed.com"], label: "Indeed" }, +}; + +/** The array fields that carry platform URLs at the capability layer. */ +const URL_FIELD_NAMES = new Set(["urls", "video_urls", "startUrls"]); + +function hostMatches(host: string, patterns: string[]): boolean { + return patterns.some((pattern) => + pattern.endsWith(".") + ? host.includes(pattern) + : host === pattern || host.endsWith(`.${pattern}`) + ); +} + +function article(word: string): "a" | "an" { + return /^[aeiou]/i.test(word) ? "an" : "a"; +} + +function isPlatformUrl(line: string, rule: PlatformUrlRule): boolean { + try { + const { protocol, hostname } = new URL(line); + if (protocol !== "http:" && protocol !== "https:") return false; + return hostMatches(hostname.toLowerCase(), rule.hosts); + } catch { + return false; + } +} + +/** + * Warn if any line in a platform ``urls`` field is not a URL for that platform. + * Returns ``undefined`` when the field/platform has no rule or every line is OK. + */ +export function urlFieldWarning( + platform: string, + fieldName: string, + value: unknown +): string | undefined { + const rule = PLATFORM_URL_RULES[platform]; + if (!rule || !URL_FIELD_NAMES.has(fieldName)) return undefined; + + const badLines = String(value ?? "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .filter((line) => !isPlatformUrl(line, rule)); + + if (badLines.length === 0) return undefined; + return `Not ${article(rule.label)} ${rule.label} URL: ${badLines.join(", ")}`; +} + +/** Per-field warnings for the current form values (empty when every URL looks valid). */ +export function urlFieldWarnings( + platform: string, + values: Record +): Record { + const warnings: Record = {}; + for (const [name, value] of Object.entries(values)) { + const warning = urlFieldWarning(platform, name, value); + if (warning) warnings[name] = warning; + } + return warnings; +} diff --git a/surfsense_web/lib/url.ts b/surfsense_web/lib/url.ts index 7bb0488d0..c351d1b18 100644 --- a/surfsense_web/lib/url.ts +++ b/surfsense_web/lib/url.ts @@ -12,3 +12,13 @@ export function tryGetHostname(url: string): string | undefined { return undefined; } } + +/** True when the value parses as an http(s) URL — mirrors the backend's boundary rule. */ +export function isHttpUrl(value: string): boolean { + try { + const { protocol } = new URL(value); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +}