From 1ad8efb6587b64fe2716b1d1c4e13379bd891d49 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 18:52:50 +0200 Subject: [PATCH] feat: add per-platform URL hints --- .../lib/playground/url-hints.test.ts | 52 +++++++++++ surfsense_web/lib/playground/url-hints.ts | 90 +++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 surfsense_web/lib/playground/url-hints.test.ts create mode 100644 surfsense_web/lib/playground/url-hints.ts diff --git a/surfsense_web/lib/playground/url-hints.test.ts b/surfsense_web/lib/playground/url-hints.test.ts new file mode 100644 index 000000000..84570c8e9 --- /dev/null +++ b/surfsense_web/lib/playground/url-hints.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { urlFieldWarning, urlFieldWarnings } from "./url-hints"; + +// Run with: pnpm exec tsx --test lib/playground/url-hints.test.ts + +test("accepts a matching platform URL (no warning)", () => { + assert.equal(urlFieldWarning("youtube", "urls", "https://www.youtube.com/watch?v=x"), undefined); + assert.equal(urlFieldWarning("youtube", "urls", "https://youtu.be/x"), undefined); +}); + +test("flags a well-formed URL for the wrong platform", () => { + assert.equal( + urlFieldWarning("youtube", "urls", "https://google.com"), + "Not a YouTube URL: https://google.com" + ); +}); + +test("flags a malformed / scheme-less URL", () => { + assert.equal( + urlFieldWarning("reddit", "urls", "reddit.com/r/python"), + "Not a Reddit URL: reddit.com/r/python" + ); +}); + +test("lists every offending line, ignoring blanks", () => { + const value = "https://www.tiktok.com/@a\n\n https://google.com \nnot-a-url"; + assert.equal( + urlFieldWarning("tiktok", "urls", value), + "Not a TikTok URL: https://google.com, not-a-url" + ); +}); + +test("matches marketplace/shortlink host variants", () => { + assert.equal(urlFieldWarning("amazon", "urls", "https://www.amazon.fr/dp/B0"), undefined); + assert.equal(urlFieldWarning("amazon", "urls", "https://a.co/d/xyz"), undefined); + assert.equal(urlFieldWarning("google_maps", "urls", "https://maps.app.goo.gl/xyz"), undefined); +}); + +test("only inspects known URL fields, and skips platforms without a rule", () => { + // search_terms is not a URL field — never warned. + assert.equal(urlFieldWarning("amazon", "search_terms", "laptop"), undefined); + // instagram has no rule (its urls accept bare @handles). + assert.equal(urlFieldWarning("instagram", "urls", "natgeo"), undefined); +}); + +test("collects per-field warnings across the form values", () => { + assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://google.com", country: "us" }), { + urls: "Not an Indeed URL: https://google.com", + }); + assert.deepEqual(urlFieldWarnings("indeed", { urls: "https://www.indeed.com/jobs?q=x" }), {}); +}); diff --git a/surfsense_web/lib/playground/url-hints.ts b/surfsense_web/lib/playground/url-hints.ts new file mode 100644 index 000000000..e45a5d8ea --- /dev/null +++ b/surfsense_web/lib/playground/url-hints.ts @@ -0,0 +1,90 @@ +/** + * Client-side, per-platform URL hints for the playground (UX only). + * + * The scraper API stays authoritative: it rejects malformed URLs (422), and + * each platform decides what it can actually scrape. These hints only *warn*, + * before a run, when a line in a platform's ``urls`` field is not a URL for + * that platform — so a typo (wrong site, missing scheme) is caught without a + * round-trip. They never block the run. + */ + +interface PlatformUrlRule { + /** Host suffixes (``youtube.com``) or ``.``-terminated prefixes (``amazon.``). */ + hosts: string[]; + /** Human platform name used in the warning message. */ + label: string; +} + +/** + * Keyed by platform slug. Instagram is intentionally absent: its ``urls`` field + * also accepts bare ``@handles``, so a non-URL line there is not a mistake. + */ +const PLATFORM_URL_RULES: Record = { + amazon: { hosts: ["amazon.", "amzn.to", "a.co"], label: "Amazon" }, + walmart: { hosts: ["walmart.com"], label: "Walmart" }, + reddit: { hosts: ["reddit.com", "redd.it"], label: "Reddit" }, + youtube: { hosts: ["youtube.com", "youtu.be"], label: "YouTube" }, + tiktok: { hosts: ["tiktok.com"], label: "TikTok" }, + google_maps: { hosts: ["google.", "goo.gl"], label: "Google Maps" }, + indeed: { hosts: ["indeed.com"], label: "Indeed" }, +}; + +/** The array fields that carry platform URLs at the capability layer. */ +const URL_FIELD_NAMES = new Set(["urls", "video_urls", "startUrls"]); + +function hostMatches(host: string, patterns: string[]): boolean { + return patterns.some((pattern) => + pattern.endsWith(".") + ? host.includes(pattern) + : host === pattern || host.endsWith(`.${pattern}`) + ); +} + +function article(word: string): "a" | "an" { + return /^[aeiou]/i.test(word) ? "an" : "a"; +} + +function isPlatformUrl(line: string, rule: PlatformUrlRule): boolean { + try { + const { protocol, hostname } = new URL(line); + if (protocol !== "http:" && protocol !== "https:") return false; + return hostMatches(hostname.toLowerCase(), rule.hosts); + } catch { + return false; + } +} + +/** + * Warn if any line in a platform ``urls`` field is not a URL for that platform. + * Returns ``undefined`` when the field/platform has no rule or every line is OK. + */ +export function urlFieldWarning( + platform: string, + fieldName: string, + value: unknown +): string | undefined { + const rule = PLATFORM_URL_RULES[platform]; + if (!rule || !URL_FIELD_NAMES.has(fieldName)) return undefined; + + const badLines = String(value ?? "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .filter((line) => !isPlatformUrl(line, rule)); + + if (badLines.length === 0) return undefined; + return `Not ${article(rule.label)} ${rule.label} URL: ${badLines.join(", ")}`; +} + +/** Per-field warnings for the current form values (empty when every URL looks valid). */ +export function urlFieldWarnings( + platform: string, + values: Record +): Record { + const warnings: Record = {}; + for (const [name, value] of Object.entries(values)) { + const warning = urlFieldWarning(platform, name, value); + if (warning) warnings[name] = warning; + } + return warnings; +}