mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
106 lines
4.5 KiB
Python
106 lines
4.5 KiB
Python
"""One-shot smoke test: hit every scraper API endpoint with a PAT, print PASS/FAIL."""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import httpx
|
|
|
|
BASE = "http://localhost:8000/api/v1/workspaces/12/scrapers"
|
|
PAT = os.environ["SURFSENSE_PAT"] # export a ss_pat_... key before running
|
|
HEADERS = {"Authorization": f"Bearer {PAT}", "Content-Type": "application/json"}
|
|
|
|
# Minimal payloads: 1-3 items each to keep credit spend tiny.
|
|
VERBS = [
|
|
("google_search/scrape", {"queries": ["surfsense github"], "max_pages_per_query": 1}),
|
|
("web/crawl", {"startUrls": ["https://example.com"], "maxCrawlDepth": 0}),
|
|
("reddit/scrape", {"urls": ["https://www.reddit.com/r/Python/"], "max_items": 3}),
|
|
("youtube/scrape", {"search_queries": ["python tutorial"], "max_results": 1}),
|
|
(
|
|
"youtube/comments",
|
|
{"urls": ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"], "max_comments": 3},
|
|
),
|
|
(
|
|
"google_maps/scrape",
|
|
{"search_queries": ["coffee shop"], "location": "New York, USA", "max_places": 1},
|
|
),
|
|
(
|
|
"google_maps/reviews",
|
|
# Google Sydney office, a stable well-known place id.
|
|
{"place_ids": ["ChIJN1t_tDeuEmsRUsoyG83frY4"], "max_reviews": 3},
|
|
),
|
|
]
|
|
|
|
results = []
|
|
client = httpx.Client(headers=HEADERS, timeout=300)
|
|
sync_run_id = None
|
|
|
|
for verb, payload in VERBS:
|
|
t0 = time.time()
|
|
try:
|
|
r = client.post(f"{BASE}/{verb}", json=payload)
|
|
dur = time.time() - t0
|
|
run_id = r.headers.get("x-run-id")
|
|
if r.status_code == 200:
|
|
body = r.json()
|
|
# Count items in whatever list field the output has.
|
|
items = next((len(v) for v in body.values() if isinstance(v, list)), "?")
|
|
results.append((verb, "PASS", f"{r.status_code} items={items} run={run_id} {dur:.1f}s"))
|
|
if sync_run_id is None and run_id:
|
|
sync_run_id = run_id
|
|
else:
|
|
results.append((verb, "FAIL", f"{r.status_code} {r.text[:200]} {dur:.1f}s"))
|
|
except Exception as e:
|
|
results.append((verb, "FAIL", f"{type(e).__name__}: {e}"))
|
|
print(f"[{results[-1][1]}] {verb}: {results[-1][2]}", flush=True)
|
|
|
|
# --- Run history endpoints ---
|
|
r = client.get(f"{BASE}/runs", params={"limit": 5})
|
|
ok = r.status_code == 200 and isinstance(r.json(), list)
|
|
results.append(("GET runs (list)", "PASS" if ok else "FAIL", f"{r.status_code} rows={len(r.json()) if ok else '?'}"))
|
|
print(f"[{results[-1][1]}] GET runs: {results[-1][2]}", flush=True)
|
|
|
|
if sync_run_id:
|
|
r = client.get(f"{BASE}/runs/{sync_run_id}")
|
|
ok = r.status_code == 200 and r.json().get("id") == sync_run_id
|
|
results.append(("GET runs/{id} (detail)", "PASS" if ok else "FAIL", f"{r.status_code} status={r.json().get('status') if ok else r.text[:100]}"))
|
|
print(f"[{results[-1][1]}] GET run detail: {results[-1][2]}", flush=True)
|
|
|
|
# --- Async mode + SSE events ---
|
|
r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"]})
|
|
if r.status_code == 202:
|
|
async_id = r.json()["run_id"]
|
|
seen, finished = [], None
|
|
with client.stream("GET", f"{BASE}/runs/{async_id}/events", timeout=120) as s:
|
|
for line in s.iter_lines():
|
|
if line.startswith("data: "):
|
|
ev = json.loads(line[6:])
|
|
seen.append(ev["type"])
|
|
if ev["type"] == "run.finished":
|
|
finished = ev.get("status")
|
|
break
|
|
ok = finished == "success"
|
|
results.append(("async + SSE events", "PASS" if ok else "FAIL", f"202 events={seen} final={finished}"))
|
|
else:
|
|
results.append(("async + SSE events", "FAIL", f"{r.status_code} {r.text[:200]}"))
|
|
print(f"[{results[-1][1]}] async+SSE: {results[-1][2]}", flush=True)
|
|
|
|
# --- Cancel endpoint ---
|
|
r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"], "maxCrawlDepth": 2, "maxCrawlPages": 50})
|
|
if r.status_code == 202:
|
|
cancel_id = r.json()["run_id"]
|
|
time.sleep(1)
|
|
r2 = client.post(f"{BASE}/runs/{cancel_id}/cancel")
|
|
ok = r2.status_code == 200 and r2.json().get("status") == "cancelled"
|
|
results.append(("POST runs/{id}/cancel", "PASS" if ok else "FAIL", f"{r2.status_code} {r2.text[:150]}"))
|
|
else:
|
|
results.append(("POST runs/{id}/cancel", "FAIL", f"setup {r.status_code}"))
|
|
print(f"[{results[-1][1]}] cancel: {results[-1][2]}", flush=True)
|
|
|
|
print("\n===== SUMMARY =====")
|
|
for name, status_, detail in results:
|
|
print(f"{status_:4} {name}: {detail}")
|
|
failed = [r for r in results if r[1] == "FAIL"]
|
|
print(f"\n{len(results) - len(failed)}/{len(results)} passed")
|
|
sys.exit(1 if failed else 0)
|