apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-08 22:22:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: bumped version to 0.0.31

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-06 21:43:15 -07:00
parent 8df8565e0a
commit 1c9ab207ef
56 changed files with 520 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

1
surfsense_backend/scripts/check_migration_flow.py
View file

@ -94,6 +94,7 @@ async def set_version(version: str | None) -> None:
async def assert_at_head() -> None:
import asyncpg
from alembic.script import ScriptDirectory
head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head()

4
surfsense_backend/scripts/check_zero_publication_bootstrap.py
View file

@ -40,7 +40,9 @@ async def main() -> None:
await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
await admin.close()
print("OK: ensure_publication creates and verifies on a create_all DB, idempotently.")
print(
"OK: ensure_publication creates and verifies on a create_all DB, idempotently."
)
asyncio.run(main())

121
surfsense_backend/scripts/e2e_google_search.py
View file

@ -28,7 +28,9 @@ sys.path.insert(0, str(_ROOT))
load_dotenv(_ROOT / ".env")
logging.basicConfig(level=logging.WARNING)
logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(logging.INFO)
logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(
logging.INFO
)
from app.proprietary.platforms.google_search import ( # noqa: E402
GoogleSearchScrapeInput,
@ -41,15 +43,19 @@ async def run_ai_mode(label: str, *, queries: str) -> None:
print(f"\n=== {label} ===")
t0 = time.perf_counter()
inp = GoogleSearchScrapeInput(
queries=queries, countryCode="us", languageCode="en",
queries=queries,
countryCode="us",
languageCode="en",
aiModeSearch={"enableAiMode": True},
)
items = await scrape_serps(inp, limit=2)
ai_items = [i for i in items if i["aiModeResult"]]
assert ai_items, f"{label}: no aiModeResult item emitted"
res = ai_items[0]["aiModeResult"]
print(f" text={len(res['text'])} chars, sources={len(res['sources'])} "
f"({time.perf_counter()-t0:.0f}s)")
print(
f" text={len(res['text'])} chars, sources={len(res['sources'])} "
f"({time.perf_counter() - t0:.0f}s)"
)
print(f" {res['text'][:130]!r}")
for s in res["sources"][:3]:
print(f" src: {(s['title'] or '')[:60]!r}")
@ -59,9 +65,16 @@ async def run_ai_mode(label: str, *, queries: str) -> None:
async def run(
label: str, *, expect_ads=False, expect_products=False, expect_paa_answers=False,
expect_sitelinks=False, expect_aio=False, expect_device=None,
expect_icons=False, **kwargs
label: str,
*,
expect_ads=False,
expect_products=False,
expect_paa_answers=False,
expect_sitelinks=False,
expect_aio=False,
expect_device=None,
expect_icons=False,
**kwargs,
) -> None:
print(f"\n=== {label} ===")
t0 = time.perf_counter()
@ -72,18 +85,24 @@ async def run(
paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]]
sitelinked = [o for o in it["organicResults"] if o["siteLinks"]]
print(f" term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}")
print(f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} "
f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} "
f"suggested={len(it['suggestedResults'])} "
f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) "
f"({time.perf_counter()-t0:.0f}s)")
print(
f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} "
f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} "
f"suggested={len(it['suggestedResults'])} "
f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) "
f"({time.perf_counter() - t0:.0f}s)"
)
for o in sitelinked[:2]:
print(f" [sitelinks on #{o['position']}] "
+ ", ".join(s["title"] for s in o["siteLinks"][:5]))
print(
f" [sitelinks on #{o['position']}] "
+ ", ".join(s["title"] for s in o["siteLinks"][:5])
)
aio = it["aiOverview"]
if aio:
print(f" [aiOverview] content={len(aio['content'])} chars, "
f"sources={len(aio['sources'])}")
print(
f" [aiOverview] content={len(aio['content'])} chars, "
f"sources={len(aio['sources'])}"
)
print(f" {aio['content'][:110]!r}")
for s in aio["sources"][:3]:
print(f" src: {(s['title'] or '')[:55]!r}")
@ -113,10 +132,15 @@ async def run(
f"{label}: device={it['searchQuery']['device']}"
)
if expect_icons:
iconed = [o for o in it["organicResults"]
if (o["icon"] or "").startswith("data:image")]
print(f" [icons] {len(iconed)}/{len(it['organicResults'])} organic "
f"carry a base64 favicon")
iconed = [
o
for o in it["organicResults"]
if (o["icon"] or "").startswith("data:image")
]
print(
f" [icons] {len(iconed)}/{len(it['organicResults'])} organic "
f"carry a base64 favicon"
)
assert iconed, f"{label}: expected base64 icons on organic results"
@ -124,24 +148,47 @@ _CASES = {
"plain": lambda: run("plain query", queries="python asyncio tutorial"),
"site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"),
"ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True),
"products": lambda: run("product ads", queries="buy running shoes", expect_products=True),
"focus": lambda: run("focusOnPaidAds (commercial)", queries="car insurance quotes",
focusOnPaidAds=True, expect_ads=True),
"focus-neg": lambda: run("focusOnPaidAds (non-commercial, retries capped)",
queries="python asyncio tutorial", focusOnPaidAds=True),
"paa": lambda: run("people also ask", queries="what is seo", expect_paa_answers=True),
"sitelinks": lambda: run("sitelinks + suggested (brand query)", queries="amazon",
expect_sitelinks=True),
"products": lambda: run(
"product ads", queries="buy running shoes", expect_products=True
),
"focus": lambda: run(
"focusOnPaidAds (commercial)",
queries="car insurance quotes",
focusOnPaidAds=True,
expect_ads=True,
),
"focus-neg": lambda: run(
"focusOnPaidAds (non-commercial, retries capped)",
queries="python asyncio tutorial",
focusOnPaidAds=True,
),
"paa": lambda: run(
"people also ask", queries="what is seo", expect_paa_answers=True
),
"sitelinks": lambda: run(
"sitelinks + suggested (brand query)", queries="amazon", expect_sitelinks=True
),
"aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True),
"mobile": lambda: run("mobile layout (mobileResults)", queries="best seo tools",
mobileResults=True, expect_device="MOBILE"),
"unfiltered": lambda: run("includeUnfilteredResults (filter=0)",
queries="python asyncio tutorial",
includeUnfilteredResults=True),
"icons": lambda: run("includeIcons (base64 favicons)", queries="github",
includeIcons=True, expect_icons=True),
"aimode": lambda: run_ai_mode("Google AI Mode (udm=50)",
queries="what is quantum computing"),
"mobile": lambda: run(
"mobile layout (mobileResults)",
queries="best seo tools",
mobileResults=True,
expect_device="MOBILE",
),
"unfiltered": lambda: run(
"includeUnfilteredResults (filter=0)",
queries="python asyncio tutorial",
includeUnfilteredResults=True,
),
"icons": lambda: run(
"includeIcons (base64 favicons)",
queries="github",
includeIcons=True,
expect_icons=True,
),
"aimode": lambda: run_ai_mode(
"Google AI Mode (udm=50)", queries="what is quantum computing"
),
}

4
surfsense_backend/scripts/e2e_reddit_scraper.py
View file

@ -182,9 +182,7 @@ async def step5_dump_fixtures() -> bool:
wrote.append("sample_post.json")
# A single comment thing, for the comment-mapping fixture.
comment_kids = children(post[1]) if len(post) > 1 else []
first_comment = next(
(c for c in comment_kids if c.get("kind") == "t1"), None
)
first_comment = next((c for c in comment_kids if c.get("kind") == "t1"), None)
if first_comment:
(_FIXTURE_DIR / "sample_comment.json").write_text(
json.dumps(first_comment), encoding="utf-8"