apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-18 23:11:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-13 17:11:25 -07:00
parent e8b3692b54
commit 2b018c4474
111 changed files with 1800 additions and 1580 deletions
Download patch file Download diff file Expand all files Collapse all files

46
surfsense_evals/tests/suites/test_frames_wiki_fetch.py
View file

@ -63,14 +63,22 @@ class TestCacheFilename:
@pytest.mark.asyncio
@respx.mock
async def test_fetch_success_writes_markdown(tmp_path: Path) -> None:
respx.get(WIKI_API).mock(return_value=httpx.Response(
200,
json={"query": {"pages": [{
"pageid": 1,
"title": "James Buchanan",
"extract": "James Buchanan was the 15th president of the United States.",
}]}},
))
respx.get(WIKI_API).mock(
return_value=httpx.Response(
200,
json={
"query": {
"pages": [
{
"pageid": 1,
"title": "James Buchanan",
"extract": "James Buchanan was the 15th president of the United States.",
}
]
}
},
)
)
fetcher = WikiFetcher(cache_dir=tmp_path, rate_limit_rps=100) # disable throttle
article = await fetcher.fetch("https://en.wikipedia.org/wiki/James_Buchanan")
assert article is not None
@ -83,13 +91,21 @@ async def test_fetch_success_writes_markdown(tmp_path: Path) -> None:
@pytest.mark.asyncio
@respx.mock
async def test_fetch_missing_page_returns_none(tmp_path: Path) -> None:
respx.get(WIKI_API).mock(return_value=httpx.Response(
200,
json={"query": {"pages": [{
"title": "DoesNotExist",
"missing": True,
}]}},
))
respx.get(WIKI_API).mock(
return_value=httpx.Response(
200,
json={
"query": {
"pages": [
{
"title": "DoesNotExist",
"missing": True,
}
]
}
},
)
)
fetcher = WikiFetcher(cache_dir=tmp_path, rate_limit_rps=100)
article = await fetcher.fetch("https://en.wikipedia.org/wiki/DoesNotExist")
assert article is None