apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-08 22:22:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(native-connector): added google maps places & reviews scrapers

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-02 21:58:24 -07:00
parent cc99bc4cae
commit 7185079bd6
114 changed files with 4688 additions and 425 deletions
Download patch file Download diff file Expand all files Collapse all files

2
surfsense_backend/tests/integration/chat/test_append_message_recovery.py
View file

@ -46,9 +46,9 @@ from app.db import (
NewChatMessage,
NewChatMessageRole,
NewChatThread,
Workspace,
TokenUsage,
User,
Workspace,
)
from app.routes import new_chat_routes
from app.services.token_tracking_service import TurnTokenAccumulator

2
surfsense_backend/tests/integration/chat/test_message_id_sse.py
View file

@ -48,8 +48,8 @@ from app.db import (
NewChatMessage,
NewChatMessageRole,
NewChatThread,
Workspace,
User,
Workspace,
)
from app.services.new_streaming_service import VercelStreamingService
from app.tasks.chat import persistence as persistence_module

2
surfsense_backend/tests/integration/chat/test_persistence.py
View file

@ -38,9 +38,9 @@ from app.db import (
NewChatMessage,
NewChatMessageRole,
NewChatThread,
Workspace,
TokenUsage,
User,
Workspace,
)
from app.services.token_tracking_service import TurnTokenAccumulator
from app.tasks.chat import persistence as persistence_module

2
surfsense_backend/tests/integration/chat/test_thread_visibility.py
View file

@ -19,10 +19,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.context import AuthContext
from app.db import (
ChatVisibility,
User,
Workspace,
WorkspaceMembership,
WorkspaceRole,
User,
)
from app.routes import new_chat_routes
from app.schemas.new_chat import (

5
surfsense_backend/tests/integration/composio/test_oauth_callback.py
View file

@ -9,8 +9,8 @@ from app.config import config
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
Workspace,
User,
Workspace,
)
from app.utils.oauth_security import OAuthStateManager
@ -57,8 +57,7 @@ async def test_callback_with_error_param_redirects_to_denied_page(
assert response.status_code in {302, 303, 307}
location = response.headers["location"]
assert (
f"/dashboard/{db_workspace.id}/connectors/callback?"
"error=composio_oauth_denied"
f"/dashboard/{db_workspace.id}/connectors/callback?error=composio_oauth_denied"
) in location
connectors = await _drive_connectors(

4
surfsense_backend/tests/integration/document_upload/test_document_upload.py
View file

@ -320,9 +320,7 @@ class TestDocumentSearchability:
doc_ids = resp.json()["document_ids"]
cleanup_doc_ids.extend(doc_ids)
await poll_document_status(
client, headers, doc_ids, workspace_id=workspace_id
)
await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id)
search_resp = await client.get(
"/api/v1/documents/search",

4
surfsense_backend/tests/integration/document_upload/test_etl_credits.py
View file

@ -219,9 +219,7 @@ class TestDocumentProcessingNotification:
doc_ids = resp.json()["document_ids"]
cleanup_doc_ids.extend(doc_ids)
await poll_document_status(
client, headers, doc_ids, workspace_id=workspace_id
)
await poll_document_status(client, headers, doc_ids, workspace_id=workspace_id)
notifications = await get_notifications(
client,

6
surfsense_backend/tests/integration/google_unification/conftest.py
View file

@ -18,8 +18,8 @@ from app.db import (
DocumentType,
SearchSourceConnector,
SearchSourceConnectorType,
Workspace,
User,
Workspace,
)
EMBEDDING_DIM = app_config.embedding_model_instance.dimension
@ -275,9 +275,7 @@ async def seed_connector(
session.add(user)
await session.flush()
space = Workspace(
name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id
)
space = Workspace(name=f"{name_prefix} {uuid.uuid4().hex[:6]}", user_id=user.id)
session.add(space)
await session.flush()
space_id = space.id

4
surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py
View file

@ -119,9 +119,7 @@ async def test_reindex_updates_content(db_session, db_workspace, db_user, mocker
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_reindex_updates_content_hash(
db_session, db_workspace, db_user, mocker
):
async def test_reindex_updates_content_hash(db_session, db_workspace, db_user, mocker):
"""Content hash is recomputed after reindexing with new source markdown."""
adapter = UploadDocumentAdapter(db_session)
await adapter.index(

16
surfsense_backend/tests/integration/indexing_pipeline/test_index_editions.py
View file

@ -55,9 +55,7 @@ async def test_edit_keeps_unchanged_rows_and_embeds_only_the_new_text(
patched_embed_texts,
):
service = IndexingPipelineService(session=db_session)
doc_v1 = make_connector_document(
workspace_id=db_workspace.id, source_markdown=_V1
)
doc_v1 = make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1)
document = await _index(service, doc_v1)
ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
@ -100,9 +98,7 @@ async def test_head_insert_shifts_positions_without_new_rows_for_old_text(
service = IndexingPipelineService(session=db_session)
document = await _index(
service,
make_connector_document(
workspace_id=db_workspace.id, source_markdown=_V1
),
make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
)
ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
@ -136,9 +132,7 @@ async def test_removed_paragraph_is_deleted_and_order_compacts(
service = IndexingPipelineService(session=db_session)
document = await _index(
service,
make_connector_document(
workspace_id=db_workspace.id, source_markdown=_V1
),
make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
)
ids_v1 = {c.content: c.id for c in await _load_chunks(db_session, document.id)}
@ -171,9 +165,7 @@ async def test_kill_switch_falls_back_to_full_replace(
service = IndexingPipelineService(session=db_session)
document = await _index(
service,
make_connector_document(
workspace_id=db_workspace.id, source_markdown=_V1
),
make_connector_document(workspace_id=db_workspace.id, source_markdown=_V1),
)
ids_v1 = {c.id for c in await _load_chunks(db_session, document.id)}

2
surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py
View file

@ -14,8 +14,8 @@ from app.db import (
DocumentType,
DocumentVersion,
Folder,
Workspace,
User,
Workspace,
)
pytestmark = pytest.mark.integration

4
surfsense_backend/tests/integration/indexing_pipeline/test_migrate_legacy_docs.py
View file

@ -59,9 +59,7 @@ async def test_legacy_composio_gmail_doc_migrated_in_db(
assert reloaded.document_type == DocumentType.GOOGLE_GMAIL_CONNECTOR
async def test_no_legacy_doc_is_noop(
db_session, db_workspace, make_connector_document
):
async def test_no_legacy_doc_is_noop(db_session, db_workspace, make_connector_document):
"""When no legacy document exists, migrate_legacy_docs does nothing."""
connector_doc = make_connector_document(
document_type=DocumentType.GOOGLE_CALENDAR_CONNECTOR,

8
surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py
View file

@ -225,9 +225,7 @@ async def test_updated_at_advances_when_title_only_changes(
db_session, db_workspace, make_connector_document
):
"""updated_at advances even when only the title changes."""
original = make_connector_document(
workspace_id=db_workspace.id, title="Old Title"
)
original = make_connector_document(workspace_id=db_workspace.id, title="Old Title")
service = IndexingPipelineService(session=db_session)
first = await service.prepare_for_indexing([original])
@ -238,9 +236,7 @@ async def test_updated_at_advances_when_title_only_changes(
)
updated_at_v1 = result.scalars().first().updated_at
renamed = make_connector_document(
workspace_id=db_workspace.id, title="New Title"
)
renamed = make_connector_document(workspace_id=db_workspace.id, title="New Title")
await service.prepare_for_indexing([renamed])
result = await db_session.execute(

2
surfsense_backend/tests/integration/notifications/test_base_handler.py
View file

@ -10,7 +10,7 @@ import pytest
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Workspace, User
from app.db import User, Workspace
from app.notifications.persistence import Notification
from app.notifications.service import NotificationService

6
surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py
View file

@ -5,7 +5,7 @@ from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Workspace, User
from app.db import User, Workspace
from app.notifications.service import NotificationService
pytestmark = pytest.mark.integration
@ -47,9 +47,7 @@ async def test_comment_reply_truncates_long_preview(
db_session: AsyncSession, db_user: User, db_workspace: Workspace
):
"""A long comment preview is truncated in the reply message."""
notification = await _notify(
db_session, db_user, db_workspace, preview="y" * 150
)
notification = await _notify(db_session, db_user, db_workspace, preview="y" * 150)
assert notification.message == "y" * 100 + "..."

2
surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py
View file

@ -10,7 +10,7 @@ from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Workspace, User
from app.db import User, Workspace
from app.notifications.service import NotificationService
pytestmark = pytest.mark.integration

2
surfsense_backend/tests/integration/notifications/test_document_processing_handler.py
View file

@ -5,7 +5,7 @@ from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Workspace, User
from app.db import User, Workspace
from app.notifications.service import NotificationService
pytestmark = pytest.mark.integration

2
surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py
View file

@ -5,7 +5,7 @@ from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Workspace, User
from app.db import User, Workspace
from app.notifications.service import NotificationService
pytestmark = pytest.mark.integration

6
surfsense_backend/tests/integration/notifications/test_mention_handler.py
View file

@ -5,7 +5,7 @@ from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Workspace, User
from app.db import User, Workspace
from app.notifications.service import NotificationService
pytestmark = pytest.mark.integration
@ -47,9 +47,7 @@ async def test_new_mention_truncates_long_preview(
db_session: AsyncSession, db_user: User, db_workspace: Workspace
):
"""A long comment preview is truncated in the mention message."""
notification = await _notify(
db_session, db_user, db_workspace, preview="x" * 150
)
notification = await _notify(db_session, db_user, db_workspace, preview="x" * 150)
assert notification.message == "x" * 100 + "..."

2
surfsense_backend/tests/integration/podcasts/conftest.py
View file

@ -26,7 +26,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.app import app, limiter
from app.auth.context import AuthContext
from app.config import config as app_config
from app.db import Workspace, User, get_async_session
from app.db import User, Workspace, get_async_session
from app.podcasts.persistence import Podcast, PodcastStatus
from app.podcasts.schemas import (
DurationTarget,

4
surfsense_backend/tests/integration/podcasts/test_cancel.py
View file

@ -38,9 +38,7 @@ async def test_cancel_from_a_terminal_state_conflicts(
assert resp.status_code == 409
async def test_cancel_of_a_regeneration_is_rejected(
client, db_workspace, make_podcast
):
async def test_cancel_of_a_regeneration_is_rejected(client, db_workspace, make_podcast):
# Cancelling here would destroy a playable episode; reverting the
# regeneration is the way back.
podcast = await make_podcast(

2
surfsense_backend/tests/integration/retriever/conftest.py
View file

@ -9,7 +9,7 @@ import pytest_asyncio
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import config as app_config
from app.db import Chunk, Document, DocumentType, Workspace, User
from app.db import Chunk, Document, DocumentType, User, Workspace
EMBEDDING_DIM = app_config.embedding_model_instance.dimension
DUMMY_EMBEDDING = [0.1] * EMBEDDING_DIM

2
surfsense_backend/tests/integration/test_connector_index_authz.py
View file

@ -27,8 +27,8 @@ from app.auth.context import AuthContext
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
Workspace,
User,
Workspace,
)
from app.routes.search_source_connectors_routes import index_connector_content
from app.routes.workspaces_routes import create_default_roles_and_membership

2
surfsense_backend/tests/integration/test_document_versioning.py
View file

@ -7,7 +7,7 @@ import pytest_asyncio
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Document, DocumentType, DocumentVersion, Workspace, User
from app.db import Document, DocumentType, DocumentVersion, User, Workspace
pytestmark = pytest.mark.integration

2
surfsense_backend/tests/integration/test_obsidian_plugin_routes.py
View file

@ -32,8 +32,8 @@ from app.auth.context import AuthContext
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
Workspace,
User,
Workspace,
)
from app.routes.obsidian_plugin_routes import (
obsidian_connect,

2
surfsense_backend/tests/integration/test_pat_fail_closed_authz.py
View file

@ -7,7 +7,7 @@ from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.context import AuthContext
from app.db import PersonalAccessToken, Workspace, User
from app.db import PersonalAccessToken, User, Workspace
from app.users import allow_any_principal, require_session_context
from app.utils.rbac import check_workspace_access

2
surfsense_backend/tests/integration/test_zero_authz_context.py
View file

@ -7,7 +7,7 @@ from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.context import AuthContext
from app.db import PersonalAccessToken, Workspace, User
from app.db import PersonalAccessToken, User, Workspace
from app.routes.workspaces_routes import create_default_roles_and_membership
from app.utils.rbac import check_workspace_access, get_allowed_read_space_ids

4
surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py
View file

@ -158,9 +158,7 @@ async def test_build_dependencies_falls_back_to_workspace(
)
workspace = SimpleNamespace(chat_model_id=-7)
result = await build_dependencies(
session=_FakeSession(workspace), workspace_id=42
)
result = await build_dependencies(session=_FakeSession(workspace), workspace_id=42)
assert captured == {"config_id": -7}
assert result.llm.name == "llm"

4
surfsense_backend/tests/unit/connector_indexers/test_web_crawl_billing.py
View file

@ -142,7 +142,9 @@ async def test_charges_owner_for_successful_crawls(indexer_env):
session, user = _make_session(_OWNER_USER, balance_micros=100_000)
indexer_env["crawler"].crawl_url.side_effect = [_outcome(True), _outcome(True)]
total, warning = await _run(indexer_env, session, ["https://a.com", "https://b.com"])
total, warning = await _run(
indexer_env, session, ["https://a.com", "https://b.com"]
)
assert total == 2
assert warning is None

4
surfsense_backend/tests/unit/indexing_pipeline/test_create_placeholder_documents.py
View file

@ -36,9 +36,7 @@ def _make_placeholder(**overrides) -> PlaceholderInfo:
def _uid_hash(p: PlaceholderInfo) -> str:
return compute_identifier_hash(
p.document_type.value, p.unique_id, p.workspace_id
)
return compute_identifier_hash(p.document_type.value, p.unique_id, p.workspace_id)
def _session_with_existing_hashes(existing: set[str] | None = None):

32
surfsense_backend/tests/unit/proprietary/web_crawler/test_captcha.py
View file

@ -15,16 +15,16 @@ pytestmark = pytest.mark.unit
def _cfg(**overrides) -> CaptchaConfig:
base = dict(
enabled=True,
solving_site="capsolver",
api_key="key-123",
max_attempts_per_url=1,
timeout_s=30,
captcha_type_default="v2",
v3_min_score=0.7,
v3_action="verify",
)
base = {
"enabled": True,
"solving_site": "capsolver",
"api_key": "key-123",
"max_attempts_per_url": 1,
"timeout_s": 30,
"captcha_type_default": "v2",
"v3_min_score": 0.7,
"v3_action": "verify",
}
base.update(overrides)
return CaptchaConfig(**base)
@ -40,7 +40,9 @@ class _FakeEl:
class _FakePage:
"""Minimal sync-Playwright-ish page for detection/injection tests."""
def __init__(self, widgets=None, iframes=None, scripts=None, url="https://t.test/p"):
def __init__(
self, widgets=None, iframes=None, scripts=None, url="https://t.test/p"
):
self._widgets = widgets or {} # selector -> _FakeEl
self._iframes = iframes or [] # list[_FakeEl]
self._scripts = scripts or [] # list[_FakeEl]
@ -116,7 +118,9 @@ class TestDetect:
def test_v3_render_param_when_default_v3(self):
page = _FakePage(
scripts=[_FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"})]
scripts=[
_FakeEl({"src": "https://www.google.com/recaptcha/api.js?render=SK_V3"})
]
)
assert cap.detect_challenge(page, _cfg(captcha_type_default="v3")) == (
"v3",
@ -217,11 +221,11 @@ class TestPageAction:
assert state == {"attempts": 1, "solved": False}
def test_no_balance_latches_and_stops(self, monkeypatch):
class ErrNoBalance(Exception):
class NoBalanceError(Exception):
pass
def _boom(*_a, **_k):
raise ErrNoBalance("out of balance")
raise NoBalanceError("out of balance")
monkeypatch.setattr(cap, "_harvest_token", _boom)
state = {"attempts": 0, "solved": False}

14
surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py
View file

@ -10,8 +10,8 @@ import pytest
from app.proprietary.web_crawler import (
CrawlOutcomeStatus,
WebCrawlerConnector,
connector as connector_module,
)
from app.proprietary.web_crawler import connector as connector_module
from app.utils.crawl import BlockType
pytestmark = pytest.mark.unit
@ -230,9 +230,7 @@ async def test_captcha_attempts_surface_even_when_crawl_fails(
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
monkeypatch.setattr(
crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty
)
monkeypatch.setattr(crawler, "_crawl_with_stealthy", _stealthy_attempt_then_empty)
outcome = await crawler.crawl_url("https://example.com")
@ -341,9 +339,7 @@ async def test_static_4xx_is_classified(
monkeypatch.setattr(connector_module, "get_proxy_url", lambda: None)
block_state: dict = {"block_type": BlockType.UNKNOWN}
result = await crawler._crawl_with_async_fetcher(
"https://example.com", block_state
)
result = await crawler._crawl_with_async_fetcher("https://example.com", block_state)
assert result is None # 4xx => fall through to next tier
assert block_state["block_type"] is BlockType.CLOUDFLARE
@ -354,7 +350,9 @@ def test_build_result_ok_on_real_content() -> None:
crawler = WebCrawlerConnector()
block_state: dict = {"block_type": BlockType.UNKNOWN}
html = "<html><body><article>" + ("Real content. " * 40) + "</article></body></html>"
html = (
"<html><body><article>" + ("Real content. " * 40) + "</article></body></html>"
)
crawler._build_result(
html,
"https://example.com",

0
surfsense_backend/tests/unit/scrapers/google_maps/__init__.py Normal file
View file

175
surfsense_backend/tests/unit/scrapers/google_maps/test_parsers.py Normal file
View file

@ -0,0 +1,175 @@
"""Offline parser tests against a captured real place ``darray`` fixture.
The fixture is the ``jd[6]`` array from a live ``/maps/preview/place`` RPC
response for "Kim's Island" (the restaurant used in the Apify output example),
regenerated by ``scripts/e2e_google_maps_scraper.py``. These tests pin the
array-index paths so a Google structure shift is caught offline.
"""
import json
from pathlib import Path
import pytest
from app.proprietary.scrapers.google_maps.fetch import strip_xssi
from app.proprietary.scrapers.google_maps.parsers import (
brace_match_json,
dig,
parse_place,
)
_FIXTURE = Path(__file__).parent / "fixtures" / "place_darray.json"
@pytest.fixture
def darray() -> list:
return json.loads(_FIXTURE.read_text(encoding="utf-8"))
def test_parse_place_core_fields(darray):
place = parse_place(darray)
assert place["title"] == "Kim's Island"
assert place["placeId"] == "ChIJJQz5EZzKw4kRCZ95UajbyGw"
assert place["fid"] == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
assert place["categoryName"] == "Chinese restaurant"
assert "Chinese restaurant" in place["categories"]
assert place["totalScore"] == 4.5
assert place["phone"] == "(718) 356-5168"
assert place["website"] == "http://kimsislandsi.com/"
assert place["plusCode"].startswith("GQ62+8M")
def test_parse_place_address_components(darray):
place = parse_place(darray)
assert place["neighborhood"] == "Tottenville"
assert place["street"] == "175 Main St"
assert place["city"] == "Staten Island"
assert place["postalCode"] == "10307"
assert place["state"] == "New York"
assert place["countryCode"] == "US"
def test_parse_place_location_and_hours(darray):
place = parse_place(darray)
assert place["location"]["lat"] == pytest.approx(40.5107736)
assert place["location"]["lng"] == pytest.approx(-74.2482624)
hours = place["openingHours"]
assert hours and all("day" in h and "hours" in h for h in hours)
def test_parse_place_detail_extras(darray):
place = parse_place(darray)
assert place["kgmid"] == "/g/1tmgdcj8"
# cid = decimal of the fid's second hex half
assert place["cid"] == str(int("0x6cc8dba851799f09", 16))
info = place["additionalInfo"]
assert "Accessibility" in info
assert {"Wheelchair accessible entrance": True} in info["Accessibility"]
# every option is a single {name: bool} pair
for entries in info.values():
for entry in entries:
assert len(entry) == 1
assert all(isinstance(v, bool) for v in entry.values())
def test_parse_place_session_gated_fields(darray):
# These fields only appear when the RPC is sent with an NID session
# cookie + the full-page pb selector; the fixture pins that payload.
place = parse_place(darray)
dist = place["reviewsDistribution"]
assert set(dist) == {"oneStar", "twoStar", "threeStar", "fourStar", "fiveStar"}
assert place["reviewsCount"] == sum(dist.values())
assert place["imagesCount"] > 0
assert "All" in place["imageCategories"]
assert all(u.startswith("https://") for u in place["imageUrls"])
tags = place["reviewsTags"]
assert tags and all(
isinstance(t["title"], str) and isinstance(t["count"], int) for t in tags
)
assert len(place["additionalInfo"]) >= 5 # full sections, not just Accessibility
def test_parse_place_hotel_fields():
# Fixture: live darray for The Plaza (NYC), captured via the detail RPC.
fixture = Path(__file__).parent / "fixtures" / "hotel_darray.json"
place = parse_place(json.loads(fixture.read_text(encoding="utf-8")))
assert place["title"] == "The Plaza"
assert place["hotelStars"] == "5 stars"
assert place["checkInDate"] < place["checkOutDate"] # ISO dates compare
assert place["hotelDescription"]
similar = place["similarHotelsNearby"]
assert len(similar) >= 3
assert all(h["title"] and h["fid"] for h in similar)
assert any("hotel" in (h.get("description") or "").lower() for h in similar)
ads = place["hotelAds"]
assert ads and all(a["url"].startswith("https://") for a in ads)
assert any(a.get("price", "").startswith("$") for a in ads)
def test_hotel_fields_absent_for_non_hotels(darray):
# Kim's Island (restaurant) must not grow hotel fields.
place = parse_place(darray)
for key in ("hotelStars", "checkInDate", "similarHotelsNearby", "hotelAds"):
assert key not in place
def test_popular_times_shapes():
from app.proprietary.scrapers.google_maps.parsers import _popular_times
darray: list = [None] * 100
darray[84] = [
[[7, [[6, 0, "", "None", "6 AM", "No wait", "6a"], [7, 35]]]],
None,
None,
None,
None,
None,
"A little busy",
[22, 76],
]
out = _popular_times(darray)
assert out["popularTimesHistogram"] == {
"Su": [
{"hour": 6, "occupancyPercent": 0},
{"hour": 7, "occupancyPercent": 35},
]
}
assert out["popularTimesLiveText"] == "A little busy"
assert out["popularTimesLivePercent"] == 76
assert _popular_times([None] * 100) == {}
def test_parse_place_reservation_links():
fixture = Path(__file__).parent / "fixtures" / "search_response.json"
from app.proprietary.scrapers.google_maps.fetch import _search_darrays
jd = json.loads(fixture.read_text(encoding="utf-8"))
place = parse_place(_search_darrays(jd)[0])
# the first search hit in the fixture carries a reservation provider
links = place["tableReservationLinks"]
assert all(set(link) == {"url", "source"} for link in links)
assert place["reserveTableUrl"] == links[0]["url"]
def test_parse_place_omits_missing_fields(darray):
# parse_place drops keys whose path missed, rather than emitting nulls.
place = parse_place(darray)
assert all(v is not None for v in place.values())
def test_dig_tolerates_ragged_paths():
assert dig([1, [2, 3]], 1, 0) == 2
assert dig([1, [2, 3]], 5) is None
assert dig([1, None], 1, 0) is None
assert dig("not a list", 0) is None
def test_strip_xssi_and_brace_match_handle_html_wrapper():
# Mirrors the real proxy response: text/plain body wrapped in <p>…</p>.
raw = ')]}\'\n[1,[2,3],"x"]</p></body></html>'
body = strip_xssi(raw)
blob = brace_match_json(body, body.index("["))
assert json.loads(blob) == [1, [2, 3], "x"]
wrapped = '<html><body><p>)]}\'\n[["a"]]'
assert strip_xssi(wrapped).startswith("[[")

116
surfsense_backend/tests/unit/scrapers/google_maps/test_reviews.py Normal file
View file

@ -0,0 +1,116 @@
"""Offline tests for the Google Maps reviews parsing + flow helpers.
``fixtures/boq_reviews_page.json`` is a real ``GetLocalBoqProxy`` page (the
raw review list, ``jd[1][10][2]``) captured by scripts/e2e_google_maps_scraper.py
step 7. Regenerate it with that script if Google shifts the structure.
"""
import json
from datetime import datetime
from pathlib import Path
import pytest
from app.proprietary.scrapers.google_maps.fetch import build_reviews_url
from app.proprietary.scrapers.google_maps.parsers import (
parse_review,
parse_reviews_page,
strip_personal_data,
)
from app.proprietary.scrapers.google_maps.reviews import _before_cutoff, _keep
_FIXTURE = Path(__file__).parent / "fixtures" / "boq_reviews_page.json"
@pytest.fixture
def reviews_raw() -> list:
return json.loads(_FIXTURE.read_text(encoding="utf-8"))
def test_parse_reviews_page_full_page(reviews_raw):
parsed = parse_reviews_page(reviews_raw)
assert len(parsed) == len(reviews_raw) == 10
for review in parsed:
assert review["name"]
assert review["reviewId"]
assert 1 <= review["stars"] <= 5
assert review["publishedAtDate"].endswith("Z")
def test_parse_review_fields(reviews_raw):
review = parse_review(reviews_raw[0])
assert review["name"] == "Greg"
assert review["stars"] == 5
assert review["reviewerId"] == "108156147079988470963"
assert review["reviewerUrl"].startswith("https://www.google.com/maps/contrib/")
assert review["reviewerNumberOfReviews"] == 63
assert review["isLocalGuide"] is True
assert review["reviewOrigin"] == "Google"
assert review["originalLanguage"] == "en"
assert review["publishAt"] == "2 months ago"
# 1776469524140 ms epoch -> UTC ISO
assert review["publishedAtDate"] == "2026-04-17T23:45:24.140Z"
assert "spiced" in review["text"]
# Guided answers split into context vs per-aspect ratings.
assert review["reviewContext"]["Order type"] == "Take out"
assert review["reviewDetailedRating"]["Food"] == 5
def test_parse_review_owner_response_and_images(reviews_raw):
with_reply = [
r
for r in (parse_review(x) for x in reviews_raw)
if r and r.get("responseFromOwnerText")
]
assert with_reply, "fixture should contain at least one owner reply"
assert with_reply[0]["responseFromOwnerText"]
with_images = [
r
for r in (parse_review(x) for x in reviews_raw)
if r and r.get("reviewImageUrls")
]
assert with_images, "fixture should contain at least one review with photos"
assert with_images[0]["reviewImageUrls"][0].startswith("https://")
def test_parse_review_rejects_garbage():
assert parse_review(None) is None
assert parse_review([]) is None
assert parse_review([None, 5]) is None # no author block
def test_strip_personal_data(reviews_raw):
review = parse_review(reviews_raw[0])
strip_personal_data(review)
for key in ("name", "reviewerId", "reviewerUrl", "reviewerPhotoUrl"):
assert key not in review
assert review["reviewId"] # non-personal fields stay
assert review["stars"] == 5
def test_before_cutoff():
cutoff = datetime(2026, 3, 1)
assert _before_cutoff({"publishedAtDate": "2026-02-01T00:00:00.000Z"}, cutoff)
assert not _before_cutoff({"publishedAtDate": "2026-04-01T00:00:00.000Z"}, cutoff)
assert not _before_cutoff({}, cutoff) # undated reviews are kept
def test_keep_filters():
review = {"text": "Great NOODLES here", "reviewOrigin": "Google"}
assert _keep(review, filter_string="", origin="all")
assert _keep(review, filter_string="noodles", origin="google")
assert not _keep(review, filter_string="pizza", origin="all")
assert not _keep({"reviewOrigin": "TripAdvisor"}, filter_string="", origin="google")
def test_build_reviews_url_shape():
fid = "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
first = build_reviews_url(fid, sort_code=2)
assert "GetLocalBoqProxy" in first
assert "hl=en" in first
assert fid in json.dumps(first) or "0x89c3ca9c11f90c25" in first
# First page requests a page size; later pages carry the token instead.
paged = build_reviews_url(fid, sort_code=2, page_token="TOKEN123")
assert "TOKEN123" in paged
assert paged != first

150
surfsense_backend/tests/unit/scrapers/google_maps/test_search.py Normal file
View file

@ -0,0 +1,150 @@
"""Offline tests for the map-search flow: response extraction, URL building,
and the Apify result filters. The fixture is a real ``search?tbm=map`` response
captured by scripts/e2e_google_maps_scraper.py (step 9, query "pizza new york").
"""
import json
from pathlib import Path
import pytest
from app.proprietary.scrapers.google_maps.fetch import (
_search_darrays,
build_search_url,
)
from app.proprietary.scrapers.google_maps.parsers import parse_place
from app.proprietary.scrapers.google_maps.schemas import GoogleMapsScrapeInput
from app.proprietary.scrapers.google_maps.scraper import (
_custom_point,
_location_text,
_passes_filters,
)
_FIXTURE = Path(__file__).parent / "fixtures" / "search_response.json"
@pytest.fixture(scope="module")
def search_jd():
return json.loads(_FIXTURE.read_text(encoding="utf-8"))
def test_search_darrays_extracts_full_page(search_jd):
darrays = _search_darrays(search_jd)
assert len(darrays) == 20
parsed = [parse_place(d) for d in darrays]
for fields in parsed:
assert fields["title"]
assert fields["fid"].startswith("0x")
assert fields["placeId"].startswith("ChIJ")
assert "location" in fields
fids = [f["fid"] for f in parsed]
assert len(set(fids)) == 20 # no dupes within a page
def test_search_result_has_detail_fields(search_jd):
fields = parse_place(_search_darrays(search_jd)[0])
# Search entries carry the full darray, so detail fields come through.
assert fields["categories"]
assert fields["address"]
assert fields["totalScore"] > 0
assert fields["city"]
def test_search_darrays_rejects_garbage():
assert _search_darrays(None) == []
assert _search_darrays([]) == []
assert _search_darrays([[None, None]]) == []
def test_build_search_url_shape():
url = build_search_url("pizza new york", offset=20, language="de")
assert url.startswith("https://www.google.com/search?tbm=map")
assert "hl=de" in url
assert "q=pizza%20new%20york" in url
assert "!8i20" in url # offset
assert "!7i20" in url # page size
# whole-earth viewport when no coordinates given
assert "!1d25000000" in url
geo_url = build_search_url("museum", lat=48.85, lng=2.35, radius_m=5000)
assert "!2d2.35" in geo_url and "!3d48.85" in geo_url
assert "!1d10000" in geo_url # radius -> diameter
def test_location_text_prefers_location_query():
assert (
_location_text(GoogleMapsScrapeInput(locationQuery="Berlin, Germany"))
== "Berlin, Germany"
)
assert (
_location_text(
GoogleMapsScrapeInput(city="Austin", state="TX", countryCode="US")
)
== "Austin, TX, US"
)
assert _location_text(GoogleMapsScrapeInput()) is None
def test_custom_point():
lat, lng, radius = _custom_point(
GoogleMapsScrapeInput(
customGeolocation={
"type": "Point",
"coordinates": [2.35, 48.85],
"radiusKm": 5,
}
)
)
assert (lat, lng, radius) == (48.85, 2.35, 5000)
assert _custom_point(GoogleMapsScrapeInput()) == (None, None, None)
assert _custom_point(
GoogleMapsScrapeInput(customGeolocation={"type": "Polygon"})
) == (None, None, None)
def test_passes_filters():
fields = {
"title": "Joe's Pizza",
"categories": ["Pizza restaurant"],
"totalScore": 4.4,
"website": "https://joes.example",
}
default = GoogleMapsScrapeInput()
assert _passes_filters(fields, "pizza", default)
assert not _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_exact")
)
assert _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(searchMatching="only_includes")
)
assert not _passes_filters(
fields, "burger", GoogleMapsScrapeInput(searchMatching="only_includes")
)
assert _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["pizza"])
)
assert not _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(categoryFilterWords=["barber"])
)
assert not _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="fourAndHalf")
)
assert _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(placeMinimumStars="four")
)
assert _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(website="withWebsite")
)
assert not _passes_filters(
fields, "pizza", GoogleMapsScrapeInput(website="withoutWebsite")
)
closed = {**fields, "permanentlyClosed": True}
assert not _passes_filters(
closed, "pizza", GoogleMapsScrapeInput(skipClosedPlaces=True)
)
assert _passes_filters(closed, "pizza", default)

97
surfsense_backend/tests/unit/scrapers/google_maps/test_skeleton.py Normal file
View file

@ -0,0 +1,97 @@
"""Offline checks for the Google Maps scraper skeleton.
Covers the pure parts: URL classification and Apify-spec schema defaults/
serialization. The live flows (place / reviews / search) are covered by the
e2e scripts and the fixture-based parser tests.
"""
import pytest
from app.proprietary.scrapers.google_maps import (
GoogleMapsReviewsInput,
GoogleMapsScrapeInput,
PlaceItem,
ReviewItem,
)
from app.proprietary.scrapers.google_maps.url_resolver import extract_fid, resolve_url
@pytest.mark.parametrize(
("url", "kind", "value"),
[
(
"https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/data=!4m6",
"place",
"Kim's Island",
),
(
"https://www.google.com/maps/search/restaurants/@52.5190603,13.388574,13z/",
"search",
"restaurants",
),
("https://www.google.com/maps/reviews/data=!4m8!14m7", "reviews", None),
(
"https://www.google.com/maps?cid=7838756667406262025",
"cid",
"7838756667406262025",
),
("https://goo.gl/maps/abc123", "shortlink", None),
("https://maps.app.goo.gl/xyz", "shortlink", None),
],
)
def test_resolve_url(url, kind, value):
resolved = resolve_url(url)
assert resolved is not None
assert resolved.kind == kind
if value is not None:
assert resolved.value == value
def test_resolve_url_rejects_non_maps():
assert resolve_url("https://example.com/maps/place/foo") is None
assert resolve_url("https://www.google.com/search?q=pizza") is None
def test_extract_fid():
url = (
"https://www.google.com/maps/place/Kim's+Island/@40.51,-74.24,17z/"
"data=!4m6!3m5!1s0x89c3ca9c11f90c25:0x6cc8dba851799f09!8m2"
)
assert extract_fid(url) == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
assert extract_fid("https://www.google.com/maps/place/Foo") is None
assert resolve_url(url).fid == "0x89c3ca9c11f90c25:0x6cc8dba851799f09"
def test_scrape_input_defaults_match_apify_spec():
inp = GoogleMapsScrapeInput()
assert inp.language == "en"
assert inp.maxCrawledPlacesPerSearch is None # empty = all places
assert inp.searchMatching == "all"
assert inp.website == "allPlaces"
assert inp.reviewsSort == "newest"
assert inp.reviewsOrigin == "all"
assert inp.scrapeReviewsPersonalData is True
assert inp.maxCompetitorsToAnalyze == 30
assert inp.scrapeSocialMediaProfiles.facebooks is False
# Unknown fields are accepted (extra="allow") so parity is additive.
GoogleMapsScrapeInput(someFutureField=1)
def test_reviews_input_defaults_match_apify_spec():
inp = GoogleMapsReviewsInput()
assert inp.maxReviews == 10_000_000
assert inp.reviewsSort == "newest"
assert inp.personalData is True
def test_output_items_serialize_full_shape():
place = PlaceItem(title="Kim's Island", placeId="ChIJx").to_output()
assert place["title"] == "Kim's Island"
assert place["permanentlyClosed"] is False
assert place["categories"] == []
assert "reviewsDistribution" in place # unsourced fields still emitted
review = ReviewItem(reviewId="abc", stars=5).to_output()
assert review["stars"] == 5
assert review["reviewImageUrls"] == []
assert "responseFromOwnerText" in review

12
surfsense_backend/tests/unit/scrapers/youtube/test_fetch_resilience.py
View file

@ -9,8 +9,6 @@ from __future__ import annotations
from collections.abc import AsyncIterator
import pytest
from app.proprietary.scrapers.youtube import innertube, scraper
from app.proprietary.scrapers.youtube.innertube import (
INNERTUBE_SEARCH_URL,
@ -40,7 +38,7 @@ class _FakeSession:
self.exc = exc
self.calls = 0
async def post(self, url, json=None): # noqa: A002 - mirror scrapling API
async def post(self, url, json=None):
self.calls += 1
if self.exc:
raise ConnectionError("boom")
@ -95,7 +93,9 @@ async def test_post_rotates_on_connection_error_then_succeeds():
async def test_post_gives_up_after_max_rotations():
# Every IP is blocked -> rotate up to the cap, then return None.
holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)])
holder = _FakeHolder(
[_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)]
)
token = _current_session.set(holder)
try:
result = await post_innertube(INNERTUBE_SEARCH_URL, _payload())
@ -136,7 +136,9 @@ async def test_fetch_html_falls_back_to_stealthy_when_all_blocked(monkeypatch):
return "<html>stealthy</html>"
monkeypatch.setattr(innertube, "_fetch_html_stealthy", _fake_stealthy)
holder = _FakeHolder([_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)])
holder = _FakeHolder(
[_FakeSession(429) for _ in range(innertube._MAX_ROTATIONS + 1)]
)
token = _current_session.set(holder)
try:
html = await fetch_html("https://www.youtube.com/watch?v=zzz")

194
surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py
View file

@ -21,17 +21,17 @@ from app.proprietary.scrapers.youtube.parsers import (
dig,
find_all,
parse_channel_about,
parse_collaborators,
parse_comment_entities,
parse_description_links,
parse_location,
parse_translation,
parse_channel_metadata,
parse_channel_shorts,
parse_playlist_video_ids,
parse_collaborators,
parse_comment_entities,
parse_count,
parse_date,
parse_description_links,
parse_location,
parse_playlist_video_ids,
parse_search_response,
parse_translation,
parse_video_page,
seconds_to_duration,
)
@ -105,12 +105,19 @@ def _video_html() -> str:
"keywords": ["#tech"],
"thumbnail": {
"thumbnails": [
{"url": "https://i.ytimg.com/vi/abc123/hq.jpg", "width": 480, "height": 360}
{
"url": "https://i.ytimg.com/vi/abc123/hq.jpg",
"width": 480,
"height": 360,
}
]
},
},
"microformat": {
"playerMicroformatRenderer": {"publishDate": "2024-08-27", "isFamilySafe": True}
"playerMicroformatRenderer": {
"publishDate": "2024-08-27",
"isFamilySafe": True,
}
},
"adPlacements": [{"adPlacementRenderer": {}}],
}
@ -200,7 +207,9 @@ def test_parse_video_page_members_only_and_paid():
player = {
"videoDetails": {"videoId": "m", "title": "t"},
"playabilityStatus": {"errorScreen": {"x": {"offerId": "sponsors_only_video"}}},
"paidContentOverlay": {"paidContentOverlayRenderer": {"text": "Includes paid promotion"}},
"paidContentOverlay": {
"paidContentOverlayRenderer": {"text": "Includes paid promotion"}
},
}
initial = {
"badges": [
@ -237,7 +246,11 @@ def test_parse_channel_metadata():
"externalId": "UCabc123",
"description": "We scrape the web.",
"vanityChannelUrl": "https://www.youtube.com/@Apify",
"avatar": {"thumbnails": [{"url": "https://a/avatar.jpg", "width": 88, "height": 88}]},
"avatar": {
"thumbnails": [
{"url": "https://a/avatar.jpg", "width": 88, "height": 88}
]
},
}
},
"header": {
@ -246,7 +259,11 @@ def test_parse_channel_metadata():
"imageBannerViewModel": {
"image": {
"sources": [
{"url": "https://a/banner.jpg", "width": 1060, "height": 175}
{
"url": "https://a/banner.jpg",
"width": 1060,
"height": 175,
}
]
}
}
@ -290,7 +307,9 @@ def test_parse_description_links():
{
"startIndex": 4,
"length": 4,
"onTap": {"innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}}},
"onTap": {
"innertubeCommand": {"urlEndpoint": {"url": "/hashtag/tag"}}
},
},
{
"startIndex": 21,
@ -318,8 +337,16 @@ def test_parse_description_links():
def test_channel_about_tokens():
initial = {
"a": {"showEngagementPanelEndpoint": {"x": {"continuationCommand": {"token": "T1"}}}},
"b": {"showEngagementPanelEndpoint": {"y": {"continuationCommand": {"token": "T2"}}}},
"a": {
"showEngagementPanelEndpoint": {
"x": {"continuationCommand": {"token": "T1"}}
}
},
"b": {
"showEngagementPanelEndpoint": {
"y": {"continuationCommand": {"token": "T2"}}
}
},
}
assert channel_about_tokens(initial) == ["T1", "T2"]
assert channel_about_tokens({}) == []
@ -340,7 +367,11 @@ def test_parse_search_response():
"shortViewCountText": {"simpleText": "451K views"},
"thumbnail": {
"thumbnails": [
{"url": "https://i.ytimg.com/vi/x/hq.jpg", "width": 360, "height": 202}
{
"url": "https://i.ytimg.com/vi/x/hq.jpg",
"width": 360,
"height": 202,
}
]
},
"ownerText": {
@ -395,7 +426,11 @@ def test_parse_channel_shorts():
"thumbnailViewModel": {
"image": {
"sources": [
{"url": "https://i.ytimg.com/s.jpg", "width": 405, "height": 720}
{
"url": "https://i.ytimg.com/s.jpg",
"width": 405,
"height": 720,
}
]
}
},
@ -422,13 +457,30 @@ def test_parse_playlist_video_ids():
data = {
"contents": {
"items": [
{"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}},
{"lockupViewModel": {"contentId": "k7RM-ot2NWY", "contentType": "VIDEO"}},
{"lockupViewModel": {"contentId": "fNk_zzaMoSs", "contentType": "VIDEO"}}, # dup
{
"lockupViewModel": {
"contentId": "fNk_zzaMoSs",
"contentType": "VIDEO",
}
},
{
"lockupViewModel": {
"contentId": "k7RM-ot2NWY",
"contentType": "VIDEO",
}
},
{
"lockupViewModel": {
"contentId": "fNk_zzaMoSs",
"contentType": "VIDEO",
}
}, # dup
]
},
# a playlist self-lockup (non-video, longer id) that must be ignored
"sidebar": {"lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}},
"sidebar": {
"lockupViewModel": {"contentId": "PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab"}
},
"continuationItemRenderer": {
"continuationEndpoint": {"continuationCommand": {"token": "PAGE2"}}
},
@ -494,7 +546,11 @@ def _primary_info(super_title_link):
"results": {
"results": {
"contents": [
{"videoPrimaryInfoRenderer": {"superTitleLink": super_title_link}}
{
"videoPrimaryInfoRenderer": {
"superTitleLink": super_title_link
}
}
]
}
}
@ -536,7 +592,9 @@ def _owner(video_owner_renderer):
"contents": [
{
"videoSecondaryInfoRenderer": {
"owner": {"videoOwnerRenderer": video_owner_renderer}
"owner": {
"videoOwnerRenderer": video_owner_renderer
}
}
}
]
@ -556,14 +614,19 @@ def _collab_row(name, base_url):
{
"onTap": {
"innertubeCommand": {
"browseEndpoint": {"browseId": "UCx", "canonicalBaseUrl": base_url}
"browseEndpoint": {
"browseId": "UCx",
"canonicalBaseUrl": base_url,
}
}
}
}
],
},
# A nested subscribe submenu — must NOT be picked up as a collaborator.
"subscribeMenu": {"listItemViewModel": {"title": {"content": "Unsubscribe"}}},
"subscribeMenu": {
"listItemViewModel": {"title": {"content": "Unsubscribe"}}
},
}
}
@ -583,15 +646,20 @@ def test_parse_collaborators_from_dialog():
"dialogViewModel": {
"header": {
"dialogHeaderViewModel": {
"headline": {"content": "Collaborators"}
"headline": {
"content": "Collaborators"
}
}
},
"customContent": {
"listViewModel": {
"listItems": [
_collab_row("Alice", "/@alice"),
_collab_row(
"Bob", "/channel/UCbob123"
"Alice", "/@alice"
),
_collab_row(
"Bob",
"/channel/UCbob123",
),
]
}
@ -610,7 +678,11 @@ def test_parse_collaborators_from_dialog():
collaborators = parse_collaborators(initial)
assert collaborators == [
{"name": "Alice", "username": "alice", "url": "https://www.youtube.com/@alice"},
{"name": "Bob", "username": None, "url": "https://www.youtube.com/channel/UCbob123"},
{
"name": "Bob",
"username": None,
"url": "https://www.youtube.com/channel/UCbob123",
},
]
@ -627,8 +699,23 @@ def test_parse_translation_from_next():
"results": {
"results": {
"contents": [
{"videoPrimaryInfoRenderer": {"title": {"runs": [{"text": "Título "}, {"text": "traducido"}]}}},
{"videoSecondaryInfoRenderer": {"attributedDescription": {"content": "Descripción traducida"}}},
{
"videoPrimaryInfoRenderer": {
"title": {
"runs": [
{"text": "Título "},
{"text": "traducido"},
]
}
}
},
{
"videoSecondaryInfoRenderer": {
"attributedDescription": {
"content": "Descripción traducida"
}
}
},
]
}
}
@ -642,7 +729,9 @@ def test_parse_translation_from_next():
def test_parse_translation_description_runs_fallback():
next_data = {
"videoSecondaryInfoRenderer": {"description": {"runs": [{"text": "old "}, {"text": "style"}]}}
"videoSecondaryInfoRenderer": {
"description": {"runs": [{"text": "old "}, {"text": "style"}]}
}
}
title, description = parse_translation(next_data)
assert title is None
@ -671,6 +760,7 @@ def _comment_cep(cid, *, level=0, hearted=False, owner=False, replies="5"):
def _next_comments_response():
"""A /next comments response: sort menu, two threads, a page token."""
def _reply_loader(token):
return {
"continuationItemRenderer": {
@ -692,7 +782,13 @@ def _next_comments_response():
"frameworkUpdates": {
"entityBatchUpdate": {
"mutations": [
{"payload": {"commentEntityPayload": _comment_cep("C1", hearted=True, owner=True)}},
{
"payload": {
"commentEntityPayload": _comment_cep(
"C1", hearted=True, owner=True
)
}
},
{"payload": {"commentEntityPayload": _comment_cep("C2")}},
]
}
@ -717,8 +813,14 @@ def _next_comments_response():
"header": {
"sortFilterSubMenuRenderer": {
"subMenuItems": [
{"title": "Top", "serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}}},
{"title": "Newest", "serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}}},
{
"title": "Top",
"serviceEndpoint": {"continuationCommand": {"token": "TOPTOK"}},
},
{
"title": "Newest",
"serviceEndpoint": {"continuationCommand": {"token": "NEWTOK"}},
},
]
}
},
@ -749,7 +851,13 @@ def test_comment_entity_reply_level_and_empty_reply_count():
"frameworkUpdates": {
"entityBatchUpdate": {
"mutations": [
{"payload": {"commentEntityPayload": _comment_cep("R1", level=1, replies="")}}
{
"payload": {
"commentEntityPayload": _comment_cep(
"R1", level=1, replies=""
)
}
}
]
}
}
@ -789,7 +897,9 @@ def test_comment_section_token_from_watch_page():
{
"continuationItemRenderer": {
"continuationEndpoint": {
"continuationCommand": {"token": "SECTIONTOK"}
"continuationCommand": {
"token": "SECTIONTOK"
}
}
}
}
@ -815,9 +925,17 @@ def test_comment_section_token_from_watch_page():
("https://youtu.be/dQw4w9WgXcQ", "video", "dQw4w9WgXcQ"),
("https://www.youtube.com/shorts/abc123", "video", "abc123"),
("https://www.youtube.com/@Apify", "channel", "Apify"),
("https://www.youtube.com/channel/UC123456789abc/videos", "channel", "UC123456789abc"),
(
"https://www.youtube.com/channel/UC123456789abc/videos",
"channel",
"UC123456789abc",
),
("https://www.youtube.com/playlist?list=PL123", "playlist", "PL123"),
("https://www.youtube.com/results?search_query=web+scraping", "search", "web scraping"),
(
"https://www.youtube.com/results?search_query=web+scraping",
"search",
"web scraping",
),
("https://www.youtube.com/hashtag/tech", "hashtag", "tech"),
],
)

4
surfsense_backend/tests/unit/tasks/chat/streaming/test_llm_bundle.py
View file

@ -32,9 +32,7 @@ def _patch_common_bundle_dependencies(monkeypatch: pytest.MonkeyPatch):
_CapturedChatLiteLLM.calls = []
async def _fake_workspace(
_session: Any, _workspace_id: int
) -> SimpleNamespace:
async def _fake_workspace(_session: Any, _workspace_id: int) -> SimpleNamespace:
return SimpleNamespace(id=42, user_id="user-1")
monkeypatch.setattr(llm_bundle, "_load_workspace", _fake_workspace)

4
surfsense_backend/tests/unit/test_pat_fail_closed_static.py
View file

@ -16,9 +16,7 @@ ALLOW_ANY_EXPECTED = {
"routes/obsidian_plugin_routes.py": (
"_auth: AuthContext = Depends(allow_any_principal)"
),
"routes/workspaces_routes.py": (
"auth: AuthContext = Depends(allow_any_principal)"
),
"routes/workspaces_routes.py": ("auth: AuthContext = Depends(allow_any_principal)"),
}
CONNECTOR_LISTERS = [

6
surfsense_backend/tests/unit/utils/test_validators.py
View file

@ -11,10 +11,10 @@ from app.utils.validators import (
validate_messages,
validate_research_mode,
validate_search_mode,
validate_workspace_id,
validate_top_k,
validate_url,
validate_uuid,
validate_workspace_id,
)
pytestmark = pytest.mark.unit
@ -334,6 +334,4 @@ def test_validate_connector_config_invalid():
# WEBCRAWLER_CONNECTOR custom validation: malformed INITIAL_URLS rejected.
with pytest.raises(ValueError):
validate_connector_config(
"WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"}
)
validate_connector_config("WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"})