refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

2026-07-16 23:01:06 +02:00 · 2026-07-13 17:11:25 -07:00 · 2026-07-13 17:11:25 -07:00 · 2b018c4474
commit 2b018c4474
parent e8b3692b54
111 changed files with 1800 additions and 1580 deletions
--- a/surfsense_mcp/mcp_server/features/knowledge_base/init.py
+++ b/surfsense_mcp/mcp_server/features/knowledge_base/init.py
@ -14,9 +14,7 @@ from ...core.workspace_context import WorkspaceContext
 from . import document_tools, search_tools


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register every knowledge-base tool on the server."""
    search_tools.register(mcp, client, context)
    document_tools.register(mcp, client, context)
--- a/surfsense_mcp/mcp_server/features/knowledge_base/document_tools.py
+++ b/surfsense_mcp/mcp_server/features/knowledge_base/document_tools.py
@ -20,9 +20,7 @@ from .annotations import DELETE, WRITE, DocumentId
 from .note_ingestion import build_note_document


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the knowledge-base write and delete tools."""

    @mcp.tool(
@ -136,8 +134,7 @@ def register(
            str,
            Field(
                min_length=1,
-                description="New full text; replaces the existing content "
-                "entirely.",
+                description="New full text; replaces the existing content entirely.",
            ),
        ],
    ) -> str:
--- a/surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py
+++ b/surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py
@ -17,9 +17,7 @@ from ...core.workspace_context import WorkspaceContext, WorkspaceParam
 from .annotations import READ, DocumentId, DocumentTypes


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the knowledge-base read tools."""

    @mcp.tool(
@ -81,12 +79,8 @@ def register(
            int | None,
            Field(description="Only documents in this folder. Omit for all."),
        ] = None,
-        page: Annotated[
-            int, Field(ge=0, description="Zero-based page number.")
-        ] = 0,
-        page_size: Annotated[
-            int, Field(ge=1, description="Documents per page.")
-        ] = 20,
+        page: Annotated[int, Field(ge=0, description="Zero-based page number.")] = 0,
+        page_size: Annotated[int, Field(ge=1, description="Documents per page.")] = 20,
        workspace: WorkspaceParam = None,
        response_format: ResponseFormatParam = "markdown",
    ) -> str:
--- a/surfsense_mcp/mcp_server/features/scrapers/init.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/init.py
@ -35,9 +35,7 @@ _REGISTRARS = (
 )


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register every scraper and run-history tool on the server."""
    for module in _REGISTRARS:
        module.register(mcp, client, context)
--- a/surfsense_mcp/mcp_server/features/scrapers/capability.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/capability.py
@ -38,9 +38,7 @@ async def run_scraper(
    return _render_markdown(platform, verb, resolved.name, result)


-def _render_markdown(
-    platform: str, verb: str, workspace_name: str, result: Any
-) -> str:
+def _render_markdown(platform: str, verb: str, workspace_name: str, result: Any) -> str:
    """A readable header plus the structured payload, clipped to a safe size."""
    header = f'# {platform}.{verb} — {_describe_size(result)} from "{workspace_name}"'
    body = clip(to_json(result))
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/google_maps.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/google_maps.py
@ -16,9 +16,7 @@ from ..capability import run_scraper
 ReviewSort = Literal["newest", "mostRelevant", "highestRanking", "lowestRanking"]


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the Google Maps place and review tools."""

    @mcp.tool(
@ -45,10 +43,7 @@ def register(
        ] = None,
        location: Annotated[
            str | None,
-            Field(
-                description="Geographic scope for a search, e.g. "
-                "'Seattle, USA'."
-            ),
+            Field(description="Geographic scope for a search, e.g. 'Seattle, USA'."),
        ] = None,
        max_places: Annotated[
            int, Field(ge=1, description="Maximum places to return.")
@ -56,8 +51,7 @@ def register(
        include_details: Annotated[
            bool,
            Field(
-                description="True adds opening hours and extra contact info "
-                "(slower)."
+                description="True adds opening hours and extra contact info (slower)."
            ),
        ] = False,
        workspace: WorkspaceParam = None,
@ -96,16 +90,11 @@ def register(
    async def google_maps_reviews(
        urls: Annotated[
            list[str] | None,
-            Field(
-                description="Google Maps URLs of places. Provide urls OR "
-                "place_ids."
-            ),
+            Field(description="Google Maps URLs of places. Provide urls OR place_ids."),
        ] = None,
        place_ids: Annotated[
            list[str] | None,
-            Field(
-                description="Google place ids from surfsense_google_maps_scrape."
-            ),
+            Field(description="Google place ids from surfsense_google_maps_scrape."),
        ] = None,
        max_reviews: Annotated[
            int, Field(ge=1, description="Maximum reviews per place.")
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/google_search.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/google_search.py
@ -14,9 +14,7 @@ from ..annotations import SCRAPE
 from ..capability import run_scraper


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the Google Search tool."""

    @mcp.tool(
@ -46,9 +44,7 @@ def register(
        ] = "",
        site: Annotated[
            str | None,
-            Field(
-                description="Restrict results to one domain, e.g. 'example.com'."
-            ),
+            Field(description="Restrict results to one domain, e.g. 'example.com'."),
        ] = None,
        workspace: WorkspaceParam = None,
        response_format: ResponseFormatParam = "markdown",
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py
@ -17,9 +17,7 @@ ResultType = Literal["posts", "reels"]
 SearchType = Literal["profile", "user"]


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the Instagram scrape and details tools (anonymous-only)."""

    @mcp.tool(
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/reddit.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/reddit.py
@ -17,9 +17,7 @@ RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"]
 RedditTime = Literal["hour", "day", "week", "month", "year", "all"]


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the Reddit tool."""

    @mcp.tool(
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/tiktok.py
@ -14,9 +14,7 @@ from ..annotations import SCRAPE
 from ..capability import run_scraper


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the TikTok tools."""

    @mcp.tool(
@ -32,7 +30,7 @@ def register(
                description="TikTok URLs: a video, a profile "
                "('https://www.tiktok.com/@nasa'), a hashtag "
                "('https://www.tiktok.com/tag/food'), or a search URL. Provide "
-                "urls OR profiles/hashtags/search_queries."
+                "urls OR profiles/hashtags."
            ),
        ] = None,
        profiles: Annotated[
@ -45,23 +43,12 @@ def register(
        hashtags: Annotated[
            list[str] | None,
            Field(
-                description="Hashtag names to scrape, without the '#', e.g. "
-                "['food']."
-            ),
-        ] = None,
-        search_queries: Annotated[
-            list[str] | None,
-            Field(
-                description="Keyword search terms, resolved via Google to public "
-                "TikTok videos (TikTok's own keyword search is login-walled). "
-                "Slower than hashtags/urls, so start with at most 3 queries and "
-                "expand only if nothing significant is found. For accounts by "
-                "keyword use surfsense_tiktok_user_search."
+                description="Hashtag names to scrape, without the '#', e.g. ['food']."
            ),
        ] = None,
        results_per_page: Annotated[
            int,
-            Field(ge=1, description="Max videos per profile/hashtag/search target."),
+            Field(ge=1, description="Max videos per profile/hashtag target."),
        ] = 10,
        max_items: Annotated[
            int, Field(ge=1, description="Maximum videos to return in total.")
@ -69,15 +56,13 @@ def register(
        workspace: WorkspaceParam = None,
        response_format: ResponseFormatParam = "markdown",
    ) -> str:
-        """Scrape public TikTok videos by hashtag, profile, URL, or keyword.
+        """Scrape public TikTok videos by hashtag, profile, or URL.

        Use for TikTok video research — a creator's videos, a hashtag feed, or a
        specific video/profile/hashtag URL — instead of a generic web search.
-        search_queries also finds videos on a topic (resolved via Google), but is
-        slower: start with at most 3 queries and expand only if nothing
-        significant is found. Returns videos with text, author, stats, music, and
-        the web URL. For accounts by keyword use surfsense_tiktok_user_search.
-        Example: hashtags=['food'], max_items=20.
+        Returns videos with text, author, stats, music, and the web URL. There is
+        no keyword-video search; for accounts by keyword use
+        surfsense_tiktok_user_search. Example: hashtags=['food'], max_items=20.
        """
        return await run_scraper(
            client,
@ -88,7 +73,6 @@ def register(
                "urls": urls,
                "profiles": profiles,
                "hashtags": hashtags,
-                "search_queries": search_queries,
                "results_per_page": results_per_page,
                "max_items": max_items,
            },
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/web.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/web.py
@ -14,9 +14,7 @@ from ..annotations import SCRAPE
 from ..capability import run_scraper


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the web crawl tool."""

    @mcp.tool(
--- a/surfsense_mcp/mcp_server/features/scrapers/platforms/youtube.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/youtube.py
@ -16,9 +16,7 @@ from ..capability import run_scraper
 CommentSort = Literal["TOP_COMMENTS", "NEWEST_FIRST"]


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the YouTube video and comment tools."""

    @mcp.tool(
--- a/surfsense_mcp/mcp_server/features/scrapers/run_history.py
+++ b/surfsense_mcp/mcp_server/features/scrapers/run_history.py
@ -17,9 +17,7 @@ from ...core.workspace_context import WorkspaceContext, WorkspaceParam
 from .annotations import READ_RUNS


-def register(
-    mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext
-) -> None:
+def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None:
    """Register the run-history tools."""

    @mcp.tool(
@ -29,9 +27,7 @@ def register(
        structured_output=False,
    )
    async def list_scraper_runs(
-        limit: Annotated[
-            int, Field(ge=1, description="Maximum runs to list.")
-        ] = 20,
+        limit: Annotated[int, Field(ge=1, description="Maximum runs to list.")] = 20,
        capability: Annotated[
            str | None,
            Field(