apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-16 23:01:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-13 17:11:25 -07:00
parent e8b3692b54
commit 2b018c4474
111 changed files with 1800 additions and 1580 deletions
Download patch file Download diff file Expand all files Collapse all files

7
surfsense_web/content/docs/connectors/native/tiktok.mdx
View file

@ -11,15 +11,14 @@ The TikTok connector pulls structured public data from TikTok across four verbs:
POST /api/v1/workspaces/{workspace_id}/scrapers/tiktok/scrape
```
Give it URLs (a video, a profile, a hashtag, or a search page) and/or profiles, hashtags, or search terms; returns videos (caption, author, play/like/comment/share counts, music, hashtags, timestamps, and the web URL). At least one of `urls`, `profiles`, `hashtags`, or `search_queries` is required.
Give it URLs (a video, a profile, a hashtag, or a search page) and/or profiles or hashtags; returns videos (caption, author, play/like/comment/share counts, music, hashtags, timestamps, and the web URL). At least one of `urls`, `profiles`, or `hashtags` is required.
| Field | Default | Description |
|-------|---------|-------------|
| `urls` | — | TikTok URLs: a video, a profile (`/@<user>`), a hashtag (`/tag/<name>`), or a search URL (max 20 sources per call) |
| `profiles` | — | Profile usernames, with or without a leading `@` |
| `hashtags` | — | Hashtag names, without the `#` |
| `search_queries` | — | Search terms to run on TikTok |
| `results_per_page` | `10` | Max videos per profile/hashtag/search target |
| `results_per_page` | `10` | Max videos per profile/hashtag target |
| `max_items` | `10` | Max total videos returned across all sources (hard cap 100) |
```bash
@ -30,7 +29,7 @@ curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/tiktok/scrape" \
```
<Callout type="info">
Video and hashtag targets are the reliable video paths. A `profiles` target returns the account's **metadata** (name, followers, bio, verification) reliably, but TikTok often withholds its **video list** from automated clients — so a profile can return metadata with no videos. Keyword **video** search is login-walled and returns a surfaced error; to find accounts by keyword use **user search** below.
Video and hashtag targets are the reliable video paths. A `profiles` target returns the account's **metadata** (name, followers, bio, verification) reliably, but TikTok often withholds its **video list** from automated clients — so a profile can return metadata with no videos. There is no keyword **video** search (TikTok's own search is login-walled); to find accounts by keyword use **user search** below.
</Callout>
## Comments

11
surfsense_web/lib/connectors-marketing/tiktok.tsx
View file

@ -159,7 +159,7 @@ export const tiktok: ConnectorPageContent = {
schema: {
requestNote:
"Provide at least one source: urls, profiles, hashtags, or search_queries. Up to 20 sources per call.",
"Provide at least one source: urls, profiles, or hashtags. Up to 20 sources per call. To find accounts by keyword, use the user search verb.",
request: [
{
name: "urls",
@ -180,13 +180,6 @@ export const tiktok: ConnectorPageContent = {
defaultValue: "[]",
description: "Hashtag names to scrape, without the # prefix. Max 20.",
},
{
name: "search_queries",
type: "string[]",
defaultValue: "[]",
description:
"Keyword search terms. Keyword video search is login-walled and returns no videos; use hashtags/profiles/urls for videos, or user_search for accounts. Max 20.",
},
{
name: "results_per_page",
type: "integer",
@ -270,7 +263,7 @@ export const tiktok: ConnectorPageContent = {
{
question: "Can I scrape TikTok comments and hashtags?",
answer:
"Yes. Pass a video URL to the comments endpoint for the public comment thread. Pass hashtag names or /tag/ URLs to the TikTok hashtag scraper to pull videos under that tag. Keyword video search is login-walled, so hashtags and direct URLs are the reliable discovery paths.",
"Yes. Pass a video URL to the comments endpoint for the public comment thread. Pass hashtag names or /tag/ URLs to the TikTok hashtag scraper to pull videos under that tag. Keyword video search is login-walled on TikTok, so hashtags and direct URLs are the reliable discovery paths; to find accounts by keyword, use the user search verb.",
},
{
question: "What are the rate limits?",