mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-24 07:31:01 +02:00
feat(fetch): Fetcher trait so vertical extractors work under any HTTP backend
Adds `webclaw_fetch::Fetcher` trait. All 28 vertical extractors now
take `client: &dyn Fetcher` instead of `client: &FetchClient` directly.
Backwards-compatible: FetchClient implements Fetcher, blanket impls
cover `&T` and `Arc<T>`, so existing CLI / MCP / self-hosted-server
callers keep working unchanged.
Motivation: the production API server (api.webclaw.io) must not do
in-process TLS fingerprinting; it delegates all HTTP to the Go
tls-sidecar. Before this trait, exposing /v1/scrape/{vertical} on
production would have required importing wreq into the server's
dep graph, violating the CLAUDE.md rule. Now production can provide
its own TlsSidecarFetcher implementation and pass it to the same
dispatcher the OSS server uses.
Changes:
- New `crates/webclaw-fetch/src/fetcher.rs` defining the trait plus
blanket impls for `&T` and `Arc<T>`.
- `FetchClient` gains a tiny impl block in client.rs that forwards to
its existing public methods.
- All 28 extractor signatures migrated from `&FetchClient` to
`&dyn Fetcher` (sed-driven bulk rewrite, no semantic change).
- `cloud::smart_fetch` and `cloud::smart_fetch_html` take `&dyn Fetcher`.
- `extractors::dispatch_by_url` and `extractors::dispatch_by_name`
take `&dyn Fetcher`.
- `async-trait 0.1` added to webclaw-fetch deps (Rust 1.75+ has
native async-fn-in-trait but dyn dispatch still needs async_trait).
- Version bumped to 0.5.1, CHANGELOG updated.
Tests: 215 passing in webclaw-fetch (no new tests needed — the existing
extractor tests exercise the trait methods transparently).
Clippy: clean workspace-wide.
This commit is contained in:
parent
aaa5103504
commit
058493bc8f
37 changed files with 241 additions and 73 deletions
|
|
@ -32,9 +32,9 @@ use regex::Regex;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::cloud::{self, CloudError};
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "amazon_product",
|
||||
|
|
@ -59,7 +59,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_asin(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let asin = parse_asin(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("amazon_product: no ASIN in '{url}'")))?;
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ use quick_xml::events::Event;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "arxiv",
|
||||
|
|
@ -32,7 +32,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/abs/") || url.contains("/pdf/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let id = parse_id(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("arxiv: cannot parse id from '{url}'")))?;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "crates_io",
|
||||
|
|
@ -30,7 +30,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/crates/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let name = parse_name(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("crates.io: cannot parse name from '{url}'")))?;
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "dev_to",
|
||||
|
|
@ -61,7 +61,7 @@ const RESERVED_FIRST_SEGS: &[&str] = &[
|
|||
"t",
|
||||
];
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (username, slug) = parse_username_slug(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("dev_to: cannot parse username/slug from '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "docker_hub",
|
||||
|
|
@ -29,7 +29,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/_/") || url.contains("/r/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (namespace, name) = parse_repo(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("docker_hub: cannot parse repo from '{url}'")))?;
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ use regex::Regex;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::cloud::{self, CloudError};
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "ebay_listing",
|
||||
|
|
@ -39,7 +39,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_item_id(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let item_id = parse_item_id(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("ebay_listing: no item id in '{url}'")))?;
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ use regex::Regex;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "ecommerce_product",
|
||||
|
|
@ -69,7 +69,7 @@ pub fn matches(url: &str) -> bool {
|
|||
!host_of(url).is_empty()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let resp = client.fetch(url).await?;
|
||||
if !(200..300).contains(&resp.status) {
|
||||
return Err(FetchError::Build(format!(
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ use regex::Regex;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::cloud::{self, CloudError};
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "etsy_listing",
|
||||
|
|
@ -49,7 +49,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_listing_id(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let listing_id = parse_listing_id(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("etsy_listing: no listing id in '{url}'")))?;
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "github_issue",
|
||||
|
|
@ -34,7 +34,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_issue(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (owner, repo, number) = parse_issue(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("github_issue: cannot parse issue URL '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "github_pr",
|
||||
|
|
@ -33,7 +33,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_pr(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (owner, repo, number) = parse_pr(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("github_pr: cannot parse pull-request URL '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "github_release",
|
||||
|
|
@ -32,7 +32,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_release(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (owner, repo, tag) = parse_release(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("github_release: cannot parse release URL '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "github_repo",
|
||||
|
|
@ -70,7 +70,7 @@ const RESERVED_OWNERS: &[&str] = &[
|
|||
"about",
|
||||
];
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (owner, repo) = parse_owner_repo(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("github_repo: cannot parse owner/repo from '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "hackernews",
|
||||
|
|
@ -40,7 +40,7 @@ pub fn matches(url: &str) -> bool {
|
|||
false
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let id = parse_item_id(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("hackernews: cannot parse item id from '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "huggingface_dataset",
|
||||
|
|
@ -38,7 +38,7 @@ pub fn matches(url: &str) -> bool {
|
|||
segs.first().copied() == Some("datasets") && (segs.len() == 2 || segs.len() == 3)
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let dataset_path = parse_dataset_path(url).ok_or_else(|| {
|
||||
FetchError::Build(format!(
|
||||
"hf_dataset: cannot parse dataset path from '{url}'"
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "huggingface_model",
|
||||
|
|
@ -61,7 +61,7 @@ const RESERVED_NAMESPACES: &[&str] = &[
|
|||
"search",
|
||||
];
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (owner, name) = parse_owner_name(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("hf model: cannot parse owner/name from '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ use serde_json::{Value, json};
|
|||
use std::sync::OnceLock;
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "instagram_post",
|
||||
|
|
@ -33,7 +33,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_shortcode(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (kind, shortcode) = parse_shortcode(url).ok_or_else(|| {
|
||||
FetchError::Build(format!(
|
||||
"instagram_post: cannot parse shortcode from '{url}'"
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "instagram_profile",
|
||||
|
|
@ -80,7 +80,7 @@ const RESERVED: &[&str] = &[
|
|||
"signup",
|
||||
];
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let username = parse_username(url).ok_or_else(|| {
|
||||
FetchError::Build(format!(
|
||||
"instagram_profile: cannot parse username from '{url}'"
|
||||
|
|
@ -198,7 +198,7 @@ fn classify(n: &MediaNode) -> &'static str {
|
|||
/// pull whatever OG tags we can. Returns less data and explicitly
|
||||
/// flags `data_completeness: "og_only"` so callers know.
|
||||
async fn og_fallback(
|
||||
client: &FetchClient,
|
||||
client: &dyn Fetcher,
|
||||
username: &str,
|
||||
original_url: &str,
|
||||
api_status: u16,
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ use serde_json::{Value, json};
|
|||
use std::sync::OnceLock;
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "linkedin_post",
|
||||
|
|
@ -36,7 +36,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/feed/update/urn:li:") || url.contains("/posts/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let urn = extract_urn(url).ok_or_else(|| {
|
||||
FetchError::Build(format!(
|
||||
"linkedin_post: cannot extract URN from '{url}' (expected /feed/update/urn:li:... or /posts/{{slug}}-{{id}})"
|
||||
|
|
|
|||
|
|
@ -46,8 +46,8 @@ pub mod youtube_video;
|
|||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
/// Public catalog entry for `/v1/extractors`. Stable shape — clients
|
||||
/// rely on `name` to pick the right `/v1/scrape/{name}` route.
|
||||
|
|
@ -102,7 +102,7 @@ pub fn list() -> Vec<ExtractorInfo> {
|
|||
/// one that claims the URL. Used by `/v1/scrape` when the caller doesn't
|
||||
/// pick a vertical explicitly.
|
||||
pub async fn dispatch_by_url(
|
||||
client: &FetchClient,
|
||||
client: &dyn Fetcher,
|
||||
url: &str,
|
||||
) -> Option<Result<(&'static str, Value), FetchError>> {
|
||||
if reddit::matches(url) {
|
||||
|
|
@ -281,7 +281,7 @@ pub async fn dispatch_by_url(
|
|||
/// users get a clear "wrong route" error instead of a confusing parse
|
||||
/// failure deep in the extractor.
|
||||
pub async fn dispatch_by_name(
|
||||
client: &FetchClient,
|
||||
client: &dyn Fetcher,
|
||||
name: &str,
|
||||
url: &str,
|
||||
) -> Result<Value, ExtractorDispatchError> {
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "npm",
|
||||
|
|
@ -31,7 +31,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/package/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let name = parse_name(url)
|
||||
.ok_or_else(|| FetchError::Build(format!("npm: cannot parse name from '{url}'")))?;
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchErro
|
|||
}))
|
||||
}
|
||||
|
||||
async fn fetch_weekly_downloads(client: &FetchClient, name: &str) -> Result<i64, FetchError> {
|
||||
async fn fetch_weekly_downloads(client: &dyn Fetcher, name: &str) -> Result<i64, FetchError> {
|
||||
let url = format!(
|
||||
"https://api.npmjs.org/downloads/point/last-week/{}",
|
||||
urlencode_segment(name)
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "pypi",
|
||||
|
|
@ -30,7 +30,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/project/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (name, version) = parse_project(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("pypi: cannot parse package name from '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "reddit",
|
||||
|
|
@ -32,7 +32,7 @@ pub fn matches(url: &str) -> bool {
|
|||
is_reddit_host && url.contains("/comments/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let json_url = build_json_url(url);
|
||||
let resp = client.fetch(&json_url).await?;
|
||||
if resp.status != 200 {
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "shopify_collection",
|
||||
|
|
@ -49,7 +49,7 @@ const NON_SHOPIFY_HOSTS: &[&str] = &[
|
|||
"github.com",
|
||||
];
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let (coll_meta_url, coll_products_url) = build_json_urls(url);
|
||||
|
||||
// Step 1: collection metadata. Shopify returns 200 on missing
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "shopify_product",
|
||||
|
|
@ -65,7 +65,7 @@ const NON_SHOPIFY_HOSTS: &[&str] = &[
|
|||
"github.com", // /products is a marketing page
|
||||
];
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let json_url = build_json_url(url);
|
||||
let resp = client.fetch(&json_url).await?;
|
||||
if resp.status == 404 {
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "stackoverflow",
|
||||
|
|
@ -31,7 +31,7 @@ pub fn matches(url: &str) -> bool {
|
|||
parse_question_id(url).is_some()
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let id = parse_question_id(url).ok_or_else(|| {
|
||||
FetchError::Build(format!(
|
||||
"stackoverflow: cannot parse question id from '{url}'"
|
||||
|
|
|
|||
|
|
@ -28,9 +28,9 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::cloud::{self, CloudError};
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "substack_post",
|
||||
|
|
@ -49,7 +49,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/p/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let slug = parse_slug(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("substack_post: cannot parse slug from '{url}'"))
|
||||
})?;
|
||||
|
|
@ -149,7 +149,7 @@ fn build_api_payload(url: &str, api_url: &str, slug: &str, p: Post) -> Value {
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn html_fallback(
|
||||
client: &FetchClient,
|
||||
client: &dyn Fetcher,
|
||||
url: &str,
|
||||
api_url: &str,
|
||||
slug: &str,
|
||||
|
|
|
|||
|
|
@ -32,9 +32,9 @@ use regex::Regex;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::cloud::{self, CloudError};
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "trustpilot_reviews",
|
||||
|
|
@ -51,7 +51,7 @@ pub fn matches(url: &str) -> bool {
|
|||
url.contains("/review/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let fetched = cloud::smart_fetch_html(client, client.cloud(), url)
|
||||
.await
|
||||
.map_err(cloud_to_fetch_err)?;
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ use serde::Deserialize;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "woocommerce_product",
|
||||
|
|
@ -42,7 +42,7 @@ pub fn matches(url: &str) -> bool {
|
|||
|| url.contains("/produit/") // common fr locale
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let slug = parse_slug(url).ok_or_else(|| {
|
||||
FetchError::Build(format!(
|
||||
"woocommerce_product: cannot parse slug from '{url}'"
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ use regex::Regex;
|
|||
use serde_json::{Value, json};
|
||||
|
||||
use super::ExtractorInfo;
|
||||
use crate::client::FetchClient;
|
||||
use crate::error::FetchError;
|
||||
use crate::fetcher::Fetcher;
|
||||
|
||||
pub const INFO: ExtractorInfo = ExtractorInfo {
|
||||
name: "youtube_video",
|
||||
|
|
@ -45,7 +45,7 @@ pub fn matches(url: &str) -> bool {
|
|||
|| url.contains("youtube-nocookie.com/embed/")
|
||||
}
|
||||
|
||||
pub async fn extract(client: &FetchClient, url: &str) -> Result<Value, FetchError> {
|
||||
pub async fn extract(client: &dyn Fetcher, url: &str) -> Result<Value, FetchError> {
|
||||
let video_id = parse_video_id(url).ok_or_else(|| {
|
||||
FetchError::Build(format!("youtube_video: cannot parse video id from '{url}'"))
|
||||
})?;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue