mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-06-13 23:15:13 +02:00
feat(fetch): periodic progress stderr line on slow fetches
Webclaw's default -t timeout is 30s; slow sites previously sat silently with no feedback. Now during a fetch, every 10s of elapsed time webclaw writes one line to stderr: # webclaw: still fetching <URL> (Ns) Fetches completing in under 10s emit nothing (the timer never fires). Stdout output is untouched - pure feedback signal on stderr. No timeout change. No new flags. Default behavior is augmented at stderr only. Implemented via tokio::select! between the fetch future and a tokio::time::interval. Latency cost: a single tokio task spawn and a 10s tick - microseconds on the fast path. 10 new tests in webclaw-fetch::progress::tests (none ignored; the slow-future test uses a 50ms test interval to keep cargo test fast). Workspace total 710 -> 720.
This commit is contained in:
parent
d5a3aa4bf9
commit
06f065cb08
3 changed files with 299 additions and 1 deletions
|
|
@ -1023,7 +1023,11 @@ async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
|||
let client =
|
||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?;
|
||||
let options = build_extraction_options(cli);
|
||||
let result = match client.fetch_and_extract_with_options(url, &options).await {
|
||||
// M13: wrap with periodic stderr progress emitter. Fast fetches see
|
||||
// zero emissions (timer never fires in <10s); slow fetches get a
|
||||
// line every 10s of elapsed time so the CLI doesn't appear hung.
|
||||
let fetch_fut = client.fetch_and_extract_with_options(url, &options);
|
||||
let result = match webclaw_fetch::with_progress(url, fetch_fut).await {
|
||||
Ok(r) => r,
|
||||
// M3: known-bad-sites registry hit. The error message is already
|
||||
// formatted per phase-A contract. Emit it to stderr verbatim and
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue