From 7f63d2f17a6f23a847a487897a89bb5649b2c5eb Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 22 Jul 2026 17:35:57 +0200 Subject: [PATCH 1/2] fix(cli): --urls-file with a single row now works (#86) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A one-line --urls-file produced entries.len()==1, which the batch gates (run_batch / run_batch_llm require >1) skipped, so control fell to the single-scrape path — which reads cli.urls (empty) and errored 'no input provided'. Two+ rows worked; exactly one did not. Backfill the lone file URL into cli.urls before the gates so a one-line file behaves identically to a positional URL (stdout, --output-dir, cloud fallback, --raw-html, --diff, LLM paths). The comma/CSV url,filename parsing is unchanged (verified irrelevant to this bug). Adds 3 network-free tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/webclaw-cli/src/main.rs | 49 +++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index e3434c5..de98438 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -864,6 +864,18 @@ fn collect_urls(cli: &Cli) -> Result)>, String> { Ok(entries) } +/// When `--urls-file` supplies exactly one URL and no positional URL was given, +/// treat it like a single positional URL so a one-line file behaves identically +/// to `webclaw ` (stdout, `--output-dir`, cloud fallback, `--raw-html`, +/// `--diff`, LLM paths). Without this the lone file URL gives `entries.len() == 1`, +/// which the batch gates skip, and the single path then finds an empty `cli.urls` +/// and errors "no input provided". See issue #86. +fn backfill_single_file_url(urls: &mut Vec, entries: &[(String, Option)]) { + if urls.is_empty() && entries.len() == 1 { + urls.push(entries[0].0.clone()); + } +} + /// Result that can be either a local extraction or a cloud API JSON response. enum FetchOutput { Local(Box), @@ -2624,7 +2636,7 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> { async fn main() { dotenvy::dotenv().ok(); - let cli = Cli::parse(); + let mut cli = Cli::parse(); init_logging(cli.verbose); // Subcommand path. Handled before the flag dispatch so a subcommand @@ -2828,6 +2840,11 @@ async fn main() { } }; + // A single URL sourced only from --urls-file must behave like a positional + // URL (issue #86): the batch gates below require len > 1, and the single + // path reads cli.urls, so a one-line file would otherwise hit "no input". + backfill_single_file_url(&mut cli.urls, &entries); + // LLM modes: --extract-json, --extract-prompt, --summarize // When multiple URLs are provided, run batch LLM extraction over all of them. if has_llm_flags(&cli) { @@ -2899,6 +2916,36 @@ mod tests { use super::*; use webclaw_core::Content; + // issue #86: a single URL sourced only from --urls-file must be promoted to + // a positional URL so it takes the single-scrape path (batch gates need >1). + #[test] + fn single_file_url_backfilled_into_positional() { + let mut urls: Vec = Vec::new(); + let entries = vec![("https://example.com".to_string(), None)]; + backfill_single_file_url(&mut urls, &entries); + assert_eq!(urls, vec!["https://example.com".to_string()]); + } + + #[test] + fn positional_urls_left_untouched() { + let mut urls = vec!["https://a.com".to_string()]; + let entries = vec![("https://a.com".to_string(), None)]; + backfill_single_file_url(&mut urls, &entries); + assert_eq!(urls, vec!["https://a.com".to_string()]); + } + + #[test] + fn multiple_file_urls_not_backfilled() { + // 2+ URLs already route to run_batch, so cli.urls stays empty here. + let mut urls: Vec = Vec::new(); + let entries = vec![ + ("https://a.com".to_string(), None), + ("https://b.com".to_string(), None), + ]; + backfill_single_file_url(&mut urls, &entries); + assert!(urls.is_empty()); + } + fn empty_result(title: Option<&str>, url: Option<&str>, markdown: &str) -> ExtractionResult { ExtractionResult { metadata: Metadata { From 36cde1d488e1e2cf7085865401d4861723fed9d6 Mon Sep 17 00:00:00 2001 From: Valerio Date: Wed, 22 Jul 2026 17:47:23 +0200 Subject: [PATCH 2/2] fix(cli): also fix --diff/--brand for single-row --urls-file (review) Independent review found the backfill ran AFTER the --diff-with and --brand gates, so single-row --urls-file + --diff/--brand still hit 'no input'. Move the collect_urls + backfill before all mode gates so a one-line file behaves like a positional URL everywhere. --watch now reuses the shared entries instead of re-collecting (avoids double-counting the backfilled URL). Verified: --brand single-row no longer errors; 37 tests pass, clippy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/webclaw-cli/src/main.rs | 36 ++++++++++++++-------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/crates/webclaw-cli/src/main.rs b/crates/webclaw-cli/src/main.rs index de98438..11da654 100644 --- a/crates/webclaw-cli/src/main.rs +++ b/crates/webclaw-cli/src/main.rs @@ -2779,6 +2779,20 @@ async fn main() { return; } + // Collect URLs from args + --urls-file up front, and backfill a lone + // --urls-file URL into cli.urls so a one-line file behaves like a positional + // URL across ALL modes below (crawl / watch / diff / brand / batch / single). + // See issue #86 — the mode gates and single path read cli.urls, which is + // empty when the URL came only from --urls-file. + let entries = match collect_urls(&cli) { + Ok(u) => u, + Err(e) => { + eprintln!("error: {e}"); + process::exit(1); + } + }; + backfill_single_file_url(&mut cli.urls, &entries); + // --crawl: recursive crawl mode if cli.crawl { if let Err(e) = run_crawl(&cli).await { @@ -2790,13 +2804,7 @@ async fn main() { // --watch: poll URL(s) for changes if cli.watch { - let watch_urls: Vec = match collect_urls(&cli) { - Ok(entries) => entries.into_iter().map(|(url, _)| url).collect(), - Err(e) => { - eprintln!("error: {e}"); - process::exit(1); - } - }; + let watch_urls: Vec = entries.iter().map(|(url, _)| url.clone()).collect(); if let Err(e) = run_watch(&cli, &watch_urls).await { eprintln!("error: {e}"); process::exit(1); @@ -2831,20 +2839,6 @@ async fn main() { return; } - // Collect all URLs from args + --urls-file - let entries = match collect_urls(&cli) { - Ok(u) => u, - Err(e) => { - eprintln!("error: {e}"); - process::exit(1); - } - }; - - // A single URL sourced only from --urls-file must behave like a positional - // URL (issue #86): the batch gates below require len > 1, and the single - // path reads cli.urls, so a one-line file would otherwise hit "no input". - backfill_single_file_url(&mut cli.urls, &entries); - // LLM modes: --extract-json, --extract-prompt, --summarize // When multiple URLs are provided, run batch LLM extraction over all of them. if has_llm_flags(&cli) {