mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-24 07:31:01 +02:00
fix(cli): --urls-file with a single row now works (#86)
A one-line --urls-file produced entries.len()==1, which the batch gates (run_batch / run_batch_llm require >1) skipped, so control fell to the single-scrape path — which reads cli.urls (empty) and errored 'no input provided'. Two+ rows worked; exactly one did not. Backfill the lone file URL into cli.urls before the gates so a one-line file behaves identically to a positional URL (stdout, --output-dir, cloud fallback, --raw-html, --diff, LLM paths). The comma/CSV url,filename parsing is unchanged (verified irrelevant to this bug). Adds 3 network-free tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
4072861677
commit
7f63d2f17a
1 changed files with 48 additions and 1 deletions
|
|
@ -864,6 +864,18 @@ fn collect_urls(cli: &Cli) -> Result<Vec<(String, Option<String>)>, String> {
|
|||
Ok(entries)
|
||||
}
|
||||
|
||||
/// When `--urls-file` supplies exactly one URL and no positional URL was given,
|
||||
/// treat it like a single positional URL so a one-line file behaves identically
|
||||
/// to `webclaw <url>` (stdout, `--output-dir`, cloud fallback, `--raw-html`,
|
||||
/// `--diff`, LLM paths). Without this the lone file URL gives `entries.len() == 1`,
|
||||
/// which the batch gates skip, and the single path then finds an empty `cli.urls`
|
||||
/// and errors "no input provided". See issue #86.
|
||||
fn backfill_single_file_url(urls: &mut Vec<String>, entries: &[(String, Option<String>)]) {
|
||||
if urls.is_empty() && entries.len() == 1 {
|
||||
urls.push(entries[0].0.clone());
|
||||
}
|
||||
}
|
||||
|
||||
/// Result that can be either a local extraction or a cloud API JSON response.
|
||||
enum FetchOutput {
|
||||
Local(Box<ExtractionResult>),
|
||||
|
|
@ -2624,7 +2636,7 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
|
|||
async fn main() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let cli = Cli::parse();
|
||||
let mut cli = Cli::parse();
|
||||
init_logging(cli.verbose);
|
||||
|
||||
// Subcommand path. Handled before the flag dispatch so a subcommand
|
||||
|
|
@ -2828,6 +2840,11 @@ async fn main() {
|
|||
}
|
||||
};
|
||||
|
||||
// A single URL sourced only from --urls-file must behave like a positional
|
||||
// URL (issue #86): the batch gates below require len > 1, and the single
|
||||
// path reads cli.urls, so a one-line file would otherwise hit "no input".
|
||||
backfill_single_file_url(&mut cli.urls, &entries);
|
||||
|
||||
// LLM modes: --extract-json, --extract-prompt, --summarize
|
||||
// When multiple URLs are provided, run batch LLM extraction over all of them.
|
||||
if has_llm_flags(&cli) {
|
||||
|
|
@ -2899,6 +2916,36 @@ mod tests {
|
|||
use super::*;
|
||||
use webclaw_core::Content;
|
||||
|
||||
// issue #86: a single URL sourced only from --urls-file must be promoted to
|
||||
// a positional URL so it takes the single-scrape path (batch gates need >1).
|
||||
#[test]
|
||||
fn single_file_url_backfilled_into_positional() {
|
||||
let mut urls: Vec<String> = Vec::new();
|
||||
let entries = vec![("https://example.com".to_string(), None)];
|
||||
backfill_single_file_url(&mut urls, &entries);
|
||||
assert_eq!(urls, vec!["https://example.com".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positional_urls_left_untouched() {
|
||||
let mut urls = vec!["https://a.com".to_string()];
|
||||
let entries = vec![("https://a.com".to_string(), None)];
|
||||
backfill_single_file_url(&mut urls, &entries);
|
||||
assert_eq!(urls, vec!["https://a.com".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_file_urls_not_backfilled() {
|
||||
// 2+ URLs already route to run_batch, so cli.urls stays empty here.
|
||||
let mut urls: Vec<String> = Vec::new();
|
||||
let entries = vec![
|
||||
("https://a.com".to_string(), None),
|
||||
("https://b.com".to_string(), None),
|
||||
];
|
||||
backfill_single_file_url(&mut urls, &entries);
|
||||
assert!(urls.is_empty());
|
||||
}
|
||||
|
||||
fn empty_result(title: Option<&str>, url: Option<&str>, markdown: &str) -> ExtractionResult {
|
||||
ExtractionResult {
|
||||
metadata: Metadata {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue