apunkt/webclaw
1
0
Fork 0
mirror of https://github.com/0xMassi/webclaw.git synced 2026-07-22 07:11:01 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(llm): add Gemini CLI provider as primary; set qwen3.5:9b as default Ollama model

Browse source 
- Add GeminiCliProvider: shells out to `gemini -p` with --output-format json,
  injection-safe prompt passing, MCP server suppression via temp workdir,
  6-slot concurrency semaphore, 60s subprocess deadline
- Add --llm-provider, --llm-model, --llm-base-url CLI flags for per-call overrides
- Provider chain: Gemini CLI → OpenAI → Ollama → Anthropic
- Move LLM timing to dispatch layer (LLM: Xs on stderr)
- Default Ollama model: qwen3:8b → qwen3.5:9b (benchmark shows better schema extraction)
- Add noxa mcp subcommand
- Add docs/reports/llm-benchmark-2026-04-11.md (Gemini vs qwen3.5:4b vs qwen3.5:9b)
- Bump version 0.3.11 → 0.4.0

Co-authored-by: Claude <claude@anthropic.com>
This commit is contained in:
Jacob Magar 2026-04-12 00:52:53 -04:00
parent 464eb1baec
commit adf4b6ba55
39 changed files with 1999 additions and 1789 deletions
Download patch file Download diff file Expand all files Collapse all files

203
crates/noxa-cli/src/config.rs
View file

@ -1,5 +1,5 @@
use serde::Deserialize;
use std::path::Path;
use std::path::{Path, PathBuf};
use crate::{Browser, OutputFormat, PdfModeArg};
@ -16,7 +16,8 @@ use crate::{Browser, OutputFormat, PdfModeArg};
/// BOOL FLAG LIMITATION:
/// only_main_content, metadata, verbose, use_sitemap set to true here
/// cannot be overridden to false from the CLI for a single run (no --no-flag
/// variant in clap). Edit config.json or use NOXA_CONFIG=/dev/null to bypass.
/// variant in clap). Edit config.json or use NOXA_CONFIG=/dev/null (or an
/// empty file) to bypass.
#[derive(Debug, Default, Deserialize)]
pub struct NoxaConfig {
// Output
@ -47,6 +48,20 @@ pub struct NoxaConfig {
// LLM (non-secret: provider name and model only; base URL stays in .env)
pub llm_provider: Option<String>,
pub llm_model: Option<String>,
pub output_dir: Option<PathBuf>,
#[serde(default)]
pub cloud: Option<CloudConfig>,
}
#[derive(Debug, Default, Deserialize, Clone)]
pub struct CloudConfig {
pub provider: Option<String>,
pub project: Option<String>,
pub zone: Option<String>,
pub cluster: Option<String>,
pub service_account_key: Option<String>,
pub disabled: Option<bool>,
}
impl NoxaConfig {
@ -65,7 +80,8 @@ impl NoxaConfig {
let path = Path::new(&path_str);
if !path.exists() {
if was_explicit {
let display_name = path.file_name()
let display_name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or(&path_str);
eprintln!("error: config file not found: {display_name}");
@ -74,7 +90,24 @@ impl NoxaConfig {
return Self::default();
}
let display_name = path.file_name()
let content = match std::fs::read_to_string(path) {
Ok(s) => s,
Err(e) => {
let display_name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or(&path_str);
eprintln!("error: cannot read config file {display_name}: {e}");
std::process::exit(1);
}
};
if path_str == "/dev/null" || content.trim().is_empty() {
return Self::default();
}
let display_name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or(&path_str);
eprintln!(
@ -83,14 +116,6 @@ impl NoxaConfig {
);
tracing::debug!("config path: {}", path.display());
let content = match std::fs::read_to_string(path) {
Ok(s) => s,
Err(e) => {
eprintln!("error: cannot read config file {display_name}: {e}");
std::process::exit(1);
}
};
match serde_json::from_str(&content) {
Ok(cfg) => cfg,
Err(e) => {
@ -142,20 +167,31 @@ pub struct ResolvedConfig {
// LLM
pub llm_provider: Option<String>,
pub llm_model: Option<String>,
pub output_dir: Option<PathBuf>,
// Cloud
pub cloud: Option<CloudConfig>,
}
use clap::parser::ValueSource;
fn resolve_optional_setting(
cli_explicit: bool,
cli_value: Option<String>,
cfg_value: Option<String>,
env_value: Option<String>,
) -> Option<String> {
if cli_explicit {
cli_value
} else {
cfg_value.or_else(|| env_value.filter(|s| !s.is_empty()))
}
}
/// Merge CLI flags (detected via ValueSource), config file, and hard defaults
/// into a single ResolvedConfig. CLI explicit values always win.
pub fn resolve(
cli: &crate::Cli,
matches: &clap::ArgMatches,
cfg: &NoxaConfig,
) -> ResolvedConfig {
let explicit = |name: &str| {
matches.value_source(name) == Some(ValueSource::CommandLine)
};
pub fn resolve(cli: &crate::Cli, matches: &clap::ArgMatches, cfg: &NoxaConfig) -> ResolvedConfig {
let explicit = |name: &str| matches.value_source(name) == Some(ValueSource::CommandLine);
ResolvedConfig {
format: if explicit("format") {
@ -240,15 +276,40 @@ pub fn resolve(
verbose: cli.verbose || cfg.verbose.unwrap_or(false),
use_sitemap: cli.sitemap || cfg.use_sitemap.unwrap_or(false),
raw_html: cli.raw_html,
llm_provider: if cli.llm_provider.is_some() {
cli.llm_provider.clone()
llm_provider: resolve_optional_setting(
explicit("llm_provider"),
cli.llm_provider.clone(),
cfg.llm_provider.clone(),
std::env::var("NOXA_LLM_PROVIDER").ok(),
),
llm_model: resolve_optional_setting(
explicit("llm_model"),
cli.llm_model.clone(),
cfg.llm_model.clone(),
std::env::var("NOXA_LLM_MODEL").ok(),
),
output_dir: if explicit("output_dir") {
cli.output_dir.clone()
} else {
cfg.llm_provider.clone()
cfg.output_dir.clone()
},
llm_model: if cli.llm_model.is_some() {
cli.llm_model.clone()
cloud: if explicit("cloud_provider")
|| explicit("cloud_project")
|| explicit("cloud_zone")
|| explicit("cloud_cluster")
|| explicit("cloud_service_account_key")
|| explicit("cloud_disabled")
{
Some(CloudConfig {
provider: cli.cloud_provider.clone(),
project: cli.cloud_project.clone(),
zone: cli.cloud_zone.clone(),
cluster: cli.cloud_cluster.clone(),
service_account_key: cli.cloud_service_account_key.clone(),
disabled: Some(cli.cloud_disabled),
})
} else {
cfg.llm_model.clone()
cfg.cloud.clone()
},
}
}
@ -256,6 +317,7 @@ pub fn resolve(
#[cfg(test)]
mod tests {
use super::*;
use clap::{CommandFactory, Parser};
#[test]
fn test_noxa_config_deserialize_full() {
@ -283,7 +345,10 @@ mod tests {
let cfg: NoxaConfig = serde_json::from_str(json).unwrap();
assert!(matches!(cfg.format, Some(crate::OutputFormat::Llm)));
assert_eq!(cfg.depth, Some(3));
assert_eq!(cfg.exclude_paths, Some(vec!["/changelog/*".to_string(), "/blog/*".to_string()]));
assert_eq!(
cfg.exclude_paths,
Some(vec!["/changelog/*".to_string(), "/blog/*".to_string()])
);
assert!(matches!(cfg.pdf_mode, Some(crate::PdfModeArg::Fast)));
}
@ -297,10 +362,39 @@ mod tests {
#[test]
fn test_noxa_config_unknown_fields_ignored() {
// Unknown fields must NOT cause a parse failure
let cfg: NoxaConfig = serde_json::from_str(r#"{"depth": 2, "future_field": true}"#).unwrap();
let cfg: NoxaConfig =
serde_json::from_str(r#"{"depth": 2, "future_field": true}"#).unwrap();
assert_eq!(cfg.depth, Some(2));
}
#[test]
fn test_noxa_config_output_dir_deserialize() {
let cfg: NoxaConfig = serde_json::from_str(r#"{"output_dir":"out"}"#).unwrap();
assert_eq!(cfg.output_dir, Some(PathBuf::from("out")));
}
#[test]
fn test_resolve_uses_config_output_dir() {
let cli = crate::Cli::parse_from(["noxa"]);
let matches = crate::Cli::command().get_matches_from(["noxa"]);
let cfg: NoxaConfig = serde_json::from_str(r#"{"output_dir":"out"}"#).unwrap();
let resolved = resolve(&cli, &matches, &cfg);
assert_eq!(resolved.output_dir, Some(PathBuf::from("out")));
}
#[test]
fn test_resolve_prefers_config_llm_provider_over_env_default() {
let resolved =
resolve_optional_setting(false, None, Some("gemini".into()), Some("ollama".into()));
assert_eq!(resolved, Some("gemini".into()));
}
#[test]
fn test_resolve_uses_env_llm_provider_when_config_missing() {
let resolved = resolve_optional_setting(false, None, None, Some("ollama".into()));
assert_eq!(resolved, Some("ollama".into()));
}
#[test]
fn test_load_implicit_missing_file_returns_default() {
// When no explicit path and ./config.json doesn't exist, silently return default.
@ -312,4 +406,57 @@ mod tests {
let cfg = NoxaConfig::load(None);
assert!(cfg.format.is_none());
}
#[test]
fn test_load_dev_null_returns_default() {
let cfg = NoxaConfig::load(Some("/dev/null"));
assert!(cfg.format.is_none());
assert!(cfg.llm_provider.is_none());
}
#[test]
fn test_load_whitespace_file_returns_default() {
let mut path = std::env::temp_dir();
let suffix = format!(
"noxa-config-{}-{}.json",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
path.push(suffix);
std::fs::write(&path, " \n\t ").unwrap();
let cfg = NoxaConfig::load(Some(path.to_str().unwrap()));
assert!(cfg.format.is_none());
assert!(cfg.llm_model.is_none());
let _ = std::fs::remove_file(&path);
}
#[test]
fn test_noxa_config_cloud_fields() {
let json = r#"{
"cloud": {
"provider": "gcp",
"project": "my-gcp-project",
"zone": "us-central1-a",
"cluster": "my-cluster",
"service_account_key": "/path/to/key.json",
"disabled": false
}
}"#;
let cfg: NoxaConfig = serde_json::from_str(json).unwrap();
let cloud = cfg.cloud.unwrap();
assert_eq!(cloud.provider, Some("gcp".to_string()));
assert_eq!(cloud.project, Some("my-gcp-project".to_string()));
assert_eq!(cloud.zone, Some("us-central1-a".to_string()));
assert_eq!(cloud.cluster, Some("my-cluster".to_string()));
assert_eq!(
cloud.service_account_key,
Some("/path/to/key.json".to_string())
);
assert_eq!(cloud.disabled, Some(false));
}
}

268
crates/noxa-cli/src/main.rs
View file

@ -255,11 +255,11 @@ struct Cli {
summarize: Option<usize>,
/// Force a specific LLM provider (gemini, ollama, openai, anthropic)
#[arg(long, env = "NOXA_LLM_PROVIDER")]
#[arg(long)]
llm_provider: Option<String>,
/// Override the LLM model name
#[arg(long, env = "NOXA_LLM_MODEL")]
#[arg(long)]
llm_model: Option<String>,
/// Override the LLM base URL (Ollama or OpenAI-compatible)
@ -275,6 +275,30 @@ struct Cli {
#[arg(long)]
cloud: bool,
/// Cloud provider to use (e.g. "gcp", "aws")
#[arg(long, env = "NOXA_CLOUD_PROVIDER")]
cloud_provider: Option<String>,
/// Cloud project ID
#[arg(long, env = "NOXA_CLOUD_PROJECT")]
cloud_project: Option<String>,
/// Cloud zone or region
#[arg(long, env = "NOXA_CLOUD_ZONE")]
cloud_zone: Option<String>,
/// Cloud cluster name
#[arg(long, env = "NOXA_CLOUD_CLUSTER")]
cloud_cluster: Option<String>,
/// Path to cloud service account key file
#[arg(long, env = "NOXA_CLOUD_SERVICE_ACCOUNT_KEY")]
cloud_service_account_key: Option<String>,
/// Disable cloud features
#[arg(long)]
cloud_disabled: bool,
/// Run deep research on a topic via the cloud API. Requires --api-key.
/// Saves full result (report + sources + findings) to a JSON file.
#[arg(long)]
@ -571,6 +595,103 @@ fn format_output(result: &ExtractionResult, format: &OutputFormat, show_metadata
}
}
fn file_extension_for_format(format: &OutputFormat) -> &'static str {
match format {
OutputFormat::Markdown | OutputFormat::Llm => "md",
OutputFormat::Json => "json",
OutputFormat::Text => "txt",
OutputFormat::Html => "html",
}
}
fn format_cloud_output(resp: &serde_json::Value, format: &OutputFormat) -> String {
match format {
OutputFormat::Json => serde_json::to_string_pretty(resp).expect("serialization failed"),
OutputFormat::Markdown => resp
.get("content")
.and_then(|c| c.get("markdown"))
.and_then(|m| m.as_str())
.or_else(|| resp.get("markdown").and_then(|m| m.as_str()))
.map(str::to_string)
.unwrap_or_else(|| serde_json::to_string_pretty(resp).expect("serialization failed")),
OutputFormat::Text => resp
.get("content")
.and_then(|c| c.get("plain_text"))
.and_then(|t| t.as_str())
.map(str::to_string)
.unwrap_or_else(|| format_cloud_output(resp, &OutputFormat::Markdown)),
OutputFormat::Llm => resp
.get("content")
.and_then(|c| c.get("llm_text"))
.and_then(|t| t.as_str())
.map(str::to_string)
.unwrap_or_else(|| format_cloud_output(resp, &OutputFormat::Markdown)),
OutputFormat::Html => resp
.get("content")
.and_then(|c| c.get("raw_html"))
.and_then(|h| h.as_str())
.map(str::to_string)
.unwrap_or_else(|| format_cloud_output(resp, &OutputFormat::Markdown)),
}
}
fn format_diff_output(diff: &ContentDiff, format: &OutputFormat) -> String {
match format {
OutputFormat::Json => serde_json::to_string_pretty(diff).expect("serialization failed"),
_ => {
let mut out = String::new();
out.push_str(&format!("Status: {:?}\n", diff.status));
out.push_str(&format!("Word count delta: {:+}\n", diff.word_count_delta));
if !diff.metadata_changes.is_empty() {
out.push_str("\nMetadata changes:\n");
for change in &diff.metadata_changes {
out.push_str(&format!(
" {}: {} -> {}\n",
change.field,
change.old.as_deref().unwrap_or("(none)"),
change.new.as_deref().unwrap_or("(none)"),
));
}
}
if !diff.links_added.is_empty() {
out.push_str("\nLinks added:\n");
for link in &diff.links_added {
out.push_str(&format!(" + {} ({})\n", link.href, link.text));
}
}
if !diff.links_removed.is_empty() {
out.push_str("\nLinks removed:\n");
for link in &diff.links_removed {
out.push_str(&format!(" - {} ({})\n", link.href, link.text));
}
}
if let Some(ref text_diff) = diff.text_diff {
out.push_str(&format!("\n{text_diff}\n"));
}
out
}
}
}
fn format_map_output(entries: &[SitemapEntry], format: &OutputFormat) -> String {
match format {
OutputFormat::Json => serde_json::to_string_pretty(entries).expect("serialization failed"),
_ => {
let mut out = String::new();
for entry in entries {
out.push_str(&entry.url);
out.push('\n');
}
out
}
}
}
/// Collect all URLs from positional args + --urls-file, normalizing bare domains.
///
/// Returns `(url, optional_custom_filename)` pairs. Custom filenames come from
@ -1295,7 +1416,7 @@ async fn run_crawl(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), S
}
}
if let Some(ref dir) = cli.output_dir {
if let Some(ref dir) = resolved.output_dir {
let mut saved = 0usize;
for page in &result.pages {
if let Some(ref extraction) = page.extraction {
@ -1364,7 +1485,20 @@ async fn run_map(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), Str
eprintln!("discovered {} URLs", entries.len());
}
print_map_output(&entries, &resolved.format);
if let Some(ref dir) = resolved.output_dir {
let content = format_map_output(&entries, &resolved.format);
let filename = format!(
"sitemap.{}",
if matches!(resolved.format, OutputFormat::Json) {
"json"
} else {
"txt"
}
);
write_to_file(dir, &filename, &content)?;
} else {
print_map_output(&entries, &resolved.format);
}
Ok(())
}
@ -1405,7 +1539,7 @@ async fn run_batch(
.filter_map(|(url, name)| name.as_deref().map(|n| (url.as_str(), n)))
.collect();
if let Some(ref dir) = cli.output_dir {
if let Some(ref dir) = resolved.output_dir {
let mut saved = 0usize;
for r in &results {
if let Ok(ref extraction) = r.result {
@ -1749,6 +1883,20 @@ async fn run_watch_multi(
eprintln!(" -> {url} (word delta: {delta:+})");
}
if let Some(ref dir) = resolved.output_dir {
let payload = serde_json::json!({
"event": "watch_changes",
"check_number": check_number,
"total_urls": urls.len(),
"changed": changed.len(),
"same": same_count,
"changes": changed,
});
let filename = format!("watch-{}.json", ts.replace(':', "-"));
let content = serde_json::to_string_pretty(&payload).unwrap_or_default();
write_to_file(dir, &filename, &content)?;
}
// Fire --on-change once with all changes
if let Some(ref cmd) = cli.on_change {
let payload = serde_json::json!({
@ -1812,7 +1960,20 @@ async fn run_diff(
let new_result = fetch_and_extract(cli, resolved).await?.into_extraction()?;
let diff = noxa_core::diff::diff(&old, &new_result);
print_diff_output(&diff, &resolved.format);
if let Some(ref dir) = resolved.output_dir {
let content = format_diff_output(&diff, &resolved.format);
let filename = format!(
"diff.{}",
if matches!(resolved.format, OutputFormat::Json) {
"json"
} else {
"txt"
}
);
write_to_file(dir, &filename, &content)?;
} else {
print_diff_output(&diff, &resolved.format);
}
Ok(())
}
@ -1824,10 +1985,12 @@ async fn run_brand(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), S
&enriched,
Some(result.url.as_str()).filter(|s| !s.is_empty()),
);
println!(
"{}",
serde_json::to_string_pretty(&brand).expect("serialization failed")
);
let output = serde_json::to_string_pretty(&brand).expect("serialization failed");
if let Some(ref dir) = resolved.output_dir {
write_to_file(dir, "brand.json", &output)?;
} else {
println!("{output}");
}
Ok(())
}
@ -1884,7 +2047,7 @@ async fn build_llm_provider(
let chain = noxa_llm::ProviderChain::default().await;
if chain.is_empty() {
return Err(
"no LLM providers available -- install the gemini CLI, start Ollama, or set OPENAI_API_KEY / ANTHROPIC_API_KEY"
"no LLM providers available (priority: Gemini CLI -> OpenAI -> Ollama -> Anthropic) -- install gemini on PATH, set OPENAI_API_KEY, OLLAMA_HOST / OLLAMA_MODEL, or ANTHROPIC_API_KEY"
.into(),
);
}
@ -1898,6 +2061,7 @@ async fn run_llm(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), Str
let provider = build_llm_provider(cli, resolved).await?;
let model = resolved.llm_model.as_deref();
let mut file_output: Option<(String, OutputFormat)> = None;
if let Some(ref schema_input) = cli.extract_json {
// Support @file syntax for loading schema from file
@ -1922,10 +2086,10 @@ async fn run_llm(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), Str
.map_err(|e| format!("LLM extraction failed: {e}"))?;
eprintln!("LLM: {:.1}s", t.elapsed().as_secs_f64());
println!(
"{}",
serde_json::to_string_pretty(&extracted).expect("serialization failed")
);
file_output = Some((
serde_json::to_string_pretty(&extracted).expect("serialization failed"),
OutputFormat::Json,
));
} else if let Some(ref prompt) = cli.extract_prompt {
let t = std::time::Instant::now();
let extracted = noxa_llm::extract::extract_with_prompt(
@ -1938,10 +2102,10 @@ async fn run_llm(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), Str
.map_err(|e| format!("LLM extraction failed: {e}"))?;
eprintln!("LLM: {:.1}s", t.elapsed().as_secs_f64());
println!(
"{}",
serde_json::to_string_pretty(&extracted).expect("serialization failed")
);
file_output = Some((
serde_json::to_string_pretty(&extracted).expect("serialization failed"),
OutputFormat::Json,
));
} else if let Some(sentences) = cli.summarize {
let t = std::time::Instant::now();
let summary = noxa_llm::summarize::summarize(
@ -1954,7 +2118,21 @@ async fn run_llm(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), Str
.map_err(|e| format!("LLM summarization failed: {e}"))?;
eprintln!("LLM: {:.1}s", t.elapsed().as_secs_f64());
println!("{summary}");
file_output = Some((summary, OutputFormat::Text));
}
if let Some((output_str, file_format)) = file_output {
if let Some(ref dir) = resolved.output_dir {
let url = cli
.urls
.first()
.map(|u| normalize_url(u))
.unwrap_or_default();
let filename = url_to_filename(&url, &file_format);
write_to_file(dir, &filename, &output_str)?;
} else {
println!("{output_str}");
}
}
Ok(())
@ -2067,11 +2245,16 @@ async fn run_batch_llm(
};
eprintln!("-> extracted {detail} ({:.1}s)", llm_elapsed.as_secs_f64());
if let Some(ref dir) = cli.output_dir {
if let Some(ref dir) = resolved.output_dir {
let file_format = if cli.summarize.is_some() {
OutputFormat::Text
} else {
OutputFormat::Json
};
let filename = custom_names
.get(url.as_str())
.map(|s| s.to_string())
.unwrap_or_else(|| url_to_filename(url, &OutputFormat::Json));
.unwrap_or_else(|| url_to_filename(url, &file_format));
write_to_file(dir, &filename, &output_str)?;
} else {
println!("--- {url}");
@ -2123,7 +2306,11 @@ fn has_llm_flags(cli: &Cli) -> bool {
cli.extract_json.is_some() || cli.extract_prompt.is_some() || cli.summarize.is_some()
}
async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
async fn run_research(
cli: &Cli,
resolved: &config::ResolvedConfig,
query: &str,
) -> Result<(), String> {
let api_key = cli
.api_key
.as_deref()
@ -2209,8 +2396,12 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
let filename = format!("research-{slug}.json");
let json = serde_json::to_string_pretty(&status_resp).unwrap_or_default();
std::fs::write(&filename, &json)
.map_err(|e| format!("failed to write {filename}: {e}"))?;
if let Some(ref dir) = resolved.output_dir {
write_to_file(dir, &filename, &json)?;
} else {
std::fs::write(&filename, &json)
.map_err(|e| format!("failed to write {filename}: {e}"))?;
}
let elapsed = status_resp
.get("elapsed_ms")
@ -2336,7 +2527,7 @@ async fn main() {
// --research: deep research via cloud API
if let Some(ref query) = cli.research {
if let Err(e) = run_research(&cli, query).await {
if let Err(e) = run_research(&cli, &resolved, query).await {
eprintln!("error: {e}");
process::exit(1);
}
@ -2377,10 +2568,7 @@ async fn main() {
}
// --raw-html: skip extraction, dump the fetched HTML
if resolved.raw_html
&& resolved.include_selectors.is_empty()
&& resolved.exclude_selectors.is_empty()
{
if resolved.raw_html {
match fetch_html(&cli, &resolved).await {
Ok(r) => println!("{}", r.html),
Err(e) => {
@ -2394,7 +2582,7 @@ async fn main() {
// Single-page extraction (handles both HTML and PDF via content-type detection)
match fetch_and_extract(&cli, &resolved).await {
Ok(FetchOutput::Local(result)) => {
if let Some(ref dir) = cli.output_dir {
if let Some(ref dir) = resolved.output_dir {
let url = cli
.urls
.first()
@ -2413,7 +2601,23 @@ async fn main() {
}
}
Ok(FetchOutput::Cloud(resp)) => {
print_cloud_output(&resp, &resolved.format);
if let Some(ref dir) = resolved.output_dir {
let url = cli
.urls
.first()
.map(|u| normalize_url(u))
.unwrap_or_default();
let custom_name = entries.first().and_then(|(_, name)| name.clone());
let filename =
custom_name.unwrap_or_else(|| url_to_filename(&url, &resolved.format));
let content = format_cloud_output(&resp, &resolved.format);
if let Err(e) = write_to_file(dir, &filename, &content) {
eprintln!("error: {e}");
process::exit(1);
}
} else {
print_cloud_output(&resp, &resolved.format);
}
}
Err(e) => {
eprintln!("{e}");

45
crates/noxa-core/src/llm/mod.rs
View file

@ -91,6 +91,51 @@ mod tests {
}
}
#[test]
fn strips_emphasis_from_body() {
let md = "# Hello\n\nThis is **bold** and this is *italic*. Also __underbold__ and _underitalic_.";
let result = make_result(md);
let out = to_llm_text(&result, None);
assert!(out.contains("This is bold and this is italic. Also underbold and underitalic."));
assert!(!out.contains("**"));
assert!(!out.contains("__"));
assert!(!out.contains("* italic *")); // regex shouldn't leave spaces usually but checking marker absence
assert!(!out.contains("_underitalic_"));
}
#[test]
fn dedups_repeated_phrases_in_line() {
let md = "Read more Read more Read more\n\nSome other text.";
let result = make_result(md);
let out = to_llm_text(&result, None);
assert!(out.contains("Read more"));
assert_eq!(out.matches("Read more").count(), 1);
}
#[test]
fn dedups_repeated_content_blocks() {
let md = "This is a block of text that is long enough to be deduped properly by the fingerprinting logic.\n\n\
This is a block of text that is long enough to be deduped properly by the fingerprinting logic.";
let result = make_result(md);
let out = to_llm_text(&result, None);
// Should only appear once
assert_eq!(out.matches("fingerprinting logic").count(), 1);
}
#[test]
fn dedups_near_duplicate_content_blocks() {
let md = "First ten words of this block should be unique enough for prefix matching.\n\n\
First ten words of this block should be unique enough for prefix matching but with extra text.";
let result = make_result(md);
let out = to_llm_text(&result, None);
// Near duplicate (same first 10 words) should be removed
assert_eq!(out.matches("First ten words").count(), 1);
}
#[test]
fn metadata_header_includes_populated_fields() {
let result = make_result("# Hello");

7
crates/noxa-fetch/src/client.rs
View file

@ -12,10 +12,10 @@ use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::time::{Duration, Instant};
use noxa_pdf::PdfMode;
use rand::seq::SliceRandom;
use tokio::sync::Semaphore;
use tracing::{debug, instrument, warn};
use noxa_pdf::PdfMode;
use crate::browser::{self, BrowserProfile, BrowserVariant};
use crate::error::FetchError;
@ -573,10 +573,7 @@ fn extract_homepage(url: &str) -> Option<String> {
}
/// Convert a noxa-pdf PdfResult into a noxa-core ExtractionResult.
fn pdf_to_extraction_result(
pdf: &noxa_pdf::PdfResult,
url: &str,
) -> noxa_core::ExtractionResult {
fn pdf_to_extraction_result(pdf: &noxa_pdf::PdfResult, url: &str) -> noxa_core::ExtractionResult {
let markdown = noxa_pdf::to_markdown(pdf);
let word_count = markdown.split_whitespace().count();

2
crates/noxa-fetch/src/lib.rs
View file

@ -17,6 +17,6 @@ pub use client::{BatchExtractResult, BatchResult, FetchClient, FetchConfig, Fetc
pub use crawler::{CrawlConfig, CrawlResult, CrawlState, Crawler, PageResult};
pub use error::FetchError;
pub use http::HeaderMap;
pub use noxa_pdf::PdfMode;
pub use proxy::{parse_proxy_file, parse_proxy_line};
pub use sitemap::SitemapEntry;
pub use noxa_pdf::PdfMode;

2
crates/noxa-fetch/src/linkedin.rs
View file

@ -1,3 +1,4 @@
use noxa_core::{Content, ExtractionResult, Metadata};
/// LinkedIn post extraction from authenticated HTML.
///
/// LinkedIn's SPA stores all data in `<code>` tags as HTML-escaped JSON.
@ -5,7 +6,6 @@
/// Profile, etc. We parse these to reconstruct post + comments as markdown.
use serde_json::Value;
use tracing::debug;
use noxa_core::{Content, ExtractionResult, Metadata};
/// Check if a URL is a LinkedIn post/activity.
pub fn is_linkedin_post(url: &str) -> bool {

2
crates/noxa-fetch/src/reddit.rs
View file

@ -1,3 +1,4 @@
use noxa_core::{Content, ExtractionResult, Metadata};
/// Reddit JSON API fallback for extracting posts + comments without JS rendering.
///
/// Reddit's new `shreddit` frontend only SSRs the post body — comments are
@ -5,7 +6,6 @@
/// comment tree as structured JSON, which we convert to clean markdown.
use serde::Deserialize;
use tracing::debug;
use noxa_core::{Content, ExtractionResult, Metadata};
/// Check if a URL points to a Reddit post/comment page.
pub fn is_reddit_url(url: &str) -> bool {

12
crates/noxa-llm/src/chain.rs
View file

@ -1,5 +1,5 @@
/// Provider chain — tries providers in order until one succeeds.
/// Default order: Ollama (local, free) -> OpenAI -> Anthropic.
/// Default order: Gemini CLI (primary) -> OpenAI -> Ollama -> Anthropic.
/// Only includes providers that are actually configured/available.
use async_trait::async_trait;
use tracing::{debug, info, warn};
@ -7,9 +7,7 @@ use tracing::{debug, info, warn};
use crate::error::LlmError;
use crate::provider::{CompletionRequest, LlmProvider};
use crate::providers::{
anthropic::AnthropicProvider,
gemini_cli::GeminiCliProvider,
ollama::OllamaProvider,
anthropic::AnthropicProvider, gemini_cli::GeminiCliProvider, ollama::OllamaProvider,
openai::OpenAiProvider,
};
@ -94,7 +92,11 @@ impl LlmProvider for ProviderChain {
let t = std::time::Instant::now();
match provider.complete(request).await {
Ok(response) => {
info!(provider = provider.name(), elapsed_ms = t.elapsed().as_millis(), "completion succeeded");
info!(
provider = provider.name(),
elapsed_ms = t.elapsed().as_millis(),
"completion succeeded"
);
return Ok(response);
}
Err(e) => {

152
crates/noxa-llm/src/extract.rs
View file

@ -8,44 +8,47 @@ use crate::provider::{CompletionRequest, LlmProvider, Message};
/// Validate a JSON value against a schema. Returns Ok(()) on success or
/// Err(LlmError::InvalidJson) with a concise error message on failure.
fn validate_schema(
value: &serde_json::Value,
schema: &serde_json::Value,
) -> Result<(), LlmError> {
let compiled = jsonschema::validator_for(schema).map_err(|e| {
LlmError::InvalidJson(format!("invalid schema: {e}"))
})?;
fn validate_schema(value: &serde_json::Value, schema: &serde_json::Value) -> Result<(), LlmError> {
let compiled = jsonschema::validator_for(schema)
.map_err(|e| LlmError::InvalidJson(format!("invalid schema: {e}")))?;
let errors: Vec<String> = compiled
.iter_errors(value)
.map(|e| format!("{} at {}", e, e.instance_path()))
.collect();
let first_error = compiled.iter_errors(value).next();
if errors.is_empty() {
Ok(())
} else {
Err(LlmError::InvalidJson(format!(
"schema validation failed: {}",
errors.join("; ")
)))
match first_error {
None => Ok(()),
Some(e) => {
let msg = format!("{} at {}", e, e.instance_path());
Err(LlmError::InvalidJson(format!(
"schema validation failed: {msg}"
)))
}
}
}
/// Compile a schema up front so invalid schemas fail before any provider call.
fn validate_schema_definition(schema: &serde_json::Value) -> Result<(), LlmError> {
jsonschema::validator_for(schema)
.map(|_| ())
.map_err(|e| LlmError::InvalidJson(format!("invalid schema: {e}")))
}
/// Extract structured JSON from content using a JSON schema.
/// The schema tells the LLM exactly what fields to extract and their types.
///
/// Retry policy:
/// - If the response cannot be parsed as JSON at all: retry once with the
/// identical request (handles transient formatting issues).
/// - If the response is valid JSON but fails schema validation: return
/// `LlmError::InvalidJson` immediately — the schema is likely unsatisfiable
/// for this content, so retrying would produce the same result.
/// - If the response cannot be parsed as JSON: retry once with a correction prompt.
/// - If the response is valid JSON but fails schema validation: retry once with
/// a tighter correction prompt that includes the specific validation error.
/// - Both retry attempts add the previous failed response as an 'assistant' message
/// and the correction instructions as a 'user' message to improve success.
pub async fn extract_json(
content: &str,
schema: &serde_json::Value,
provider: &dyn LlmProvider,
model: Option<&str>,
) -> Result<serde_json::Value, LlmError> {
validate_schema_definition(schema)?;
let system = format!(
"You are a JSON extraction engine. Extract data from the content according to this schema.\n\
Return ONLY valid JSON matching the schema. No explanations, no markdown, no commentary.\n\n\
@ -53,18 +56,20 @@ pub async fn extract_json(
serde_json::to_string_pretty(schema).unwrap_or_else(|_| schema.to_string())
);
let request = CompletionRequest {
let mut messages = vec![
Message {
role: "system".into(),
content: system,
},
Message {
role: "user".into(),
content: content.to_string(),
},
];
let mut request = CompletionRequest {
model: model.unwrap_or_default().to_string(),
messages: vec![
Message {
role: "system".into(),
content: system,
},
Message {
role: "user".into(),
content: content.to_string(),
},
],
messages: messages.clone(),
temperature: Some(0.0),
max_tokens: None,
json_mode: true,
@ -72,23 +77,54 @@ pub async fn extract_json(
let response = provider.complete(&request).await?;
match parse_json_response(&response) {
Ok(value) => {
// Valid JSON — now validate against the schema.
// Schema mismatches do not retry (unsatisfiable → same result).
validate_schema(&value, schema)?;
Ok(value)
}
Err(_parse_err) => {
// Unparseable JSON — retry once with the identical request.
match parse_and_validate(&response, schema) {
Ok(value) => Ok(value),
Err(e) => {
// First attempt failed — retry once with a correction prompt.
// Construct a concise correction prompt based on the error type.
let correction_prompt = match &e {
LlmError::InvalidJson(msg) if msg.contains("schema validation failed") => {
let error_msg = msg.replace("schema validation failed: ", "");
format!("Correction required: {}. Return ONLY the corrected JSON.", error_msg)
}
_ => {
"Your response was not valid JSON. Please return ONLY valid JSON matching the schema.".to_string()
}
};
// Limit correction context to prevent token blowup on large hallucinated outputs.
let capped_response = if response.len() > 2000 {
format!("{}... [truncated]", &response[..2000])
} else {
response.clone()
};
messages.push(Message {
role: "assistant".into(),
content: capped_response,
});
messages.push(Message {
role: "user".into(),
content: correction_prompt,
});
request.messages = messages;
let retry_response = provider.complete(&request).await?;
let value = parse_json_response(&retry_response)?;
validate_schema(&value, schema)?;
Ok(value)
parse_and_validate(&retry_response, schema)
}
}
}
/// Helper: parse response string as JSON and validate it against the schema.
fn parse_and_validate(
response: &str,
schema: &serde_json::Value,
) -> Result<serde_json::Value, LlmError> {
let value = parse_json_response(response)?;
validate_schema(&value, schema)?;
Ok(value)
}
/// Extract information using a natural language prompt.
/// More flexible than schema extraction — the user describes what they want.
pub async fn extract_with_prompt(
@ -301,9 +337,7 @@ mod tests {
],
);
let result = extract_json("content", &schema, &mock, None)
.await
.unwrap();
let result = extract_json("content", &schema, &mock, None).await.unwrap();
assert_eq!(result["title"], "Retry succeeded");
}
@ -318,10 +352,7 @@ mod tests {
let mock = SequenceMockProvider::new(
"mock-seq",
vec![
Ok("not json".to_string()),
Ok("also not json".to_string()),
],
vec![Ok("not json".to_string()), Ok("also not json".to_string())],
);
let result = extract_json("content", &schema, &mock, None).await;
@ -332,7 +363,7 @@ mod tests {
}
#[tokio::test]
async fn schema_mismatch_does_not_retry() {
async fn schema_mismatch_triggers_retry() {
use crate::testing::mock::SequenceMockProvider;
let schema = serde_json::json!({
@ -343,20 +374,17 @@ mod tests {
}
});
// Both calls return valid JSON with wrong schema — but only one call should happen.
// First call: valid JSON but schema mismatch (price is string).
// Second call: valid JSON matching schema.
let mock = SequenceMockProvider::new(
"mock-seq",
vec![
Ok(r#"{"price": "wrong-type"}"#.to_string()),
Ok(r#"{"price": 9.99}"#.to_string()), // would succeed — but shouldn't be called
Ok(r#"{"price": 9.99}"#.to_string()),
],
);
// Should return InvalidJson without calling second response.
let result = extract_json("content", &schema, &mock, None).await;
assert!(
matches!(result, Err(LlmError::InvalidJson(_))),
"schema mismatch should not trigger retry"
);
let result = extract_json("content", &schema, &mock, None).await.unwrap();
assert_eq!(result["price"], 9.99);
}
}

2
crates/noxa-llm/src/lib.rs
View file

@ -2,7 +2,7 @@
///
/// Provider chain: Gemini CLI (primary) → OpenAI → Ollama → Anthropic.
/// Gemini CLI requires the `gemini` binary on PATH; GEMINI_MODEL env var sets the model.
/// Provides schema-validated extraction (with one retry on parse failure),
/// Provides schema-validated extraction (with one retry on parse or schema mismatch),
/// prompt extraction, and summarization on top of noxa-core's content pipeline.
pub mod chain;
pub mod clean;

26
crates/noxa-llm/src/providers/gemini_cli.rs
View file

@ -12,11 +12,11 @@
///
/// Two flags reduce this:
/// - `--extensions ""` — skips extension loading (~3 s saved)
/// - `current_dir` set to a temp workdir containing `.gemini/settings.json` with
/// `{"mcpServers":{}}` — workspace settings override user settings, so all 6 MCP
/// - `current_dir` set to a best-effort temp workdir containing `.gemini/settings.json`
/// with `{"mcpServers":{}}` — workspace settings override user settings, so all 6 MCP
/// servers are skipped at subprocess startup (major speedup).
///
/// The workdir is created once at construction and reused for every call.
/// The workdir is created once at construction and reused for every call when available.
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
@ -36,10 +36,6 @@ const MAX_CONCURRENT: usize = 6;
/// Subprocess deadline — prevents hung `gemini` processes blocking the chain.
const SUBPROCESS_TIMEOUT: Duration = Duration::from_secs(60);
/// Fixed workdir used for every subprocess call.
/// A workspace-level `.gemini/settings.json` here overrides the user's MCP server config.
const NOXA_GEMINI_WORKDIR: &str = "/tmp/noxa-gemini";
pub struct GeminiCliProvider {
default_model: String,
semaphore: Arc<Semaphore>,
@ -56,7 +52,7 @@ impl GeminiCliProvider {
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "gemini-2.5-pro".into());
let workdir = PathBuf::from(NOXA_GEMINI_WORKDIR);
let workdir = std::env::temp_dir().join("noxa-gemini");
ensure_gemini_workdir(&workdir);
Self {
@ -106,11 +102,14 @@ impl LlmProvider for GeminiCliProvider {
// Workspace settings in self.workdir override the user's ~/.gemini/settings.json,
// replacing the user's MCP server list with {} so none are spawned at startup.
// Without this, each of the user's MCP servers adds latency to every call.
cmd.current_dir(&self.workdir);
if self.workdir.is_dir() {
cmd.current_dir(&self.workdir);
}
cmd.stdin(std::process::Stdio::null());
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
cmd.kill_on_drop(true);
debug!(model, workdir = %self.workdir.display(), "spawning gemini subprocess");
@ -169,7 +168,9 @@ fn extract_response_from_output(stdout: &str) -> Result<String, LlmError> {
let json_str = &stdout[json_start..];
let outer: serde_json::Value = serde_json::from_str(json_str).map_err(|e| {
let preview = &json_str[..json_str.len().min(300)];
LlmError::ProviderError(format!("failed to parse gemini JSON output: {e} — {preview}"))
LlmError::ProviderError(format!(
"failed to parse gemini JSON output: {e} — {preview}"
))
})?;
// `response` holds the model's actual text output.
@ -320,10 +321,7 @@ mod tests {
fn extracts_response_skipping_mcp_noise() {
// MCP warning line appears before the JSON object in real gemini output.
let stdout = "MCP issues detected. Run /mcp list for status.\n{\"session_id\":\"abc\",\"response\":\"the answer\",\"stats\":{}}";
assert_eq!(
extract_response_from_output(stdout).unwrap(),
"the answer"
);
assert_eq!(extract_response_from_output(stdout).unwrap(), "the answer");
}
#[test]

5
crates/noxa-llm/src/providers/mod.rs
View file

@ -29,9 +29,6 @@ mod tests {
#[test]
fn none_override_with_no_env_returns_none() {
assert_eq!(
load_api_key(None, "NOXA_TEST_NONEXISTENT_KEY_12345"),
None
);
assert_eq!(load_api_key(None, "NOXA_TEST_NONEXISTENT_KEY_12345"), None);
}
}

39
crates/noxa-llm/src/providers/ollama.rs
View file

@ -8,6 +8,8 @@ use crate::clean::strip_thinking_tags;
use crate::error::LlmError;
use crate::provider::{CompletionRequest, LlmProvider};
const DEFAULT_HEALTH_TIMEOUT_MS: u64 = 2_000;
pub struct OllamaProvider {
client: reqwest::Client,
base_url: String,
@ -22,7 +24,7 @@ impl OllamaProvider {
let default_model = model
.or_else(|| std::env::var("OLLAMA_MODEL").ok())
.unwrap_or_else(|| "qwen3:8b".into());
.unwrap_or_else(|| "qwen3.5:9b".into());
Self {
client: reqwest::Client::new(),
@ -98,7 +100,7 @@ impl LlmProvider for OllamaProvider {
async fn is_available(&self) -> bool {
let url = format!("{}/api/tags", self.base_url);
matches!(
tokio::time::timeout(Duration::from_millis(500), self.client.get(&url).send()).await,
tokio::time::timeout(health_timeout(), self.client.get(&url).send()).await,
Ok(Ok(r)) if r.status().is_success()
)
}
@ -108,6 +110,18 @@ impl LlmProvider for OllamaProvider {
}
}
fn health_timeout() -> Duration {
health_timeout_from_env(std::env::var("OLLAMA_HEALTH_TIMEOUT_MS").ok())
}
fn health_timeout_from_env(value: Option<String>) -> Duration {
value
.and_then(|v| v.parse::<u64>().ok())
.filter(|ms| *ms > 0)
.map(Duration::from_millis)
.unwrap_or_else(|| Duration::from_millis(DEFAULT_HEALTH_TIMEOUT_MS))
}
#[cfg(test)]
mod tests {
use super::*;
@ -142,6 +156,27 @@ mod tests {
assert_eq!(provider.default_model(), "phi3:mini");
}
#[test]
fn health_timeout_from_env_defaults_when_unset() {
assert_eq!(health_timeout_from_env(None), Duration::from_millis(2000));
}
#[test]
fn health_timeout_from_env_parses_override() {
assert_eq!(
health_timeout_from_env(Some("1500".into())),
Duration::from_millis(1500)
);
}
#[test]
fn health_timeout_from_env_ignores_invalid_values() {
assert_eq!(
health_timeout_from_env(Some("not-a-number".into())),
Duration::from_millis(2000)
);
}
// Env var fallback is a trivial `env::var().ok()` -- not worth the flakiness
// of manipulating process-global state. Run in isolation if needed:
// cargo test -p noxa-llm env_var_fallback -- --ignored --test-threads=1

13
crates/noxa-llm/src/testing.rs
View file

@ -4,8 +4,8 @@
/// extract, chain, and other modules that need a fake LLM backend.
#[cfg(test)]
pub(crate) mod mock {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use async_trait::async_trait;
@ -50,7 +50,7 @@ pub(crate) mod mock {
}
/// A mock provider that returns responses from a sequence.
/// Call N → returns responses[N], wrapping at the end.
/// Call N → returns responses[N], clamping to the final response.
/// Useful for testing first-failure / second-success retry paths.
pub struct SequenceMockProvider {
pub name: &'static str,
@ -60,10 +60,11 @@ pub(crate) mod mock {
}
impl SequenceMockProvider {
pub fn new(
name: &'static str,
responses: Vec<Result<String, String>>,
) -> Self {
pub fn new(name: &'static str, responses: Vec<Result<String, String>>) -> Self {
assert!(
!responses.is_empty(),
"SequenceMockProvider requires at least one response"
);
Self {
name,
responses,

1
crates/noxa-mcp/src/cloud.rs
View file

@ -7,7 +7,6 @@ use std::time::Duration;
use serde_json::{Value, json};
use tracing::info;
const API_BASE: &str = "https://api.noxa.io/v1";
/// Lightweight client for the noxa cloud API.

20
crates/noxa-mcp/src/server.rs
View file

@ -18,6 +18,8 @@ use url::Url;
use crate::cloud::{self, CloudClient, SmartFetchResult};
use crate::tools::*;
const NO_LLM_PROVIDERS_MESSAGE: &str = "No LLM providers available (priority: Gemini CLI -> OpenAI -> Ollama -> Anthropic). Install gemini on PATH, set OPENAI_API_KEY, OLLAMA_HOST / OLLAMA_MODEL, or ANTHROPIC_API_KEY, or set NOXA_API_KEY for cloud fallback.";
pub struct NoxaMcp {
tool_router: ToolRouter<Self>,
fetch_client: Arc<noxa_fetch::FetchClient>,
@ -89,7 +91,7 @@ impl NoxaMcp {
let chain = noxa_llm::ProviderChain::default().await;
let llm_chain = if chain.is_empty() {
warn!("no LLM providers available (gemini CLI, OPENAI_API_KEY, ANTHROPIC_API_KEY) -- extract/summarize tools will fail");
warn!("{NO_LLM_PROVIDERS_MESSAGE} -- extract/summarize tools will fail");
None
} else {
info!(providers = chain.len(), "LLM provider chain ready");
@ -333,9 +335,7 @@ impl NoxaMcp {
// No local LLM — fall back to cloud API directly
if self.llm_chain.is_none() {
let cloud = self.cloud.as_ref().ok_or(
"No LLM providers available. Install the gemini CLI, set OPENAI_API_KEY, ANTHROPIC_API_KEY, or NOXA_API_KEY for cloud fallback.",
)?;
let cloud = self.cloud.as_ref().ok_or(NO_LLM_PROVIDERS_MESSAGE)?;
let mut body = json!({"url": params.url});
if let Some(ref schema) = params.schema {
body["schema"] = json!(schema);
@ -386,9 +386,7 @@ impl NoxaMcp {
// No local LLM — fall back to cloud API directly
if self.llm_chain.is_none() {
let cloud = self.cloud.as_ref().ok_or(
"No LLM providers available. Install the gemini CLI, set OPENAI_API_KEY, ANTHROPIC_API_KEY, or NOXA_API_KEY for cloud fallback.",
)?;
let cloud = self.cloud.as_ref().ok_or(NO_LLM_PROVIDERS_MESSAGE)?;
let mut body = json!({"url": params.url});
if let Some(sentences) = params.max_sentences {
body["max_sentences"] = json!(sentences);
@ -425,9 +423,8 @@ impl NoxaMcp {
#[tool]
async fn diff(&self, Parameters(params): Parameters<DiffParams>) -> Result<String, String> {
validate_url(&params.url)?;
let previous: noxa_core::ExtractionResult =
serde_json::from_str(&params.previous_snapshot)
.map_err(|e| format!("Failed to parse previous_snapshot JSON: {e}"))?;
let previous: noxa_core::ExtractionResult = serde_json::from_str(&params.previous_snapshot)
.map_err(|e| format!("Failed to parse previous_snapshot JSON: {e}"))?;
let result = cloud::smart_fetch(
&self.fetch_client,
@ -515,8 +512,7 @@ impl NoxaMcp {
}
}
let identity =
noxa_core::brand::extract_brand(&fetch_result.html, Some(&fetch_result.url));
let identity = noxa_core::brand::extract_brand(&fetch_result.html, Some(&fetch_result.url));
Ok(serde_json::to_string_pretty(&identity).unwrap_or_default())
}