mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-21 07:01:01 +02:00
refactor: add noxa mcp subcommand
This commit is contained in:
parent
251979edfe
commit
a25103667e
8 changed files with 66 additions and 22 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1753,6 +1753,7 @@ dependencies = [
|
|||
"noxa-core",
|
||||
"noxa-fetch",
|
||||
"noxa-llm",
|
||||
"noxa-mcp",
|
||||
"noxa-pdf",
|
||||
"rand 0.8.5",
|
||||
"regex",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ noxa-core = { path = "crates/noxa-core" }
|
|||
noxa-fetch = { path = "crates/noxa-fetch" }
|
||||
noxa-llm = { path = "crates/noxa-llm" }
|
||||
noxa-pdf = { path = "crates/noxa-pdf" }
|
||||
noxa-mcp = { path = "crates/noxa-mcp" }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
|
@ -21,3 +22,6 @@ tracing = "0.1"
|
|||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
dotenvy = "0.15"
|
||||
rmcp = { version = "1.2", features = ["server", "macros", "transport-io", "schemars"] }
|
||||
schemars = "1.0"
|
||||
dirs = "6.0.0"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ noxa-core = { workspace = true }
|
|||
noxa-fetch = { workspace = true }
|
||||
noxa-llm = { workspace = true }
|
||||
noxa-pdf = { workspace = true }
|
||||
noxa-mcp = { workspace = true }
|
||||
dotenvy = { workspace = true }
|
||||
rand = "0.8"
|
||||
serde_json = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@ use std::sync::Arc;
|
|||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use clap::{CommandFactory, FromArgMatches, Parser, ValueEnum};
|
||||
use serde::Deserialize;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use noxa_core::{
|
||||
ChangeStatus, ContentDiff, ExtractionOptions, ExtractionResult, Metadata, extract_with_options,
|
||||
to_llm_text,
|
||||
|
|
@ -22,7 +20,10 @@ use noxa_fetch::{
|
|||
FetchConfig, FetchResult, PageResult, SitemapEntry,
|
||||
};
|
||||
use noxa_llm::LlmProvider;
|
||||
use noxa_mcp;
|
||||
use noxa_pdf::PdfMode;
|
||||
use serde::Deserialize;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
/// Known anti-bot challenge page titles (case-insensitive prefix match).
|
||||
const ANTIBOT_TITLES: &[&str] = &[
|
||||
|
|
@ -347,6 +348,15 @@ fn init_logging(verbose: bool) {
|
|||
tracing_subscriber::fmt().with_env_filter(filter).init();
|
||||
}
|
||||
|
||||
fn init_mcp_logging() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||
.with_writer(std::io::stderr)
|
||||
.with_ansi(false)
|
||||
.try_init()
|
||||
.ok();
|
||||
}
|
||||
|
||||
/// Build FetchConfig from CLI flags.
|
||||
///
|
||||
/// `--proxy` sets a single static proxy (no rotation).
|
||||
|
|
@ -655,8 +665,7 @@ async fn fetch_and_extract(
|
|||
|
||||
// --cloud: skip local, go straight to cloud API
|
||||
if cli.cloud {
|
||||
let c =
|
||||
cloud_client.ok_or("--cloud requires NOXA_API_KEY (set via env or --api-key)")?;
|
||||
let c = cloud_client.ok_or("--cloud requires NOXA_API_KEY (set via env or --api-key)")?;
|
||||
let options = build_extraction_options(resolved);
|
||||
let format_str = match resolved.format {
|
||||
OutputFormat::Markdown => "markdown",
|
||||
|
|
@ -2253,6 +2262,16 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
|
|||
async fn main() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
if matches!(std::env::args().nth(1).as_deref(), Some("mcp")) {
|
||||
init_mcp_logging();
|
||||
|
||||
if let Err(e) = noxa_mcp::run().await {
|
||||
eprintln!("error: {e}");
|
||||
process::exit(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Use low-level API to get both typed Cli and ArgMatches for ValueSource detection.
|
||||
let matches = Cli::command().get_matches();
|
||||
let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit());
|
||||
|
|
@ -2358,7 +2377,10 @@ async fn main() {
|
|||
}
|
||||
|
||||
// --raw-html: skip extraction, dump the fetched HTML
|
||||
if resolved.raw_html && resolved.include_selectors.is_empty() && resolved.exclude_selectors.is_empty() {
|
||||
if resolved.raw_html
|
||||
&& resolved.include_selectors.is_empty()
|
||||
&& resolved.exclude_selectors.is_empty()
|
||||
{
|
||||
match fetch_html(&cli, &resolved).await {
|
||||
Ok(r) => println!("{}", r.html),
|
||||
Err(e) => {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
/// First choice in the provider chain: free, private, fast on Apple Silicon.
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::clean::strip_thinking_tags;
|
||||
use crate::error::LlmError;
|
||||
|
|
@ -96,7 +97,10 @@ impl LlmProvider for OllamaProvider {
|
|||
|
||||
async fn is_available(&self) -> bool {
|
||||
let url = format!("{}/api/tags", self.base_url);
|
||||
matches!(self.client.get(&url).send().await, Ok(r) if r.status().is_success())
|
||||
matches!(
|
||||
tokio::time::timeout(Duration::from_millis(500), self.client.get(&url).send()).await,
|
||||
Ok(Ok(r)) if r.status().is_success()
|
||||
)
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@ version.workspace = true
|
|||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "noxa_mcp"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "noxa-mcp"
|
||||
path = "src/main.rs"
|
||||
|
|
@ -14,8 +18,8 @@ noxa-core = { workspace = true }
|
|||
noxa-fetch = { workspace = true }
|
||||
noxa-llm = { workspace = true }
|
||||
noxa-pdf = { workspace = true }
|
||||
rmcp = { version = "1.2", features = ["server", "macros", "transport-io", "schemars"] }
|
||||
schemars = "1.0"
|
||||
rmcp = { workspace = true }
|
||||
schemars = { workspace = true }
|
||||
dotenvy = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
|
@ -24,4 +28,4 @@ tracing = { workspace = true }
|
|||
tracing-subscriber = { workspace = true }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
url = "2"
|
||||
dirs = "6.0.0"
|
||||
dirs = { workspace = true }
|
||||
|
|
|
|||
20
crates/noxa-mcp/src/lib.rs
Normal file
20
crates/noxa-mcp/src/lib.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
/// noxa-mcp library wrapper.
|
||||
///
|
||||
/// This exposes the MCP server so it can be embedded by the `noxa` CLI via
|
||||
/// `noxa mcp` without duplicating the transport/bootstrap code.
|
||||
///
|
||||
/// Callers must initialize tracing before calling `run()`. Stdout must remain
|
||||
/// untouched after `run()` begins because it carries the MCP wire protocol.
|
||||
pub(crate) mod cloud;
|
||||
pub(crate) mod server;
|
||||
pub(crate) mod tools;
|
||||
|
||||
use rmcp::ServiceExt;
|
||||
use rmcp::transport::stdio;
|
||||
|
||||
/// Start the MCP server over stdio and block until the client disconnects.
|
||||
pub async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let service = server::NoxaMcp::new().await.serve(stdio()).await?;
|
||||
service.waiting().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1,15 +1,6 @@
|
|||
/// noxa-mcp: MCP (Model Context Protocol) server for noxa.
|
||||
/// Exposes web extraction tools over stdio transport for AI agents
|
||||
/// like Claude Desktop, Claude Code, and other MCP clients.
|
||||
mod cloud;
|
||||
mod server;
|
||||
mod tools;
|
||||
|
||||
use rmcp::ServiceExt;
|
||||
use rmcp::transport::stdio;
|
||||
|
||||
use server::NoxaMcp;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
dotenvy::dotenv().ok();
|
||||
|
|
@ -21,8 +12,5 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
.with_ansi(false)
|
||||
.init();
|
||||
|
||||
let service = NoxaMcp::new().await.serve(stdio()).await?;
|
||||
|
||||
service.waiting().await?;
|
||||
Ok(())
|
||||
noxa_mcp::run().await
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue