mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-23 07:21:02 +02:00
refactor: add noxa mcp subcommand
This commit is contained in:
parent
251979edfe
commit
a25103667e
8 changed files with 66 additions and 22 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1753,6 +1753,7 @@ dependencies = [
|
||||||
"noxa-core",
|
"noxa-core",
|
||||||
"noxa-fetch",
|
"noxa-fetch",
|
||||||
"noxa-llm",
|
"noxa-llm",
|
||||||
|
"noxa-mcp",
|
||||||
"noxa-pdf",
|
"noxa-pdf",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"regex",
|
"regex",
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ noxa-core = { path = "crates/noxa-core" }
|
||||||
noxa-fetch = { path = "crates/noxa-fetch" }
|
noxa-fetch = { path = "crates/noxa-fetch" }
|
||||||
noxa-llm = { path = "crates/noxa-llm" }
|
noxa-llm = { path = "crates/noxa-llm" }
|
||||||
noxa-pdf = { path = "crates/noxa-pdf" }
|
noxa-pdf = { path = "crates/noxa-pdf" }
|
||||||
|
noxa-mcp = { path = "crates/noxa-mcp" }
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
@ -21,3 +22,6 @@ tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
clap = { version = "4", features = ["derive", "env"] }
|
clap = { version = "4", features = ["derive", "env"] }
|
||||||
dotenvy = "0.15"
|
dotenvy = "0.15"
|
||||||
|
rmcp = { version = "1.2", features = ["server", "macros", "transport-io", "schemars"] }
|
||||||
|
schemars = "1.0"
|
||||||
|
dirs = "6.0.0"
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ noxa-core = { workspace = true }
|
||||||
noxa-fetch = { workspace = true }
|
noxa-fetch = { workspace = true }
|
||||||
noxa-llm = { workspace = true }
|
noxa-llm = { workspace = true }
|
||||||
noxa-pdf = { workspace = true }
|
noxa-pdf = { workspace = true }
|
||||||
|
noxa-mcp = { workspace = true }
|
||||||
dotenvy = { workspace = true }
|
dotenvy = { workspace = true }
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,6 @@ use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
use clap::{CommandFactory, FromArgMatches, Parser, ValueEnum};
|
use clap::{CommandFactory, FromArgMatches, Parser, ValueEnum};
|
||||||
use serde::Deserialize;
|
|
||||||
use tracing_subscriber::EnvFilter;
|
|
||||||
use noxa_core::{
|
use noxa_core::{
|
||||||
ChangeStatus, ContentDiff, ExtractionOptions, ExtractionResult, Metadata, extract_with_options,
|
ChangeStatus, ContentDiff, ExtractionOptions, ExtractionResult, Metadata, extract_with_options,
|
||||||
to_llm_text,
|
to_llm_text,
|
||||||
|
|
@ -22,7 +20,10 @@ use noxa_fetch::{
|
||||||
FetchConfig, FetchResult, PageResult, SitemapEntry,
|
FetchConfig, FetchResult, PageResult, SitemapEntry,
|
||||||
};
|
};
|
||||||
use noxa_llm::LlmProvider;
|
use noxa_llm::LlmProvider;
|
||||||
|
use noxa_mcp;
|
||||||
use noxa_pdf::PdfMode;
|
use noxa_pdf::PdfMode;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use tracing_subscriber::EnvFilter;
|
||||||
|
|
||||||
/// Known anti-bot challenge page titles (case-insensitive prefix match).
|
/// Known anti-bot challenge page titles (case-insensitive prefix match).
|
||||||
const ANTIBOT_TITLES: &[&str] = &[
|
const ANTIBOT_TITLES: &[&str] = &[
|
||||||
|
|
@ -347,6 +348,15 @@ fn init_logging(verbose: bool) {
|
||||||
tracing_subscriber::fmt().with_env_filter(filter).init();
|
tracing_subscriber::fmt().with_env_filter(filter).init();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn init_mcp_logging() {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_ansi(false)
|
||||||
|
.try_init()
|
||||||
|
.ok();
|
||||||
|
}
|
||||||
|
|
||||||
/// Build FetchConfig from CLI flags.
|
/// Build FetchConfig from CLI flags.
|
||||||
///
|
///
|
||||||
/// `--proxy` sets a single static proxy (no rotation).
|
/// `--proxy` sets a single static proxy (no rotation).
|
||||||
|
|
@ -655,8 +665,7 @@ async fn fetch_and_extract(
|
||||||
|
|
||||||
// --cloud: skip local, go straight to cloud API
|
// --cloud: skip local, go straight to cloud API
|
||||||
if cli.cloud {
|
if cli.cloud {
|
||||||
let c =
|
let c = cloud_client.ok_or("--cloud requires NOXA_API_KEY (set via env or --api-key)")?;
|
||||||
cloud_client.ok_or("--cloud requires NOXA_API_KEY (set via env or --api-key)")?;
|
|
||||||
let options = build_extraction_options(resolved);
|
let options = build_extraction_options(resolved);
|
||||||
let format_str = match resolved.format {
|
let format_str = match resolved.format {
|
||||||
OutputFormat::Markdown => "markdown",
|
OutputFormat::Markdown => "markdown",
|
||||||
|
|
@ -2253,6 +2262,16 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
|
||||||
async fn main() {
|
async fn main() {
|
||||||
dotenvy::dotenv().ok();
|
dotenvy::dotenv().ok();
|
||||||
|
|
||||||
|
if matches!(std::env::args().nth(1).as_deref(), Some("mcp")) {
|
||||||
|
init_mcp_logging();
|
||||||
|
|
||||||
|
if let Err(e) = noxa_mcp::run().await {
|
||||||
|
eprintln!("error: {e}");
|
||||||
|
process::exit(1);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Use low-level API to get both typed Cli and ArgMatches for ValueSource detection.
|
// Use low-level API to get both typed Cli and ArgMatches for ValueSource detection.
|
||||||
let matches = Cli::command().get_matches();
|
let matches = Cli::command().get_matches();
|
||||||
let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit());
|
let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit());
|
||||||
|
|
@ -2358,7 +2377,10 @@ async fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// --raw-html: skip extraction, dump the fetched HTML
|
// --raw-html: skip extraction, dump the fetched HTML
|
||||||
if resolved.raw_html && resolved.include_selectors.is_empty() && resolved.exclude_selectors.is_empty() {
|
if resolved.raw_html
|
||||||
|
&& resolved.include_selectors.is_empty()
|
||||||
|
&& resolved.exclude_selectors.is_empty()
|
||||||
|
{
|
||||||
match fetch_html(&cli, &resolved).await {
|
match fetch_html(&cli, &resolved).await {
|
||||||
Ok(r) => println!("{}", r.html),
|
Ok(r) => println!("{}", r.html),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
/// First choice in the provider chain: free, private, fast on Apple Silicon.
|
/// First choice in the provider chain: free, private, fast on Apple Silicon.
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::clean::strip_thinking_tags;
|
use crate::clean::strip_thinking_tags;
|
||||||
use crate::error::LlmError;
|
use crate::error::LlmError;
|
||||||
|
|
@ -96,7 +97,10 @@ impl LlmProvider for OllamaProvider {
|
||||||
|
|
||||||
async fn is_available(&self) -> bool {
|
async fn is_available(&self) -> bool {
|
||||||
let url = format!("{}/api/tags", self.base_url);
|
let url = format!("{}/api/tags", self.base_url);
|
||||||
matches!(self.client.get(&url).send().await, Ok(r) if r.status().is_success())
|
matches!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(500), self.client.get(&url).send()).await,
|
||||||
|
Ok(Ok(r)) if r.status().is_success()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn name(&self) -> &str {
|
fn name(&self) -> &str {
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,10 @@ version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
name = "noxa_mcp"
|
||||||
|
path = "src/lib.rs"
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "noxa-mcp"
|
name = "noxa-mcp"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
@ -14,8 +18,8 @@ noxa-core = { workspace = true }
|
||||||
noxa-fetch = { workspace = true }
|
noxa-fetch = { workspace = true }
|
||||||
noxa-llm = { workspace = true }
|
noxa-llm = { workspace = true }
|
||||||
noxa-pdf = { workspace = true }
|
noxa-pdf = { workspace = true }
|
||||||
rmcp = { version = "1.2", features = ["server", "macros", "transport-io", "schemars"] }
|
rmcp = { workspace = true }
|
||||||
schemars = "1.0"
|
schemars = { workspace = true }
|
||||||
dotenvy = { workspace = true }
|
dotenvy = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
|
@ -24,4 +28,4 @@ tracing = { workspace = true }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||||
url = "2"
|
url = "2"
|
||||||
dirs = "6.0.0"
|
dirs = { workspace = true }
|
||||||
|
|
|
||||||
20
crates/noxa-mcp/src/lib.rs
Normal file
20
crates/noxa-mcp/src/lib.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
/// noxa-mcp library wrapper.
|
||||||
|
///
|
||||||
|
/// This exposes the MCP server so it can be embedded by the `noxa` CLI via
|
||||||
|
/// `noxa mcp` without duplicating the transport/bootstrap code.
|
||||||
|
///
|
||||||
|
/// Callers must initialize tracing before calling `run()`. Stdout must remain
|
||||||
|
/// untouched after `run()` begins because it carries the MCP wire protocol.
|
||||||
|
pub(crate) mod cloud;
|
||||||
|
pub(crate) mod server;
|
||||||
|
pub(crate) mod tools;
|
||||||
|
|
||||||
|
use rmcp::ServiceExt;
|
||||||
|
use rmcp::transport::stdio;
|
||||||
|
|
||||||
|
/// Start the MCP server over stdio and block until the client disconnects.
|
||||||
|
pub async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let service = server::NoxaMcp::new().await.serve(stdio()).await?;
|
||||||
|
service.waiting().await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -1,15 +1,6 @@
|
||||||
/// noxa-mcp: MCP (Model Context Protocol) server for noxa.
|
/// noxa-mcp: MCP (Model Context Protocol) server for noxa.
|
||||||
/// Exposes web extraction tools over stdio transport for AI agents
|
/// Exposes web extraction tools over stdio transport for AI agents
|
||||||
/// like Claude Desktop, Claude Code, and other MCP clients.
|
/// like Claude Desktop, Claude Code, and other MCP clients.
|
||||||
mod cloud;
|
|
||||||
mod server;
|
|
||||||
mod tools;
|
|
||||||
|
|
||||||
use rmcp::ServiceExt;
|
|
||||||
use rmcp::transport::stdio;
|
|
||||||
|
|
||||||
use server::NoxaMcp;
|
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
dotenvy::dotenv().ok();
|
dotenvy::dotenv().ok();
|
||||||
|
|
@ -21,8 +12,5 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
let service = NoxaMcp::new().await.serve(stdio()).await?;
|
noxa_mcp::run().await
|
||||||
|
|
||||||
service.waiting().await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue