mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-22 07:11:01 +02:00
Merge pull request #2 from jmagar/feature/noxa-mcp-subcommand
refactor: add noxa mcp subcommand
This commit is contained in:
jmagar
GitHub
commit
464eb1baec
23 changed files with 1874 additions and 199 deletions
16
.gitignore
vendored
16
.gitignore
vendored
|
|
@ -1,5 +1,21 @@
|
||||||
target/
|
target/
|
||||||
.DS_Store
|
.DS_Store
|
||||||
.env
|
.env
|
||||||
|
config.json
|
||||||
proxies.txt
|
proxies.txt
|
||||||
.claude/skills/
|
.claude/skills/
|
||||||
|
.omc
|
||||||
|
.lavra
|
||||||
|
.beads
|
||||||
|
.cache
|
||||||
|
docs/plans
|
||||||
|
docs/superpowers
|
||||||
|
docs/reports
|
||||||
|
docs/sessions
|
||||||
|
benchmarks
|
||||||
|
docs
|
||||||
|
|
||||||
|
# Beads / Dolt files (added by bd init)
|
||||||
|
.dolt/
|
||||||
|
*.db
|
||||||
|
.beads-credential-key
|
||||||
|
|
|
||||||
16
CLAUDE.md
16
CLAUDE.md
|
|
@ -15,8 +15,8 @@ noxa/
|
||||||
# + proxy pool rotation (per-request)
|
# + proxy pool rotation (per-request)
|
||||||
# + PDF content-type detection
|
# + PDF content-type detection
|
||||||
# + document parsing (DOCX, XLSX, CSV)
|
# + document parsing (DOCX, XLSX, CSV)
|
||||||
noxa-llm/ # LLM provider chain (Ollama -> OpenAI -> Anthropic)
|
noxa-llm/ # LLM provider chain (Gemini CLI -> OpenAI -> Ollama -> Anthropic)
|
||||||
# + JSON schema extraction, prompt extraction, summarization
|
# + JSON schema extraction (validated + retry), prompt extraction, summarization
|
||||||
noxa-pdf/ # PDF text extraction via pdf-extract
|
noxa-pdf/ # PDF text extraction via pdf-extract
|
||||||
noxa-mcp/ # MCP server (Model Context Protocol) for AI agents
|
noxa-mcp/ # MCP server (Model Context Protocol) for AI agents
|
||||||
noxa/ # CLI binary
|
noxa/ # CLI binary
|
||||||
|
|
@ -48,8 +48,10 @@ Two binaries: `noxa` (CLI), `noxa-mcp` (MCP server).
|
||||||
- `search.rs` — Web search via Serper.dev with parallel result scraping
|
- `search.rs` — Web search via Serper.dev with parallel result scraping
|
||||||
|
|
||||||
### LLM Modules (`noxa-llm`)
|
### LLM Modules (`noxa-llm`)
|
||||||
- Provider chain: Ollama (local-first) -> OpenAI -> Anthropic
|
- Provider chain: Gemini CLI (primary) -> OpenAI -> Ollama -> Anthropic
|
||||||
- JSON schema extraction, prompt-based extraction, summarization
|
- Gemini CLI requires the `gemini` binary on PATH; `GEMINI_MODEL` env var controls model (default: `gemini-2.5-pro`)
|
||||||
|
- JSON schema extraction with jsonschema validation; parse failures retry once; schema mismatches fail immediately
|
||||||
|
- Prompt-based extraction, summarization
|
||||||
|
|
||||||
### PDF Modules (`noxa-pdf`)
|
### PDF Modules (`noxa-pdf`)
|
||||||
- PDF text extraction via pdf-extract crate
|
- PDF text extraction via pdf-extract crate
|
||||||
|
|
@ -105,11 +107,15 @@ noxa https://example.com --diff-with snap.json
|
||||||
# Brand extraction
|
# Brand extraction
|
||||||
noxa https://example.com --brand
|
noxa https://example.com --brand
|
||||||
|
|
||||||
# LLM features (Ollama local-first)
|
# LLM features (Gemini CLI primary; requires `gemini` on PATH)
|
||||||
noxa https://example.com --summarize
|
noxa https://example.com --summarize
|
||||||
noxa https://example.com --extract-prompt "Get all pricing tiers"
|
noxa https://example.com --extract-prompt "Get all pricing tiers"
|
||||||
noxa https://example.com --extract-json '{"type":"object","properties":{"title":{"type":"string"}}}'
|
noxa https://example.com --extract-json '{"type":"object","properties":{"title":{"type":"string"}}}'
|
||||||
|
|
||||||
|
# Force a specific LLM provider
|
||||||
|
noxa https://example.com --llm-provider gemini --summarize
|
||||||
|
noxa https://example.com --llm-provider openai --summarize
|
||||||
|
|
||||||
# PDF (auto-detected via Content-Type)
|
# PDF (auto-detected via Content-Type)
|
||||||
noxa https://example.com/report.pdf
|
noxa https://example.com/report.pdf
|
||||||
|
|
||||||
|
|
|
||||||
246
Cargo.lock
generated
246
Cargo.lock
generated
|
|
@ -35,7 +35,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
|
"getrandom 0.3.4",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
|
"serde",
|
||||||
"version_check",
|
"version_check",
|
||||||
"zerocopy",
|
"zerocopy",
|
||||||
]
|
]
|
||||||
|
|
@ -64,6 +66,12 @@ dependencies = [
|
||||||
"alloc-no-stdlib",
|
"alloc-no-stdlib",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "allocator-api2"
|
||||||
|
version = "0.2.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "android_system_properties"
|
name = "android_system_properties"
|
||||||
version = "0.1.5"
|
version = "0.1.5"
|
||||||
|
|
@ -206,6 +214,21 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bit-set"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
|
||||||
|
dependencies = [
|
||||||
|
"bit-vec",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bit-vec"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "2.11.0"
|
version = "2.11.0"
|
||||||
|
|
@ -246,6 +269,12 @@ dependencies = [
|
||||||
"openssl-macros",
|
"openssl-macros",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "borrow-or-share"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "brotli"
|
name = "brotli"
|
||||||
version = "8.0.2"
|
version = "8.0.2"
|
||||||
|
|
@ -273,6 +302,12 @@ version = "3.20.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
|
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bytecount"
|
||||||
|
version = "0.6.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "byteorder"
|
name = "byteorder"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -601,6 +636,12 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "data-encoding"
|
||||||
|
version = "2.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "debug_unsafe"
|
name = "debug_unsafe"
|
||||||
version = "0.1.4"
|
version = "0.1.4"
|
||||||
|
|
@ -726,6 +767,15 @@ version = "1.15.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "email_address"
|
||||||
|
version = "0.2.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "encoding_rs"
|
name = "encoding_rs"
|
||||||
version = "0.8.35"
|
version = "0.8.35"
|
||||||
|
|
@ -760,6 +810,17 @@ dependencies = [
|
||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fancy-regex"
|
||||||
|
version = "0.17.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
|
||||||
|
dependencies = [
|
||||||
|
"bit-set",
|
||||||
|
"regex-automata",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fast-float2"
|
name = "fast-float2"
|
||||||
version = "0.2.3"
|
version = "0.2.3"
|
||||||
|
|
@ -789,6 +850,17 @@ dependencies = [
|
||||||
"zlib-rs",
|
"zlib-rs",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fluent-uri"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e"
|
||||||
|
dependencies = [
|
||||||
|
"borrow-or-share",
|
||||||
|
"ref-cast",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fnv"
|
name = "fnv"
|
||||||
version = "1.0.7"
|
version = "1.0.7"
|
||||||
|
|
@ -801,6 +873,12 @@ version = "0.1.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "foldhash"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "foreign-types"
|
name = "foreign-types"
|
||||||
version = "0.5.0"
|
version = "0.5.0"
|
||||||
|
|
@ -837,6 +915,16 @@ dependencies = [
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fraction"
|
||||||
|
version = "0.15.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static",
|
||||||
|
"num",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fs_extra"
|
name = "fs_extra"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
|
|
@ -1037,7 +1125,7 @@ version = "0.15.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"foldhash",
|
"foldhash 0.1.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1045,6 +1133,11 @@ name = "hashbrown"
|
||||||
version = "0.16.1"
|
version = "0.16.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||||
|
dependencies = [
|
||||||
|
"allocator-api2",
|
||||||
|
"equivalent",
|
||||||
|
"foldhash 0.2.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "heck"
|
name = "heck"
|
||||||
|
|
@ -1410,6 +1503,33 @@ dependencies = [
|
||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jsonschema"
|
||||||
|
version = "0.46.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "84695c6689b01384700a3d93acecbd07231ee6fff1bf22ae980b4c307e6ddfd5"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
"bytecount",
|
||||||
|
"data-encoding",
|
||||||
|
"email_address",
|
||||||
|
"fancy-regex",
|
||||||
|
"fraction",
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"idna",
|
||||||
|
"itoa",
|
||||||
|
"num-cmp",
|
||||||
|
"num-traits",
|
||||||
|
"percent-encoding",
|
||||||
|
"referencing",
|
||||||
|
"regex",
|
||||||
|
"regex-syntax",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"unicode-general-category",
|
||||||
|
"uuid-simd",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lazy_static"
|
name = "lazy_static"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -1575,6 +1695,12 @@ version = "2.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "micromap"
|
||||||
|
version = "0.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "minimal-lexical"
|
name = "minimal-lexical"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
|
|
@ -1627,10 +1753,12 @@ dependencies = [
|
||||||
"noxa-core",
|
"noxa-core",
|
||||||
"noxa-fetch",
|
"noxa-fetch",
|
||||||
"noxa-llm",
|
"noxa-llm",
|
||||||
|
"noxa-mcp",
|
||||||
"noxa-pdf",
|
"noxa-pdf",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
@ -1683,6 +1811,7 @@ name = "noxa-llm"
|
||||||
version = "0.3.11"
|
version = "0.3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"jsonschema",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|
@ -1730,12 +1859,82 @@ dependencies = [
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
|
||||||
|
dependencies = [
|
||||||
|
"num-bigint",
|
||||||
|
"num-complex",
|
||||||
|
"num-integer",
|
||||||
|
"num-iter",
|
||||||
|
"num-rational",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-bigint"
|
||||||
|
version = "0.4.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
|
||||||
|
dependencies = [
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-cmp"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-complex"
|
||||||
|
version = "0.4.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-conv"
|
name = "num-conv"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-integer"
|
||||||
|
version = "0.1.46"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-iter"
|
||||||
|
version = "0.1.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-rational"
|
||||||
|
version = "0.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
|
||||||
|
dependencies = [
|
||||||
|
"num-bigint",
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-traits"
|
name = "num-traits"
|
||||||
version = "0.2.19"
|
version = "0.2.19"
|
||||||
|
|
@ -1774,6 +1973,12 @@ version = "0.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "outref"
|
||||||
|
version = "0.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.5"
|
version = "0.12.5"
|
||||||
|
|
@ -2160,6 +2365,23 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "referencing"
|
||||||
|
version = "0.46.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a2d5554bf79f4acf770dc3193b44b2d63b348f5f7b7448a0ea1191b37b620728"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
"fluent-uri",
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"hashbrown 0.16.1",
|
||||||
|
"itoa",
|
||||||
|
"micromap",
|
||||||
|
"parking_lot",
|
||||||
|
"percent-encoding",
|
||||||
|
"serde_json",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex"
|
name = "regex"
|
||||||
version = "1.12.3"
|
version = "1.12.3"
|
||||||
|
|
@ -2985,6 +3207,12 @@ version = "1.19.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
|
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-general-category"
|
||||||
|
version = "1.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-ident"
|
name = "unicode-ident"
|
||||||
version = "1.0.24"
|
version = "1.0.24"
|
||||||
|
|
@ -3049,6 +3277,16 @@ version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "uuid-simd"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8"
|
||||||
|
dependencies = [
|
||||||
|
"outref",
|
||||||
|
"vsimd",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "valuable"
|
name = "valuable"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
|
|
@ -3061,6 +3299,12 @@ version = "0.9.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "vsimd"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "want"
|
name = "want"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ noxa-core = { path = "crates/noxa-core" }
|
||||||
noxa-fetch = { path = "crates/noxa-fetch" }
|
noxa-fetch = { path = "crates/noxa-fetch" }
|
||||||
noxa-llm = { path = "crates/noxa-llm" }
|
noxa-llm = { path = "crates/noxa-llm" }
|
||||||
noxa-pdf = { path = "crates/noxa-pdf" }
|
noxa-pdf = { path = "crates/noxa-pdf" }
|
||||||
|
noxa-mcp = { path = "crates/noxa-mcp" }
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
@ -21,3 +22,6 @@ tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
clap = { version = "4", features = ["derive", "env"] }
|
clap = { version = "4", features = ["derive", "env"] }
|
||||||
dotenvy = "0.15"
|
dotenvy = "0.15"
|
||||||
|
rmcp = { version = "1.2", features = ["server", "macros", "transport-io", "schemars"] }
|
||||||
|
schemars = "1.0"
|
||||||
|
dirs = "6.0.0"
|
||||||
|
|
|
||||||
292
README.md
292
README.md
|
|
@ -77,7 +77,7 @@ Download from [GitHub Releases](https://github.com/jmagar/noxa/releases) for mac
|
||||||
### Cargo (from source)
|
### Cargo (from source)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cargo install --git https://github.com/jmagar/noxa.git noxa
|
cargo install --git https://github.com/jmagar/noxa.git noxa-cli --bin noxa
|
||||||
cargo install --git https://github.com/jmagar/noxa.git noxa-mcp
|
cargo install --git https://github.com/jmagar/noxa.git noxa-mcp
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -159,6 +159,271 @@ Crawling... 50/50 pages extracted
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
### Basic Extraction
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Extract as markdown (default)
|
||||||
|
noxa https://example.com
|
||||||
|
|
||||||
|
# Multiple output formats
|
||||||
|
noxa https://example.com -f markdown # Clean markdown
|
||||||
|
noxa https://example.com -f json # Full structured JSON
|
||||||
|
noxa https://example.com -f text # Plain text (no formatting)
|
||||||
|
noxa https://example.com -f llm # Token-optimized for LLMs (67% fewer tokens)
|
||||||
|
|
||||||
|
# Bare domains work (auto-prepends https://)
|
||||||
|
noxa example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
### Content Filtering
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Only extract main content (skip nav, sidebar, footer)
|
||||||
|
noxa https://docs.rs/tokio --only-main-content
|
||||||
|
|
||||||
|
# Include specific CSS selectors
|
||||||
|
noxa https://news.ycombinator.com --include ".titleline,.score"
|
||||||
|
|
||||||
|
# Exclude specific elements
|
||||||
|
noxa https://example.com --exclude "nav,footer,.ads,.sidebar"
|
||||||
|
|
||||||
|
# Combine both
|
||||||
|
noxa https://docs.rs/reqwest --only-main-content --exclude ".sidebar"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Brand Identity Extraction
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Extract colors, fonts, logos from any website
|
||||||
|
noxa --brand https://stripe.com
|
||||||
|
# Output: { "name": "Stripe", "colors": [...], "fonts": ["Sohne"], "logos": [...] }
|
||||||
|
|
||||||
|
noxa --brand https://github.com
|
||||||
|
# Output: { "name": "GitHub", "colors": [{"hex": "#1F2328", ...}], "fonts": ["Mona Sans"], ... }
|
||||||
|
|
||||||
|
noxa --brand wikipedia.org
|
||||||
|
# Output: 10 colors, 5 fonts, favicon, logo URL
|
||||||
|
```
|
||||||
|
|
||||||
|
### Sitemap Discovery
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Discover all URLs from a site's sitemaps
|
||||||
|
noxa --map https://sitemaps.org
|
||||||
|
# Output: one URL per line (84 URLs found)
|
||||||
|
|
||||||
|
# JSON output with metadata
|
||||||
|
noxa --map https://sitemaps.org -f json
|
||||||
|
# Output: [{ "url": "...", "last_modified": "...", "priority": 0.8 }]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Recursive Crawling
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Crawl a site (default: depth 1, max 20 pages)
|
||||||
|
noxa --crawl https://example.com
|
||||||
|
|
||||||
|
# Control depth and page limit
|
||||||
|
noxa --crawl --depth 2 --max-pages 50 https://docs.rs/tokio
|
||||||
|
|
||||||
|
# Crawl with sitemap seeding (finds more pages)
|
||||||
|
noxa --crawl --sitemap --depth 2 https://docs.rs/tokio
|
||||||
|
|
||||||
|
# Filter crawl paths
|
||||||
|
noxa --crawl --include-paths "/api/*,/guide/*" https://docs.example.com
|
||||||
|
noxa --crawl --exclude-paths "/changelog/*,/blog/*" https://docs.example.com
|
||||||
|
|
||||||
|
# Control concurrency and delay
|
||||||
|
noxa --crawl --concurrency 10 --delay 200 https://example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
### Change Detection (Diff)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Step 1: Save a snapshot
|
||||||
|
noxa https://example.com -f json > snapshot.json
|
||||||
|
|
||||||
|
# Step 2: Later, compare against the snapshot
|
||||||
|
noxa --diff-with snapshot.json https://example.com
|
||||||
|
# Output:
|
||||||
|
# Status: Same
|
||||||
|
# Word count delta: +0
|
||||||
|
|
||||||
|
# If the page changed:
|
||||||
|
# Status: Changed
|
||||||
|
# Word count delta: +42
|
||||||
|
# --- old
|
||||||
|
# +++ new
|
||||||
|
# @@ -1,3 +1,3 @@
|
||||||
|
# -Old content here
|
||||||
|
# +New content here
|
||||||
|
```
|
||||||
|
|
||||||
|
### PDF Extraction
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# PDF URLs are auto-detected via Content-Type
|
||||||
|
noxa https://example.com/report.pdf
|
||||||
|
|
||||||
|
# Control PDF mode
|
||||||
|
noxa --pdf-mode auto https://example.com/report.pdf # Error on empty (catches scanned PDFs)
|
||||||
|
noxa --pdf-mode fast https://example.com/report.pdf # Return whatever text is found
|
||||||
|
```
|
||||||
|
|
||||||
|
### Batch Processing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Multiple URLs in one command
|
||||||
|
noxa https://example.com https://httpbin.org/html https://rust-lang.org
|
||||||
|
|
||||||
|
# URLs from a file (one per line, # comments supported)
|
||||||
|
noxa --urls-file urls.txt
|
||||||
|
|
||||||
|
# Batch with JSON output
|
||||||
|
noxa --urls-file urls.txt -f json
|
||||||
|
|
||||||
|
# Proxy rotation for large batches
|
||||||
|
noxa --urls-file urls.txt --proxy-file proxies.txt --concurrency 10
|
||||||
|
```
|
||||||
|
|
||||||
|
### Local Files & Stdin
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Extract from a local HTML file
|
||||||
|
noxa --file page.html
|
||||||
|
|
||||||
|
# Pipe HTML from another command
|
||||||
|
curl -s https://example.com | noxa --stdin
|
||||||
|
|
||||||
|
# Chain with other tools
|
||||||
|
noxa https://example.com -f text | wc -w # Word count
|
||||||
|
noxa https://example.com -f json | jq '.metadata.title' # Extract title with jq
|
||||||
|
```
|
||||||
|
|
||||||
|
### Browser Impersonation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Chrome (default) — latest Chrome TLS fingerprint
|
||||||
|
noxa https://example.com
|
||||||
|
|
||||||
|
# Firefox fingerprint
|
||||||
|
noxa --browser firefox https://example.com
|
||||||
|
|
||||||
|
# Random browser per request (good for batch)
|
||||||
|
noxa --browser random --urls-file urls.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Custom Headers & Cookies
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Custom headers
|
||||||
|
noxa -H "Authorization: Bearer token123" https://api.example.com
|
||||||
|
noxa -H "Accept-Language: de-DE" https://example.com
|
||||||
|
|
||||||
|
# Cookies
|
||||||
|
noxa --cookie "session=abc123; theme=dark" https://example.com
|
||||||
|
|
||||||
|
# Multiple headers
|
||||||
|
noxa -H "X-Custom: value" -H "Authorization: Bearer token" https://example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
### LLM-Powered Features
|
||||||
|
|
||||||
|
These require an LLM provider (Ollama local, or OpenAI/Anthropic API key).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Summarize a page (default: 3 sentences)
|
||||||
|
noxa --summarize https://example.com
|
||||||
|
|
||||||
|
# Control summary length
|
||||||
|
noxa --summarize 5 https://example.com
|
||||||
|
|
||||||
|
# Extract structured JSON with a schema
|
||||||
|
noxa --extract-json '{"type":"object","properties":{"title":{"type":"string"},"price":{"type":"number"}}}' https://example.com/product
|
||||||
|
|
||||||
|
# Extract with a schema from file
|
||||||
|
noxa --extract-json @schema.json https://example.com/product
|
||||||
|
|
||||||
|
# Extract with natural language prompt
|
||||||
|
noxa --extract-prompt "Get all pricing tiers with name, price, and features" https://stripe.com/pricing
|
||||||
|
|
||||||
|
# Use a specific LLM provider
|
||||||
|
noxa --llm-provider ollama --summarize https://example.com
|
||||||
|
noxa --llm-provider openai --llm-model gpt-4o --extract-prompt "..." https://example.com
|
||||||
|
noxa --llm-provider anthropic --summarize https://example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
### Raw HTML Output
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Get the raw fetched HTML (no extraction)
|
||||||
|
noxa --raw-html https://example.com
|
||||||
|
|
||||||
|
# Useful for debugging extraction issues
|
||||||
|
noxa --raw-html https://example.com > raw.html
|
||||||
|
noxa --file raw.html # Then extract locally
|
||||||
|
```
|
||||||
|
|
||||||
|
### Metadata & Verbose Mode
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Include YAML frontmatter with metadata
|
||||||
|
noxa --metadata https://example.com
|
||||||
|
# Output:
|
||||||
|
# ---
|
||||||
|
# title: "Example Domain"
|
||||||
|
# source: "https://example.com"
|
||||||
|
# word_count: 20
|
||||||
|
# ---
|
||||||
|
# # Example Domain
|
||||||
|
# ...
|
||||||
|
|
||||||
|
# Verbose logging (debug extraction pipeline)
|
||||||
|
noxa -v https://example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
### Proxy Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Single proxy
|
||||||
|
noxa --proxy http://user:pass@proxy.example.com:8080 https://example.com
|
||||||
|
|
||||||
|
# SOCKS5 proxy
|
||||||
|
noxa --proxy socks5://proxy.example.com:1080 https://example.com
|
||||||
|
|
||||||
|
# Proxy rotation from file (one per line: host:port:user:pass)
|
||||||
|
noxa --proxy-file proxies.txt https://example.com
|
||||||
|
|
||||||
|
# Auto-load proxies.txt from current directory
|
||||||
|
echo "proxy1.com:8080:user:pass" > proxies.txt
|
||||||
|
noxa https://example.com # Automatically detects and uses proxies.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Real-World Recipes
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Monitor competitor pricing — save today's pricing
|
||||||
|
noxa --extract-json '{"type":"array","items":{"type":"object","properties":{"plan":{"type":"string"},"price":{"type":"string"}}}}' \
|
||||||
|
https://competitor.com/pricing -f json > pricing-$(date +%Y%m%d).json
|
||||||
|
|
||||||
|
# Build a documentation search index
|
||||||
|
noxa --crawl --sitemap --depth 3 --max-pages 500 -f llm https://docs.example.com > docs.txt
|
||||||
|
|
||||||
|
# Extract all images from a page
|
||||||
|
noxa https://example.com -f json | jq -r '.content.images[].src'
|
||||||
|
|
||||||
|
# Get all external links
|
||||||
|
noxa https://example.com -f json | jq -r '.content.links[] | select(.href | startswith("http")) | .href'
|
||||||
|
|
||||||
|
# Compare two pages
|
||||||
|
noxa https://site-a.com -f json > a.json
|
||||||
|
noxa https://site-b.com --diff-with a.json
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## MCP Server — 10 tools for AI agents
|
## MCP Server — 10 tools for AI agents
|
||||||
|
|
||||||
<a href="https://glama.ai/mcp/servers/jmagar/noxa"><img src="https://glama.ai/mcp/servers/jmagar/noxa/badge" alt="noxa MCP server" /></a>
|
<a href="https://glama.ai/mcp/servers/jmagar/noxa"><img src="https://glama.ai/mcp/servers/jmagar/noxa/badge" alt="noxa MCP server" /></a>
|
||||||
|
|
@ -327,6 +592,31 @@ noxa/
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
|
Non-secret defaults live in `config.json` in your working directory. Copy the example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp config.example.json config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
**Precedence:** CLI flags > `config.json` > built-in defaults
|
||||||
|
|
||||||
|
**Secrets and URLs** (API keys, proxy, webhook, LLM base URL) always go in `.env`, not `config.json`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp env.example .env
|
||||||
|
```
|
||||||
|
|
||||||
|
**Override config path** for a single run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NOXA_CONFIG=/path/to/other-config.json noxa https://example.com
|
||||||
|
NOXA_CONFIG=/dev/null noxa https://example.com # bypass config entirely
|
||||||
|
```
|
||||||
|
|
||||||
|
**Bool flag limitation:** flags like `--metadata`, `--only-main-content`, `--verbose` set to `true` in `config.json` cannot be overridden to `false` from the CLI for a single run (clap has no `--no-flag` variant). Use `NOXA_CONFIG=/dev/null` to bypass.
|
||||||
|
|
||||||
|
### Environment variables
|
||||||
|
|
||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| `NOXA_API_KEY` | Cloud API key (enables bot bypass, JS rendering, search, research) |
|
| `NOXA_API_KEY` | Cloud API key (enables bot bypass, JS rendering, search, research) |
|
||||||
|
|
|
||||||
34
config.example.json
Normal file
34
config.example.json
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
{
|
||||||
|
"_doc": [
|
||||||
|
"Copy to config.json and remove fields you don't need.",
|
||||||
|
"Secrets (api_key, proxy, webhook, llm_base_url) go in .env — NOT here.",
|
||||||
|
"BOOL FLAG LIMITATION: once set to true here, cannot be overridden to false",
|
||||||
|
"from the CLI for a single run (no --no-flag support). Use NOXA_CONFIG=/dev/null",
|
||||||
|
"on the command line to bypass this config entirely.",
|
||||||
|
"on_change is intentionally absent — it must remain a CLI-only flag.",
|
||||||
|
"Unknown fields are silently ignored, so this file works across noxa versions."
|
||||||
|
],
|
||||||
|
|
||||||
|
"format": "markdown",
|
||||||
|
"browser": "chrome",
|
||||||
|
"timeout": 30,
|
||||||
|
"pdf_mode": "auto",
|
||||||
|
"metadata": false,
|
||||||
|
"verbose": false,
|
||||||
|
"only_main_content": false,
|
||||||
|
|
||||||
|
"include_selectors": [],
|
||||||
|
"exclude_selectors": ["nav", "footer", ".sidebar", ".cookie-banner"],
|
||||||
|
|
||||||
|
"depth": 1,
|
||||||
|
"max_pages": 20,
|
||||||
|
"concurrency": 5,
|
||||||
|
"delay": 100,
|
||||||
|
"path_prefix": null,
|
||||||
|
"include_paths": [],
|
||||||
|
"exclude_paths": ["/changelog/*", "/blog/*", "/releases/*"],
|
||||||
|
"use_sitemap": false,
|
||||||
|
|
||||||
|
"llm_provider": "gemini",
|
||||||
|
"llm_model": "gemini-2.5-pro"
|
||||||
|
}
|
||||||
|
|
@ -14,9 +14,11 @@ noxa-core = { workspace = true }
|
||||||
noxa-fetch = { workspace = true }
|
noxa-fetch = { workspace = true }
|
||||||
noxa-llm = { workspace = true }
|
noxa-llm = { workspace = true }
|
||||||
noxa-pdf = { workspace = true }
|
noxa-pdf = { workspace = true }
|
||||||
|
noxa-mcp = { workspace = true }
|
||||||
dotenvy = { workspace = true }
|
dotenvy = { workspace = true }
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
serde = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
clap = { workspace = true }
|
clap = { workspace = true }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
|
|
|
||||||
315
crates/noxa-cli/src/config.rs
Normal file
315
crates/noxa-cli/src/config.rs
Normal file
|
|
@ -0,0 +1,315 @@
|
||||||
|
use serde::Deserialize;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use crate::{Browser, OutputFormat, PdfModeArg};
|
||||||
|
|
||||||
|
/// Non-secret, non-URL configuration defaults loaded from config.json.
|
||||||
|
/// All fields optional — absent means "use the hard default".
|
||||||
|
/// Unknown fields are silently ignored (serde default) so config files
|
||||||
|
/// written for a newer version of noxa work on older binaries.
|
||||||
|
///
|
||||||
|
/// DELIBERATELY EXCLUDED:
|
||||||
|
/// - on_change: passes content to sh -c; must remain CLI-only to prevent
|
||||||
|
/// shell injection via config file writes.
|
||||||
|
/// - Secrets/URLs (api_key, proxy, webhook, llm_base_url): stay in .env.
|
||||||
|
///
|
||||||
|
/// BOOL FLAG LIMITATION:
|
||||||
|
/// only_main_content, metadata, verbose, use_sitemap set to true here
|
||||||
|
/// cannot be overridden to false from the CLI for a single run (no --no-flag
|
||||||
|
/// variant in clap). Edit config.json or use NOXA_CONFIG=/dev/null to bypass.
|
||||||
|
#[derive(Debug, Default, Deserialize)]
|
||||||
|
pub struct NoxaConfig {
|
||||||
|
// Output
|
||||||
|
pub format: Option<OutputFormat>,
|
||||||
|
pub metadata: Option<bool>,
|
||||||
|
pub verbose: Option<bool>,
|
||||||
|
|
||||||
|
// Fetch
|
||||||
|
pub browser: Option<Browser>,
|
||||||
|
pub timeout: Option<u64>,
|
||||||
|
pub pdf_mode: Option<PdfModeArg>,
|
||||||
|
pub only_main_content: Option<bool>,
|
||||||
|
|
||||||
|
// CSS selectors
|
||||||
|
pub include_selectors: Option<Vec<String>>,
|
||||||
|
pub exclude_selectors: Option<Vec<String>>,
|
||||||
|
|
||||||
|
// Crawl
|
||||||
|
pub depth: Option<usize>,
|
||||||
|
pub max_pages: Option<usize>,
|
||||||
|
pub concurrency: Option<usize>,
|
||||||
|
pub delay: Option<u64>,
|
||||||
|
pub path_prefix: Option<String>,
|
||||||
|
pub include_paths: Option<Vec<String>>,
|
||||||
|
pub exclude_paths: Option<Vec<String>>,
|
||||||
|
pub use_sitemap: Option<bool>,
|
||||||
|
|
||||||
|
// LLM (non-secret: provider name and model only; base URL stays in .env)
|
||||||
|
pub llm_provider: Option<String>,
|
||||||
|
pub llm_model: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NoxaConfig {
|
||||||
|
/// Load config from an explicit path, NOXA_CONFIG env var, or ./config.json.
|
||||||
|
/// Returns an empty (all-None) config if the file doesn't exist.
|
||||||
|
/// Prints an error and exits if the file exists but is invalid JSON.
|
||||||
|
pub fn load(explicit_path: Option<&str>) -> Self {
|
||||||
|
let noxa_config_env = std::env::var("NOXA_CONFIG").ok();
|
||||||
|
let was_explicit = explicit_path.is_some() || noxa_config_env.is_some();
|
||||||
|
|
||||||
|
let path_str = explicit_path
|
||||||
|
.map(String::from)
|
||||||
|
.or(noxa_config_env)
|
||||||
|
.unwrap_or_else(|| "config.json".to_string());
|
||||||
|
|
||||||
|
let path = Path::new(&path_str);
|
||||||
|
if !path.exists() {
|
||||||
|
if was_explicit {
|
||||||
|
let display_name = path.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or(&path_str);
|
||||||
|
eprintln!("error: config file not found: {display_name}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
return Self::default();
|
||||||
|
}
|
||||||
|
|
||||||
|
let display_name = path.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or(&path_str);
|
||||||
|
eprintln!(
|
||||||
|
"noxa: config loaded from {display_name} \
|
||||||
|
(API keys and secrets belong in .env, not config.json)"
|
||||||
|
);
|
||||||
|
tracing::debug!("config path: {}", path.display());
|
||||||
|
|
||||||
|
let content = match std::fs::read_to_string(path) {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("error: cannot read config file {display_name}: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match serde_json::from_str(&content) {
|
||||||
|
Ok(cfg) => cfg,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("error: invalid JSON in config file {display_name}: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fully resolved configuration after merging CLI flags > config file > hard defaults.
|
||||||
|
/// All fields are concrete — no Option<T>. This is what the rest of main.rs reads.
|
||||||
|
///
|
||||||
|
/// The merge uses clap's ValueSource to detect which fields were explicitly set on
|
||||||
|
/// the command line. CLI-explicit values always win. Config fills in the rest.
|
||||||
|
/// Hard defaults are the fallback of last resort.
|
||||||
|
pub struct ResolvedConfig {
|
||||||
|
// Output
|
||||||
|
pub format: OutputFormat,
|
||||||
|
pub metadata: bool,
|
||||||
|
pub verbose: bool,
|
||||||
|
|
||||||
|
// Fetch
|
||||||
|
pub browser: Browser,
|
||||||
|
pub timeout: u64,
|
||||||
|
pub pdf_mode: PdfModeArg,
|
||||||
|
pub only_main_content: bool,
|
||||||
|
/// CLI-only output flag — not configurable via config.json (it is a per-run mode, not a persistent default).
|
||||||
|
pub raw_html: bool,
|
||||||
|
|
||||||
|
// CSS selectors
|
||||||
|
/// Vec<String> — CSS selectors passed directly to extraction filter.
|
||||||
|
pub include_selectors: Vec<String>,
|
||||||
|
/// Vec<String> — CSS selectors passed directly to extraction filter.
|
||||||
|
pub exclude_selectors: Vec<String>,
|
||||||
|
|
||||||
|
// Crawl
|
||||||
|
pub depth: usize,
|
||||||
|
pub max_pages: usize,
|
||||||
|
pub concurrency: usize,
|
||||||
|
pub delay: u64,
|
||||||
|
pub path_prefix: Option<String>,
|
||||||
|
/// Vec<String> — never joined to a comma-string. Passed directly to CrawlConfig.
|
||||||
|
pub include_paths: Vec<String>,
|
||||||
|
/// Vec<String> — never joined to a comma-string. Passed directly to CrawlConfig.
|
||||||
|
pub exclude_paths: Vec<String>,
|
||||||
|
pub use_sitemap: bool,
|
||||||
|
|
||||||
|
// LLM
|
||||||
|
pub llm_provider: Option<String>,
|
||||||
|
pub llm_model: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
use clap::parser::ValueSource;
|
||||||
|
|
||||||
|
/// Merge CLI flags (detected via ValueSource), config file, and hard defaults
|
||||||
|
/// into a single ResolvedConfig. CLI explicit values always win.
|
||||||
|
pub fn resolve(
|
||||||
|
cli: &crate::Cli,
|
||||||
|
matches: &clap::ArgMatches,
|
||||||
|
cfg: &NoxaConfig,
|
||||||
|
) -> ResolvedConfig {
|
||||||
|
let explicit = |name: &str| {
|
||||||
|
matches.value_source(name) == Some(ValueSource::CommandLine)
|
||||||
|
};
|
||||||
|
|
||||||
|
ResolvedConfig {
|
||||||
|
format: if explicit("format") {
|
||||||
|
cli.format.clone()
|
||||||
|
} else {
|
||||||
|
cfg.format.clone().unwrap_or(crate::OutputFormat::Markdown)
|
||||||
|
},
|
||||||
|
browser: if explicit("browser") {
|
||||||
|
cli.browser.clone()
|
||||||
|
} else {
|
||||||
|
cfg.browser.clone().unwrap_or(crate::Browser::Chrome)
|
||||||
|
},
|
||||||
|
pdf_mode: if explicit("pdf_mode") {
|
||||||
|
cli.pdf_mode.clone()
|
||||||
|
} else {
|
||||||
|
cfg.pdf_mode.clone().unwrap_or(crate::PdfModeArg::Auto)
|
||||||
|
},
|
||||||
|
timeout: if explicit("timeout") {
|
||||||
|
cli.timeout
|
||||||
|
} else {
|
||||||
|
cfg.timeout.unwrap_or(30)
|
||||||
|
},
|
||||||
|
depth: if explicit("depth") {
|
||||||
|
cli.depth
|
||||||
|
} else {
|
||||||
|
cfg.depth.unwrap_or(1)
|
||||||
|
},
|
||||||
|
max_pages: if explicit("max_pages") {
|
||||||
|
cli.max_pages
|
||||||
|
} else {
|
||||||
|
cfg.max_pages.unwrap_or(20)
|
||||||
|
},
|
||||||
|
concurrency: if explicit("concurrency") {
|
||||||
|
cli.concurrency
|
||||||
|
} else {
|
||||||
|
cfg.concurrency.unwrap_or(5)
|
||||||
|
},
|
||||||
|
delay: if explicit("delay") {
|
||||||
|
cli.delay
|
||||||
|
} else {
|
||||||
|
cfg.delay.unwrap_or(100)
|
||||||
|
},
|
||||||
|
path_prefix: if explicit("path_prefix") {
|
||||||
|
cli.path_prefix.clone()
|
||||||
|
} else {
|
||||||
|
cfg.path_prefix.clone()
|
||||||
|
},
|
||||||
|
include_paths: if explicit("include_paths") {
|
||||||
|
cli.include_paths
|
||||||
|
.as_deref()
|
||||||
|
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
cfg.include_paths.clone().unwrap_or_default()
|
||||||
|
},
|
||||||
|
exclude_paths: if explicit("exclude_paths") {
|
||||||
|
cli.exclude_paths
|
||||||
|
.as_deref()
|
||||||
|
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
cfg.exclude_paths.clone().unwrap_or_default()
|
||||||
|
},
|
||||||
|
include_selectors: if explicit("include") {
|
||||||
|
cli.include
|
||||||
|
.as_deref()
|
||||||
|
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
cfg.include_selectors.clone().unwrap_or_default()
|
||||||
|
},
|
||||||
|
exclude_selectors: if explicit("exclude") {
|
||||||
|
cli.exclude
|
||||||
|
.as_deref()
|
||||||
|
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
} else {
|
||||||
|
cfg.exclude_selectors.clone().unwrap_or_default()
|
||||||
|
},
|
||||||
|
only_main_content: cli.only_main_content || cfg.only_main_content.unwrap_or(false),
|
||||||
|
metadata: cli.metadata || cfg.metadata.unwrap_or(false),
|
||||||
|
verbose: cli.verbose || cfg.verbose.unwrap_or(false),
|
||||||
|
use_sitemap: cli.sitemap || cfg.use_sitemap.unwrap_or(false),
|
||||||
|
raw_html: cli.raw_html,
|
||||||
|
llm_provider: if cli.llm_provider.is_some() {
|
||||||
|
cli.llm_provider.clone()
|
||||||
|
} else {
|
||||||
|
cfg.llm_provider.clone()
|
||||||
|
},
|
||||||
|
llm_model: if cli.llm_model.is_some() {
|
||||||
|
cli.llm_model.clone()
|
||||||
|
} else {
|
||||||
|
cfg.llm_model.clone()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_noxa_config_deserialize_full() {
|
||||||
|
let json = r#"{
|
||||||
|
"format": "llm",
|
||||||
|
"depth": 3,
|
||||||
|
"max_pages": 100,
|
||||||
|
"concurrency": 10,
|
||||||
|
"delay": 200,
|
||||||
|
"browser": "firefox",
|
||||||
|
"timeout": 60,
|
||||||
|
"only_main_content": true,
|
||||||
|
"use_sitemap": true,
|
||||||
|
"path_prefix": "/docs/",
|
||||||
|
"include_paths": ["/docs/*", "/api/*"],
|
||||||
|
"exclude_paths": ["/changelog/*", "/blog/*"],
|
||||||
|
"include_selectors": ["article", ".content"],
|
||||||
|
"exclude_selectors": ["nav", "footer"],
|
||||||
|
"llm_provider": "gemini",
|
||||||
|
"llm_model": "gemini-2.5-pro",
|
||||||
|
"pdf_mode": "fast",
|
||||||
|
"metadata": true,
|
||||||
|
"verbose": false
|
||||||
|
}"#;
|
||||||
|
let cfg: NoxaConfig = serde_json::from_str(json).unwrap();
|
||||||
|
assert!(matches!(cfg.format, Some(crate::OutputFormat::Llm)));
|
||||||
|
assert_eq!(cfg.depth, Some(3));
|
||||||
|
assert_eq!(cfg.exclude_paths, Some(vec!["/changelog/*".to_string(), "/blog/*".to_string()]));
|
||||||
|
assert!(matches!(cfg.pdf_mode, Some(crate::PdfModeArg::Fast)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_noxa_config_empty() {
|
||||||
|
let cfg: NoxaConfig = serde_json::from_str("{}").unwrap();
|
||||||
|
assert!(cfg.format.is_none());
|
||||||
|
assert!(cfg.depth.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_noxa_config_unknown_fields_ignored() {
|
||||||
|
// Unknown fields must NOT cause a parse failure
|
||||||
|
let cfg: NoxaConfig = serde_json::from_str(r#"{"depth": 2, "future_field": true}"#).unwrap();
|
||||||
|
assert_eq!(cfg.depth, Some(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_load_implicit_missing_file_returns_default() {
|
||||||
|
// When no explicit path and ./config.json doesn't exist, silently return default.
|
||||||
|
// The simplest test: call with None and rely on ./config.json not existing in test env.
|
||||||
|
// If CWD has config.json this test is skipped to avoid flakiness.
|
||||||
|
if std::path::Path::new("config.json").exists() {
|
||||||
|
return; // skip: CWD has config.json
|
||||||
|
}
|
||||||
|
let cfg = NoxaConfig::load(None);
|
||||||
|
assert!(cfg.format.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
/// CLI entry point -- wires noxa-core and noxa-fetch into a single command.
|
/// CLI entry point -- wires noxa-core and noxa-fetch into a single command.
|
||||||
/// All extraction and fetching logic lives in sibling crates; this is pure plumbing.
|
/// All extraction and fetching logic lives in sibling crates; this is pure plumbing.
|
||||||
mod cloud;
|
mod cloud;
|
||||||
|
mod config;
|
||||||
|
|
||||||
use std::io::{self, Read as _};
|
use std::io::{self, Read as _};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
@ -9,8 +10,7 @@ use std::process;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
use clap::{Parser, ValueEnum};
|
use clap::{CommandFactory, FromArgMatches, Parser, ValueEnum};
|
||||||
use tracing_subscriber::EnvFilter;
|
|
||||||
use noxa_core::{
|
use noxa_core::{
|
||||||
ChangeStatus, ContentDiff, ExtractionOptions, ExtractionResult, Metadata, extract_with_options,
|
ChangeStatus, ContentDiff, ExtractionOptions, ExtractionResult, Metadata, extract_with_options,
|
||||||
to_llm_text,
|
to_llm_text,
|
||||||
|
|
@ -20,7 +20,10 @@ use noxa_fetch::{
|
||||||
FetchConfig, FetchResult, PageResult, SitemapEntry,
|
FetchConfig, FetchResult, PageResult, SitemapEntry,
|
||||||
};
|
};
|
||||||
use noxa_llm::LlmProvider;
|
use noxa_llm::LlmProvider;
|
||||||
|
use noxa_mcp;
|
||||||
use noxa_pdf::PdfMode;
|
use noxa_pdf::PdfMode;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use tracing_subscriber::EnvFilter;
|
||||||
|
|
||||||
/// Known anti-bot challenge page titles (case-insensitive prefix match).
|
/// Known anti-bot challenge page titles (case-insensitive prefix match).
|
||||||
const ANTIBOT_TITLES: &[&str] = &[
|
const ANTIBOT_TITLES: &[&str] = &[
|
||||||
|
|
@ -87,6 +90,10 @@ fn warn_empty(url: &str, reason: &EmptyReason) {
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(name = "noxa", about = "Extract web content for LLMs", version)]
|
#[command(name = "noxa", about = "Extract web content for LLMs", version)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
|
/// Path to config.json (default: ./config.json, override with NOXA_CONFIG env var)
|
||||||
|
#[arg(long, global = true)]
|
||||||
|
config: Option<String>,
|
||||||
|
|
||||||
/// URLs to fetch (multiple allowed)
|
/// URLs to fetch (multiple allowed)
|
||||||
#[arg()]
|
#[arg()]
|
||||||
urls: Vec<String>,
|
urls: Vec<String>,
|
||||||
|
|
@ -247,7 +254,7 @@ struct Cli {
|
||||||
#[arg(long, num_args = 0..=1, default_missing_value = "3")]
|
#[arg(long, num_args = 0..=1, default_missing_value = "3")]
|
||||||
summarize: Option<usize>,
|
summarize: Option<usize>,
|
||||||
|
|
||||||
/// Force a specific LLM provider (ollama, openai, anthropic)
|
/// Force a specific LLM provider (gemini, ollama, openai, anthropic)
|
||||||
#[arg(long, env = "NOXA_LLM_PROVIDER")]
|
#[arg(long, env = "NOXA_LLM_PROVIDER")]
|
||||||
llm_provider: Option<String>,
|
llm_provider: Option<String>,
|
||||||
|
|
||||||
|
|
@ -284,7 +291,8 @@ struct Cli {
|
||||||
output_dir: Option<PathBuf>,
|
output_dir: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, ValueEnum)]
|
#[derive(Clone, Debug, ValueEnum, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
enum OutputFormat {
|
enum OutputFormat {
|
||||||
Markdown,
|
Markdown,
|
||||||
Json,
|
Json,
|
||||||
|
|
@ -293,14 +301,16 @@ enum OutputFormat {
|
||||||
Html,
|
Html,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, ValueEnum)]
|
#[derive(Clone, Debug, ValueEnum, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
enum Browser {
|
enum Browser {
|
||||||
Chrome,
|
Chrome,
|
||||||
Firefox,
|
Firefox,
|
||||||
Random,
|
Random,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, ValueEnum, Default)]
|
#[derive(Clone, Debug, ValueEnum, Default, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
enum PdfModeArg {
|
enum PdfModeArg {
|
||||||
/// Error if PDF has no extractable text (catches scanned PDFs)
|
/// Error if PDF has no extractable text (catches scanned PDFs)
|
||||||
#[default]
|
#[default]
|
||||||
|
|
@ -338,12 +348,21 @@ fn init_logging(verbose: bool) {
|
||||||
tracing_subscriber::fmt().with_env_filter(filter).init();
|
tracing_subscriber::fmt().with_env_filter(filter).init();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn init_mcp_logging() {
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_ansi(false)
|
||||||
|
.try_init()
|
||||||
|
.ok();
|
||||||
|
}
|
||||||
|
|
||||||
/// Build FetchConfig from CLI flags.
|
/// Build FetchConfig from CLI flags.
|
||||||
///
|
///
|
||||||
/// `--proxy` sets a single static proxy (no rotation).
|
/// `--proxy` sets a single static proxy (no rotation).
|
||||||
/// `--proxy-file` loads a pool of proxies and rotates per-request.
|
/// `--proxy-file` loads a pool of proxies and rotates per-request.
|
||||||
/// `--proxy` takes priority: if both are set, only the single proxy is used.
|
/// `--proxy` takes priority: if both are set, only the single proxy is used.
|
||||||
fn build_fetch_config(cli: &Cli) -> FetchConfig {
|
fn build_fetch_config(cli: &Cli, resolved: &config::ResolvedConfig) -> FetchConfig {
|
||||||
let (proxy, proxy_pool) = if cli.proxy.is_some() {
|
let (proxy, proxy_pool) = if cli.proxy.is_some() {
|
||||||
(cli.proxy.clone(), Vec::new())
|
(cli.proxy.clone(), Vec::new())
|
||||||
} else if let Some(ref path) = cli.proxy_file {
|
} else if let Some(ref path) = cli.proxy_file {
|
||||||
|
|
@ -403,11 +422,11 @@ fn build_fetch_config(cli: &Cli) -> FetchConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
FetchConfig {
|
FetchConfig {
|
||||||
browser: cli.browser.clone().into(),
|
browser: resolved.browser.clone().into(),
|
||||||
proxy,
|
proxy,
|
||||||
proxy_pool,
|
proxy_pool,
|
||||||
timeout: std::time::Duration::from_secs(cli.timeout),
|
timeout: std::time::Duration::from_secs(resolved.timeout),
|
||||||
pdf_mode: cli.pdf_mode.clone().into(),
|
pdf_mode: resolved.pdf_mode.clone().into(),
|
||||||
headers,
|
headers,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
|
|
@ -436,20 +455,12 @@ fn parse_cookie_file(path: &str) -> Result<String, String> {
|
||||||
Ok(pairs.join("; "))
|
Ok(pairs.join("; "))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_extraction_options(cli: &Cli) -> ExtractionOptions {
|
fn build_extraction_options(resolved: &config::ResolvedConfig) -> ExtractionOptions {
|
||||||
ExtractionOptions {
|
ExtractionOptions {
|
||||||
include_selectors: cli
|
include_selectors: resolved.include_selectors.clone(),
|
||||||
.include
|
exclude_selectors: resolved.exclude_selectors.clone(),
|
||||||
.as_deref()
|
only_main_content: resolved.only_main_content,
|
||||||
.map(|s| s.split(',').map(|s| s.trim().to_string()).collect())
|
include_raw_html: resolved.raw_html || matches!(resolved.format, OutputFormat::Html),
|
||||||
.unwrap_or_default(),
|
|
||||||
exclude_selectors: cli
|
|
||||||
.exclude
|
|
||||||
.as_deref()
|
|
||||||
.map(|s| s.split(',').map(|s| s.trim().to_string()).collect())
|
|
||||||
.unwrap_or_default(),
|
|
||||||
only_main_content: cli.only_main_content,
|
|
||||||
include_raw_html: cli.raw_html || matches!(cli.format, OutputFormat::Html),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -618,14 +629,17 @@ impl FetchOutput {
|
||||||
|
|
||||||
/// Fetch a URL and extract content, handling PDF detection automatically.
|
/// Fetch a URL and extract content, handling PDF detection automatically.
|
||||||
/// Falls back to cloud API when bot protection or JS rendering is detected.
|
/// Falls back to cloud API when bot protection or JS rendering is detected.
|
||||||
async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
async fn fetch_and_extract(
|
||||||
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
|
) -> Result<FetchOutput, String> {
|
||||||
// Local sources: read and extract as HTML
|
// Local sources: read and extract as HTML
|
||||||
if cli.stdin {
|
if cli.stdin {
|
||||||
let mut buf = String::new();
|
let mut buf = String::new();
|
||||||
io::stdin()
|
io::stdin()
|
||||||
.read_to_string(&mut buf)
|
.read_to_string(&mut buf)
|
||||||
.map_err(|e| format!("failed to read stdin: {e}"))?;
|
.map_err(|e| format!("failed to read stdin: {e}"))?;
|
||||||
let options = build_extraction_options(cli);
|
let options = build_extraction_options(resolved);
|
||||||
return extract_with_options(&buf, None, &options)
|
return extract_with_options(&buf, None, &options)
|
||||||
.map(|r| FetchOutput::Local(Box::new(r)))
|
.map(|r| FetchOutput::Local(Box::new(r)))
|
||||||
.map_err(|e| format!("extraction error: {e}"));
|
.map_err(|e| format!("extraction error: {e}"));
|
||||||
|
|
@ -634,7 +648,7 @@ async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
||||||
if let Some(ref path) = cli.file {
|
if let Some(ref path) = cli.file {
|
||||||
let html =
|
let html =
|
||||||
std::fs::read_to_string(path).map_err(|e| format!("failed to read {path}: {e}"))?;
|
std::fs::read_to_string(path).map_err(|e| format!("failed to read {path}: {e}"))?;
|
||||||
let options = build_extraction_options(cli);
|
let options = build_extraction_options(resolved);
|
||||||
return extract_with_options(&html, None, &options)
|
return extract_with_options(&html, None, &options)
|
||||||
.map(|r| FetchOutput::Local(Box::new(r)))
|
.map(|r| FetchOutput::Local(Box::new(r)))
|
||||||
.map_err(|e| format!("extraction error: {e}"));
|
.map_err(|e| format!("extraction error: {e}"));
|
||||||
|
|
@ -651,10 +665,9 @@ async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
||||||
|
|
||||||
// --cloud: skip local, go straight to cloud API
|
// --cloud: skip local, go straight to cloud API
|
||||||
if cli.cloud {
|
if cli.cloud {
|
||||||
let c =
|
let c = cloud_client.ok_or("--cloud requires NOXA_API_KEY (set via env or --api-key)")?;
|
||||||
cloud_client.ok_or("--cloud requires NOXA_API_KEY (set via env or --api-key)")?;
|
let options = build_extraction_options(resolved);
|
||||||
let options = build_extraction_options(cli);
|
let format_str = match resolved.format {
|
||||||
let format_str = match cli.format {
|
|
||||||
OutputFormat::Markdown => "markdown",
|
OutputFormat::Markdown => "markdown",
|
||||||
OutputFormat::Json => "json",
|
OutputFormat::Json => "json",
|
||||||
OutputFormat::Text => "text",
|
OutputFormat::Text => "text",
|
||||||
|
|
@ -674,9 +687,9 @@ async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normal path: try local first
|
// Normal path: try local first
|
||||||
let client =
|
let client = FetchClient::new(build_fetch_config(cli, resolved))
|
||||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?;
|
.map_err(|e| format!("client error: {e}"))?;
|
||||||
let options = build_extraction_options(cli);
|
let options = build_extraction_options(resolved);
|
||||||
let result = client
|
let result = client
|
||||||
.fetch_and_extract_with_options(url, &options)
|
.fetch_and_extract_with_options(url, &options)
|
||||||
.await
|
.await
|
||||||
|
|
@ -687,7 +700,7 @@ async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
||||||
if !matches!(reason, EmptyReason::None) {
|
if !matches!(reason, EmptyReason::None) {
|
||||||
if let Some(ref c) = cloud_client {
|
if let Some(ref c) = cloud_client {
|
||||||
eprintln!("\x1b[36minfo:\x1b[0m falling back to cloud API...");
|
eprintln!("\x1b[36minfo:\x1b[0m falling back to cloud API...");
|
||||||
let format_str = match cli.format {
|
let format_str = match resolved.format {
|
||||||
OutputFormat::Markdown => "markdown",
|
OutputFormat::Markdown => "markdown",
|
||||||
OutputFormat::Json => "json",
|
OutputFormat::Json => "json",
|
||||||
OutputFormat::Text => "text",
|
OutputFormat::Text => "text",
|
||||||
|
|
@ -718,7 +731,7 @@ async fn fetch_and_extract(cli: &Cli) -> Result<FetchOutput, String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch raw HTML from a URL (no extraction). Used for --raw-html and brand extraction.
|
/// Fetch raw HTML from a URL (no extraction). Used for --raw-html and brand extraction.
|
||||||
async fn fetch_html(cli: &Cli) -> Result<FetchResult, String> {
|
async fn fetch_html(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<FetchResult, String> {
|
||||||
if cli.stdin {
|
if cli.stdin {
|
||||||
let mut buf = String::new();
|
let mut buf = String::new();
|
||||||
io::stdin()
|
io::stdin()
|
||||||
|
|
@ -751,8 +764,8 @@ async fn fetch_html(cli: &Cli) -> Result<FetchResult, String> {
|
||||||
.ok_or("no input provided -- pass a URL, --file, or --stdin")?;
|
.ok_or("no input provided -- pass a URL, --file, or --stdin")?;
|
||||||
let url = normalize_url(raw_url);
|
let url = normalize_url(raw_url);
|
||||||
|
|
||||||
let client =
|
let client = FetchClient::new(build_fetch_config(cli, resolved))
|
||||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?;
|
.map_err(|e| format!("client error: {e}"))?;
|
||||||
client
|
client
|
||||||
.fetch(&url)
|
.fetch(&url)
|
||||||
.await
|
.await
|
||||||
|
|
@ -1166,7 +1179,7 @@ fn format_progress(page: &PageResult, index: usize, max_pages: usize) -> String
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
async fn run_crawl(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), String> {
|
||||||
let url = cli
|
let url = cli
|
||||||
.urls
|
.urls
|
||||||
.first()
|
.first()
|
||||||
|
|
@ -1178,16 +1191,8 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
||||||
return Err("--crawl cannot be used with --file or --stdin".into());
|
return Err("--crawl cannot be used with --file or --stdin".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let include_patterns: Vec<String> = cli
|
let include_patterns = resolved.include_paths.clone();
|
||||||
.include_paths
|
let exclude_patterns = resolved.exclude_paths.clone();
|
||||||
.as_deref()
|
|
||||||
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
|
||||||
.unwrap_or_default();
|
|
||||||
let exclude_patterns: Vec<String> = cli
|
|
||||||
.exclude_paths
|
|
||||||
.as_deref()
|
|
||||||
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
// Set up streaming progress channel
|
// Set up streaming progress channel
|
||||||
let (progress_tx, mut progress_rx) = tokio::sync::broadcast::channel::<PageResult>(100);
|
let (progress_tx, mut progress_rx) = tokio::sync::broadcast::channel::<PageResult>(100);
|
||||||
|
|
@ -1207,13 +1212,13 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
let config = CrawlConfig {
|
let config = CrawlConfig {
|
||||||
fetch: build_fetch_config(cli),
|
fetch: build_fetch_config(cli, resolved),
|
||||||
max_depth: cli.depth,
|
max_depth: resolved.depth,
|
||||||
max_pages: cli.max_pages,
|
max_pages: resolved.max_pages,
|
||||||
concurrency: cli.concurrency,
|
concurrency: resolved.concurrency,
|
||||||
delay: std::time::Duration::from_millis(cli.delay),
|
delay: std::time::Duration::from_millis(resolved.delay),
|
||||||
path_prefix: cli.path_prefix.clone(),
|
path_prefix: resolved.path_prefix.clone(),
|
||||||
use_sitemap: cli.sitemap,
|
use_sitemap: resolved.use_sitemap,
|
||||||
include_patterns,
|
include_patterns,
|
||||||
exclude_patterns,
|
exclude_patterns,
|
||||||
progress_tx: Some(progress_tx),
|
progress_tx: Some(progress_tx),
|
||||||
|
|
@ -1232,7 +1237,7 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
let max_pages = cli.max_pages;
|
let max_pages = resolved.max_pages;
|
||||||
let completed_offset = resume_state.as_ref().map_or(0, |s| s.completed_pages);
|
let completed_offset = resume_state.as_ref().map_or(0, |s| s.completed_pages);
|
||||||
|
|
||||||
// Spawn background task to print streaming progress to stderr
|
// Spawn background task to print streaming progress to stderr
|
||||||
|
|
@ -1261,8 +1266,8 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
||||||
&result.visited,
|
&result.visited,
|
||||||
&result.remaining_frontier,
|
&result.remaining_frontier,
|
||||||
completed_offset + result.pages.len(),
|
completed_offset + result.pages.len(),
|
||||||
cli.max_pages,
|
resolved.max_pages,
|
||||||
cli.depth,
|
resolved.depth,
|
||||||
)?;
|
)?;
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"Crawl state saved to {} ({} pages completed). Resume with --crawl-state {}",
|
"Crawl state saved to {} ({} pages completed). Resume with --crawl-state {}",
|
||||||
|
|
@ -1294,15 +1299,15 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
||||||
let mut saved = 0usize;
|
let mut saved = 0usize;
|
||||||
for page in &result.pages {
|
for page in &result.pages {
|
||||||
if let Some(ref extraction) = page.extraction {
|
if let Some(ref extraction) = page.extraction {
|
||||||
let filename = url_to_filename(&page.url, &cli.format);
|
let filename = url_to_filename(&page.url, &resolved.format);
|
||||||
let content = format_output(extraction, &cli.format, cli.metadata);
|
let content = format_output(extraction, &resolved.format, resolved.metadata);
|
||||||
write_to_file(dir, &filename, &content)?;
|
write_to_file(dir, &filename, &content)?;
|
||||||
saved += 1;
|
saved += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
eprintln!("Saved {saved} files to {}", dir.display());
|
eprintln!("Saved {saved} files to {}", dir.display());
|
||||||
} else {
|
} else {
|
||||||
print_crawl_output(&result, &cli.format, cli.metadata);
|
print_crawl_output(&result, &resolved.format, resolved.metadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
eprintln!(
|
eprintln!(
|
||||||
|
|
@ -1338,7 +1343,7 @@ async fn run_crawl(cli: &Cli) -> Result<(), String> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_map(cli: &Cli) -> Result<(), String> {
|
async fn run_map(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), String> {
|
||||||
let url = cli
|
let url = cli
|
||||||
.urls
|
.urls
|
||||||
.first()
|
.first()
|
||||||
|
|
@ -1346,8 +1351,8 @@ async fn run_map(cli: &Cli) -> Result<(), String> {
|
||||||
.map(|u| normalize_url(u))?;
|
.map(|u| normalize_url(u))?;
|
||||||
let url = url.as_str();
|
let url = url.as_str();
|
||||||
|
|
||||||
let client =
|
let client = FetchClient::new(build_fetch_config(cli, resolved))
|
||||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?;
|
.map_err(|e| format!("client error: {e}"))?;
|
||||||
|
|
||||||
let entries = noxa_fetch::sitemap::discover(&client, url)
|
let entries = noxa_fetch::sitemap::discover(&client, url)
|
||||||
.await
|
.await
|
||||||
|
|
@ -1359,19 +1364,24 @@ async fn run_map(cli: &Cli) -> Result<(), String> {
|
||||||
eprintln!("discovered {} URLs", entries.len());
|
eprintln!("discovered {} URLs", entries.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
print_map_output(&entries, &cli.format);
|
print_map_output(&entries, &resolved.format);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_batch(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<(), String> {
|
async fn run_batch(
|
||||||
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
|
entries: &[(String, Option<String>)],
|
||||||
|
) -> Result<(), String> {
|
||||||
let client = Arc::new(
|
let client = Arc::new(
|
||||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?,
|
FetchClient::new(build_fetch_config(cli, resolved))
|
||||||
|
.map_err(|e| format!("client error: {e}"))?,
|
||||||
);
|
);
|
||||||
|
|
||||||
let urls: Vec<&str> = entries.iter().map(|(u, _)| u.as_str()).collect();
|
let urls: Vec<&str> = entries.iter().map(|(u, _)| u.as_str()).collect();
|
||||||
let options = build_extraction_options(cli);
|
let options = build_extraction_options(resolved);
|
||||||
let results = client
|
let results = client
|
||||||
.fetch_and_extract_batch_with_options(&urls, cli.concurrency, &options)
|
.fetch_and_extract_batch_with_options(&urls, resolved.concurrency, &options)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let ok = results.iter().filter(|r| r.result.is_ok()).count();
|
let ok = results.iter().filter(|r| r.result.is_ok()).count();
|
||||||
|
|
@ -1402,15 +1412,15 @@ async fn run_batch(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<()
|
||||||
let filename = custom_names
|
let filename = custom_names
|
||||||
.get(r.url.as_str())
|
.get(r.url.as_str())
|
||||||
.map(|s| s.to_string())
|
.map(|s| s.to_string())
|
||||||
.unwrap_or_else(|| url_to_filename(&r.url, &cli.format));
|
.unwrap_or_else(|| url_to_filename(&r.url, &resolved.format));
|
||||||
let content = format_output(extraction, &cli.format, cli.metadata);
|
let content = format_output(extraction, &resolved.format, resolved.metadata);
|
||||||
write_to_file(dir, &filename, &content)?;
|
write_to_file(dir, &filename, &content)?;
|
||||||
saved += 1;
|
saved += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
eprintln!("Saved {saved} files to {}", dir.display());
|
eprintln!("Saved {saved} files to {}", dir.display());
|
||||||
} else {
|
} else {
|
||||||
print_batch_output(&results, &cli.format, cli.metadata);
|
print_batch_output(&results, &resolved.format, resolved.metadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
eprintln!(
|
eprintln!(
|
||||||
|
|
@ -1514,15 +1524,20 @@ fn fire_webhook(url: &str, payload: &serde_json::Value) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_watch(cli: &Cli, urls: &[String]) -> Result<(), String> {
|
async fn run_watch(
|
||||||
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
|
urls: &[String],
|
||||||
|
) -> Result<(), String> {
|
||||||
if urls.is_empty() {
|
if urls.is_empty() {
|
||||||
return Err("--watch requires at least one URL".into());
|
return Err("--watch requires at least one URL".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let client = Arc::new(
|
let client = Arc::new(
|
||||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?,
|
FetchClient::new(build_fetch_config(cli, resolved))
|
||||||
|
.map_err(|e| format!("client error: {e}"))?,
|
||||||
);
|
);
|
||||||
let options = build_extraction_options(cli);
|
let options = build_extraction_options(resolved);
|
||||||
|
|
||||||
// Ctrl+C handler
|
// Ctrl+C handler
|
||||||
let cancelled = Arc::new(AtomicBool::new(false));
|
let cancelled = Arc::new(AtomicBool::new(false));
|
||||||
|
|
@ -1534,16 +1549,17 @@ async fn run_watch(cli: &Cli, urls: &[String]) -> Result<(), String> {
|
||||||
|
|
||||||
// Single-URL mode: preserve original behavior exactly
|
// Single-URL mode: preserve original behavior exactly
|
||||||
if urls.len() == 1 {
|
if urls.len() == 1 {
|
||||||
return run_watch_single(cli, &client, &options, &urls[0], &cancelled).await;
|
return run_watch_single(cli, resolved, &client, &options, &urls[0], &cancelled).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Multi-URL mode: batch fetch, diff each, report aggregate
|
// Multi-URL mode: batch fetch, diff each, report aggregate
|
||||||
run_watch_multi(cli, &client, &options, urls, &cancelled).await
|
run_watch_multi(cli, resolved, &client, &options, urls, &cancelled).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Original single-URL watch loop -- backward compatible.
|
/// Original single-URL watch loop -- backward compatible.
|
||||||
async fn run_watch_single(
|
async fn run_watch_single(
|
||||||
cli: &Cli,
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
client: &Arc<FetchClient>,
|
client: &Arc<FetchClient>,
|
||||||
options: &ExtractionOptions,
|
options: &ExtractionOptions,
|
||||||
url: &str,
|
url: &str,
|
||||||
|
|
@ -1580,7 +1596,7 @@ async fn run_watch_single(
|
||||||
if diff.status == ChangeStatus::Same {
|
if diff.status == ChangeStatus::Same {
|
||||||
eprintln!("[watch] No changes ({})", timestamp());
|
eprintln!("[watch] No changes ({})", timestamp());
|
||||||
} else {
|
} else {
|
||||||
print_diff_output(&diff, &cli.format);
|
print_diff_output(&diff, &resolved.format);
|
||||||
eprintln!("[watch] Changes detected! ({})", timestamp());
|
eprintln!("[watch] Changes detected! ({})", timestamp());
|
||||||
|
|
||||||
if let Some(ref cmd) = cli.on_change {
|
if let Some(ref cmd) = cli.on_change {
|
||||||
|
|
@ -1627,6 +1643,7 @@ async fn run_watch_single(
|
||||||
/// Multi-URL watch loop -- batch fetch all URLs, diff each, report aggregate.
|
/// Multi-URL watch loop -- batch fetch all URLs, diff each, report aggregate.
|
||||||
async fn run_watch_multi(
|
async fn run_watch_multi(
|
||||||
cli: &Cli,
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
client: &Arc<FetchClient>,
|
client: &Arc<FetchClient>,
|
||||||
options: &ExtractionOptions,
|
options: &ExtractionOptions,
|
||||||
urls: &[String],
|
urls: &[String],
|
||||||
|
|
@ -1636,7 +1653,7 @@ async fn run_watch_multi(
|
||||||
|
|
||||||
// Initial pass: fetch all URLs in parallel
|
// Initial pass: fetch all URLs in parallel
|
||||||
let initial_results = client
|
let initial_results = client
|
||||||
.fetch_and_extract_batch_with_options(&url_refs, cli.concurrency, options)
|
.fetch_and_extract_batch_with_options(&url_refs, resolved.concurrency, options)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let mut snapshots = std::collections::HashMap::new();
|
let mut snapshots = std::collections::HashMap::new();
|
||||||
|
|
@ -1676,7 +1693,7 @@ async fn run_watch_multi(
|
||||||
check_number += 1;
|
check_number += 1;
|
||||||
|
|
||||||
let current_results = client
|
let current_results = client
|
||||||
.fetch_and_extract_batch_with_options(&url_refs, cli.concurrency, options)
|
.fetch_and_extract_batch_with_options(&url_refs, resolved.concurrency, options)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let mut changed: Vec<serde_json::Value> = Vec::new();
|
let mut changed: Vec<serde_json::Value> = Vec::new();
|
||||||
|
|
@ -1780,7 +1797,11 @@ async fn run_watch_multi(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_diff(cli: &Cli, snapshot_path: &str) -> Result<(), String> {
|
async fn run_diff(
|
||||||
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
|
snapshot_path: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
// Load previous snapshot
|
// Load previous snapshot
|
||||||
let snapshot_json = std::fs::read_to_string(snapshot_path)
|
let snapshot_json = std::fs::read_to_string(snapshot_path)
|
||||||
.map_err(|e| format!("failed to read snapshot {snapshot_path}: {e}"))?;
|
.map_err(|e| format!("failed to read snapshot {snapshot_path}: {e}"))?;
|
||||||
|
|
@ -1788,16 +1809,16 @@ async fn run_diff(cli: &Cli, snapshot_path: &str) -> Result<(), String> {
|
||||||
.map_err(|e| format!("failed to parse snapshot JSON: {e}"))?;
|
.map_err(|e| format!("failed to parse snapshot JSON: {e}"))?;
|
||||||
|
|
||||||
// Extract current version (handles PDF detection for URLs)
|
// Extract current version (handles PDF detection for URLs)
|
||||||
let new_result = fetch_and_extract(cli).await?.into_extraction()?;
|
let new_result = fetch_and_extract(cli, resolved).await?.into_extraction()?;
|
||||||
|
|
||||||
let diff = noxa_core::diff::diff(&old, &new_result);
|
let diff = noxa_core::diff::diff(&old, &new_result);
|
||||||
print_diff_output(&diff, &cli.format);
|
print_diff_output(&diff, &resolved.format);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_brand(cli: &Cli) -> Result<(), String> {
|
async fn run_brand(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), String> {
|
||||||
let result = fetch_html(cli).await?;
|
let result = fetch_html(cli, resolved).await?;
|
||||||
let enriched = enrich_html_with_stylesheets(&result.html, &result.url).await;
|
let enriched = enrich_html_with_stylesheets(&result.html, &result.url).await;
|
||||||
let brand = noxa_core::brand::extract_brand(
|
let brand = noxa_core::brand::extract_brand(
|
||||||
&enriched,
|
&enriched,
|
||||||
|
|
@ -1811,13 +1832,27 @@ async fn run_brand(cli: &Cli) -> Result<(), String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build an LLM provider based on CLI flags, or fall back to the default chain.
|
/// Build an LLM provider based on CLI flags, or fall back to the default chain.
|
||||||
async fn build_llm_provider(cli: &Cli) -> Result<Box<dyn LlmProvider>, String> {
|
async fn build_llm_provider(
|
||||||
if let Some(ref name) = cli.llm_provider {
|
cli: &Cli,
|
||||||
|
resolved: &config::ResolvedConfig,
|
||||||
|
) -> Result<Box<dyn LlmProvider>, String> {
|
||||||
|
if let Some(ref name) = resolved.llm_provider {
|
||||||
match name.as_str() {
|
match name.as_str() {
|
||||||
|
"gemini" => {
|
||||||
|
let provider = noxa_llm::providers::gemini_cli::GeminiCliProvider::new(
|
||||||
|
resolved.llm_model.clone(),
|
||||||
|
);
|
||||||
|
if !provider.is_available().await {
|
||||||
|
return Err(
|
||||||
|
"gemini CLI not found on PATH -- install it or omit --llm-provider".into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(Box::new(provider))
|
||||||
|
}
|
||||||
"ollama" => {
|
"ollama" => {
|
||||||
let provider = noxa_llm::providers::ollama::OllamaProvider::new(
|
let provider = noxa_llm::providers::ollama::OllamaProvider::new(
|
||||||
cli.llm_base_url.clone(),
|
cli.llm_base_url.clone(),
|
||||||
cli.llm_model.clone(),
|
resolved.llm_model.clone(),
|
||||||
);
|
);
|
||||||
if !provider.is_available().await {
|
if !provider.is_available().await {
|
||||||
return Err("ollama is not running or unreachable".into());
|
return Err("ollama is not running or unreachable".into());
|
||||||
|
|
@ -1828,7 +1863,7 @@ async fn build_llm_provider(cli: &Cli) -> Result<Box<dyn LlmProvider>, String> {
|
||||||
let provider = noxa_llm::providers::openai::OpenAiProvider::new(
|
let provider = noxa_llm::providers::openai::OpenAiProvider::new(
|
||||||
None,
|
None,
|
||||||
cli.llm_base_url.clone(),
|
cli.llm_base_url.clone(),
|
||||||
cli.llm_model.clone(),
|
resolved.llm_model.clone(),
|
||||||
)
|
)
|
||||||
.ok_or("OPENAI_API_KEY not set")?;
|
.ok_or("OPENAI_API_KEY not set")?;
|
||||||
Ok(Box::new(provider))
|
Ok(Box::new(provider))
|
||||||
|
|
@ -1836,20 +1871,20 @@ async fn build_llm_provider(cli: &Cli) -> Result<Box<dyn LlmProvider>, String> {
|
||||||
"anthropic" => {
|
"anthropic" => {
|
||||||
let provider = noxa_llm::providers::anthropic::AnthropicProvider::new(
|
let provider = noxa_llm::providers::anthropic::AnthropicProvider::new(
|
||||||
None,
|
None,
|
||||||
cli.llm_model.clone(),
|
resolved.llm_model.clone(),
|
||||||
)
|
)
|
||||||
.ok_or("ANTHROPIC_API_KEY not set")?;
|
.ok_or("ANTHROPIC_API_KEY not set")?;
|
||||||
Ok(Box::new(provider))
|
Ok(Box::new(provider))
|
||||||
}
|
}
|
||||||
other => Err(format!(
|
other => Err(format!(
|
||||||
"unknown LLM provider: {other} (use ollama, openai, or anthropic)"
|
"unknown LLM provider: {other} (use gemini, ollama, openai, or anthropic)"
|
||||||
)),
|
)),
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
let chain = noxa_llm::ProviderChain::default().await;
|
let chain = noxa_llm::ProviderChain::default().await;
|
||||||
if chain.is_empty() {
|
if chain.is_empty() {
|
||||||
return Err(
|
return Err(
|
||||||
"no LLM providers available -- start Ollama or set OPENAI_API_KEY / ANTHROPIC_API_KEY"
|
"no LLM providers available -- install the gemini CLI, start Ollama, or set OPENAI_API_KEY / ANTHROPIC_API_KEY"
|
||||||
.into(),
|
.into(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -1857,12 +1892,12 @@ async fn build_llm_provider(cli: &Cli) -> Result<Box<dyn LlmProvider>, String> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_llm(cli: &Cli) -> Result<(), String> {
|
async fn run_llm(cli: &Cli, resolved: &config::ResolvedConfig) -> Result<(), String> {
|
||||||
// Extract content from source first (handles PDF detection for URLs)
|
// Extract content from source first (handles PDF detection for URLs)
|
||||||
let result = fetch_and_extract(cli).await?.into_extraction()?;
|
let result = fetch_and_extract(cli, resolved).await?.into_extraction()?;
|
||||||
|
|
||||||
let provider = build_llm_provider(cli).await?;
|
let provider = build_llm_provider(cli, resolved).await?;
|
||||||
let model = cli.llm_model.as_deref();
|
let model = resolved.llm_model.as_deref();
|
||||||
|
|
||||||
if let Some(ref schema_input) = cli.extract_json {
|
if let Some(ref schema_input) = cli.extract_json {
|
||||||
// Support @file syntax for loading schema from file
|
// Support @file syntax for loading schema from file
|
||||||
|
|
@ -1876,6 +1911,7 @@ async fn run_llm(cli: &Cli) -> Result<(), String> {
|
||||||
let schema: serde_json::Value =
|
let schema: serde_json::Value =
|
||||||
serde_json::from_str(&schema_str).map_err(|e| format!("invalid JSON schema: {e}"))?;
|
serde_json::from_str(&schema_str).map_err(|e| format!("invalid JSON schema: {e}"))?;
|
||||||
|
|
||||||
|
let t = std::time::Instant::now();
|
||||||
let extracted = noxa_llm::extract::extract_json(
|
let extracted = noxa_llm::extract::extract_json(
|
||||||
&result.content.plain_text,
|
&result.content.plain_text,
|
||||||
&schema,
|
&schema,
|
||||||
|
|
@ -1884,12 +1920,14 @@ async fn run_llm(cli: &Cli) -> Result<(), String> {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("LLM extraction failed: {e}"))?;
|
.map_err(|e| format!("LLM extraction failed: {e}"))?;
|
||||||
|
eprintln!("LLM: {:.1}s", t.elapsed().as_secs_f64());
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
serde_json::to_string_pretty(&extracted).expect("serialization failed")
|
serde_json::to_string_pretty(&extracted).expect("serialization failed")
|
||||||
);
|
);
|
||||||
} else if let Some(ref prompt) = cli.extract_prompt {
|
} else if let Some(ref prompt) = cli.extract_prompt {
|
||||||
|
let t = std::time::Instant::now();
|
||||||
let extracted = noxa_llm::extract::extract_with_prompt(
|
let extracted = noxa_llm::extract::extract_with_prompt(
|
||||||
&result.content.plain_text,
|
&result.content.plain_text,
|
||||||
prompt,
|
prompt,
|
||||||
|
|
@ -1898,12 +1936,14 @@ async fn run_llm(cli: &Cli) -> Result<(), String> {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("LLM extraction failed: {e}"))?;
|
.map_err(|e| format!("LLM extraction failed: {e}"))?;
|
||||||
|
eprintln!("LLM: {:.1}s", t.elapsed().as_secs_f64());
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
serde_json::to_string_pretty(&extracted).expect("serialization failed")
|
serde_json::to_string_pretty(&extracted).expect("serialization failed")
|
||||||
);
|
);
|
||||||
} else if let Some(sentences) = cli.summarize {
|
} else if let Some(sentences) = cli.summarize {
|
||||||
|
let t = std::time::Instant::now();
|
||||||
let summary = noxa_llm::summarize::summarize(
|
let summary = noxa_llm::summarize::summarize(
|
||||||
&result.content.plain_text,
|
&result.content.plain_text,
|
||||||
Some(sentences),
|
Some(sentences),
|
||||||
|
|
@ -1912,6 +1952,7 @@ async fn run_llm(cli: &Cli) -> Result<(), String> {
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("LLM summarization failed: {e}"))?;
|
.map_err(|e| format!("LLM summarization failed: {e}"))?;
|
||||||
|
eprintln!("LLM: {:.1}s", t.elapsed().as_secs_f64());
|
||||||
|
|
||||||
println!("{summary}");
|
println!("{summary}");
|
||||||
}
|
}
|
||||||
|
|
@ -1921,12 +1962,16 @@ async fn run_llm(cli: &Cli) -> Result<(), String> {
|
||||||
|
|
||||||
/// Batch LLM extraction: fetch each URL, run LLM on extracted content, save/print results.
|
/// Batch LLM extraction: fetch each URL, run LLM on extracted content, save/print results.
|
||||||
/// URLs are processed sequentially to respect LLM provider rate limits.
|
/// URLs are processed sequentially to respect LLM provider rate limits.
|
||||||
async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Result<(), String> {
|
async fn run_batch_llm(
|
||||||
let client =
|
cli: &Cli,
|
||||||
FetchClient::new(build_fetch_config(cli)).map_err(|e| format!("client error: {e}"))?;
|
resolved: &config::ResolvedConfig,
|
||||||
let options = build_extraction_options(cli);
|
entries: &[(String, Option<String>)],
|
||||||
let provider = build_llm_provider(cli).await?;
|
) -> Result<(), String> {
|
||||||
let model = cli.llm_model.as_deref();
|
let client = FetchClient::new(build_fetch_config(cli, resolved))
|
||||||
|
.map_err(|e| format!("client error: {e}"))?;
|
||||||
|
let options = build_extraction_options(resolved);
|
||||||
|
let provider = build_llm_provider(cli, resolved).await?;
|
||||||
|
let model = resolved.llm_model.as_deref();
|
||||||
|
|
||||||
// Pre-parse schema once if --extract-json is used
|
// Pre-parse schema once if --extract-json is used
|
||||||
let schema = if let Some(ref schema_input) = cli.extract_json {
|
let schema = if let Some(ref schema_input) = cli.extract_json {
|
||||||
|
|
@ -1974,6 +2019,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
|
||||||
let text = &extraction.content.plain_text;
|
let text = &extraction.content.plain_text;
|
||||||
|
|
||||||
// Run the appropriate LLM operation
|
// Run the appropriate LLM operation
|
||||||
|
let llm_start = std::time::Instant::now();
|
||||||
let llm_result = if let Some(ref schema) = schema {
|
let llm_result = if let Some(ref schema) = schema {
|
||||||
noxa_llm::extract::extract_json(text, schema, provider.as_ref(), model)
|
noxa_llm::extract::extract_json(text, schema, provider.as_ref(), model)
|
||||||
.await
|
.await
|
||||||
|
|
@ -1989,6 +2035,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
|
||||||
} else {
|
} else {
|
||||||
unreachable!("run_batch_llm called without LLM flags")
|
unreachable!("run_batch_llm called without LLM flags")
|
||||||
};
|
};
|
||||||
|
let llm_elapsed = llm_start.elapsed();
|
||||||
|
|
||||||
match llm_result {
|
match llm_result {
|
||||||
Ok(output) => {
|
Ok(output) => {
|
||||||
|
|
@ -2018,7 +2065,7 @@ async fn run_batch_llm(cli: &Cli, entries: &[(String, Option<String>)]) -> Resul
|
||||||
format!("{words} words")
|
format!("{words} words")
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
eprintln!("-> extracted {detail}");
|
eprintln!("-> extracted {detail} ({:.1}s)", llm_elapsed.as_secs_f64());
|
||||||
|
|
||||||
if let Some(ref dir) = cli.output_dir {
|
if let Some(ref dir) = cli.output_dir {
|
||||||
let filename = custom_names
|
let filename = custom_names
|
||||||
|
|
@ -2215,12 +2262,29 @@ async fn run_research(cli: &Cli, query: &str) -> Result<(), String> {
|
||||||
async fn main() {
|
async fn main() {
|
||||||
dotenvy::dotenv().ok();
|
dotenvy::dotenv().ok();
|
||||||
|
|
||||||
let cli = Cli::parse();
|
if matches!(std::env::args().nth(1).as_deref(), Some("mcp")) {
|
||||||
init_logging(cli.verbose);
|
init_mcp_logging();
|
||||||
|
|
||||||
|
if let Err(e) = noxa_mcp::run().await {
|
||||||
|
eprintln!("error: {e}");
|
||||||
|
process::exit(1);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use low-level API to get both typed Cli and ArgMatches for ValueSource detection.
|
||||||
|
let matches = Cli::command().get_matches();
|
||||||
|
let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit());
|
||||||
|
|
||||||
|
// Load config BEFORE init_logging so verbose from config takes effect.
|
||||||
|
let cfg = config::NoxaConfig::load(cli.config.as_deref());
|
||||||
|
let resolved = config::resolve(&cli, &matches, &cfg);
|
||||||
|
|
||||||
|
init_logging(resolved.verbose);
|
||||||
|
|
||||||
// --map: sitemap discovery mode
|
// --map: sitemap discovery mode
|
||||||
if cli.map {
|
if cli.map {
|
||||||
if let Err(e) = run_map(&cli).await {
|
if let Err(e) = run_map(&cli, &resolved).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2229,7 +2293,7 @@ async fn main() {
|
||||||
|
|
||||||
// --crawl: recursive crawl mode
|
// --crawl: recursive crawl mode
|
||||||
if cli.crawl {
|
if cli.crawl {
|
||||||
if let Err(e) = run_crawl(&cli).await {
|
if let Err(e) = run_crawl(&cli, &resolved).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2245,7 +2309,7 @@ async fn main() {
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Err(e) = run_watch(&cli, &watch_urls).await {
|
if let Err(e) = run_watch(&cli, &resolved, &watch_urls).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2254,7 +2318,7 @@ async fn main() {
|
||||||
|
|
||||||
// --diff-with: change tracking mode
|
// --diff-with: change tracking mode
|
||||||
if let Some(ref snapshot_path) = cli.diff_with {
|
if let Some(ref snapshot_path) = cli.diff_with {
|
||||||
if let Err(e) = run_diff(&cli, snapshot_path).await {
|
if let Err(e) = run_diff(&cli, &resolved, snapshot_path).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2263,7 +2327,7 @@ async fn main() {
|
||||||
|
|
||||||
// --brand: brand identity extraction mode
|
// --brand: brand identity extraction mode
|
||||||
if cli.brand {
|
if cli.brand {
|
||||||
if let Err(e) = run_brand(&cli).await {
|
if let Err(e) = run_brand(&cli, &resolved).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2292,11 +2356,11 @@ async fn main() {
|
||||||
// When multiple URLs are provided, run batch LLM extraction over all of them.
|
// When multiple URLs are provided, run batch LLM extraction over all of them.
|
||||||
if has_llm_flags(&cli) {
|
if has_llm_flags(&cli) {
|
||||||
if entries.len() > 1 {
|
if entries.len() > 1 {
|
||||||
if let Err(e) = run_batch_llm(&cli, &entries).await {
|
if let Err(e) = run_batch_llm(&cli, &resolved, &entries).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
} else if let Err(e) = run_llm(&cli).await {
|
} else if let Err(e) = run_llm(&cli, &resolved).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2305,7 +2369,7 @@ async fn main() {
|
||||||
|
|
||||||
// Multi-URL batch mode
|
// Multi-URL batch mode
|
||||||
if entries.len() > 1 {
|
if entries.len() > 1 {
|
||||||
if let Err(e) = run_batch(&cli, &entries).await {
|
if let Err(e) = run_batch(&cli, &resolved, &entries).await {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -2313,8 +2377,11 @@ async fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// --raw-html: skip extraction, dump the fetched HTML
|
// --raw-html: skip extraction, dump the fetched HTML
|
||||||
if cli.raw_html && cli.include.is_none() && cli.exclude.is_none() {
|
if resolved.raw_html
|
||||||
match fetch_html(&cli).await {
|
&& resolved.include_selectors.is_empty()
|
||||||
|
&& resolved.exclude_selectors.is_empty()
|
||||||
|
{
|
||||||
|
match fetch_html(&cli, &resolved).await {
|
||||||
Ok(r) => println!("{}", r.html),
|
Ok(r) => println!("{}", r.html),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
|
|
@ -2325,7 +2392,7 @@ async fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Single-page extraction (handles both HTML and PDF via content-type detection)
|
// Single-page extraction (handles both HTML and PDF via content-type detection)
|
||||||
match fetch_and_extract(&cli).await {
|
match fetch_and_extract(&cli, &resolved).await {
|
||||||
Ok(FetchOutput::Local(result)) => {
|
Ok(FetchOutput::Local(result)) => {
|
||||||
if let Some(ref dir) = cli.output_dir {
|
if let Some(ref dir) = cli.output_dir {
|
||||||
let url = cli
|
let url = cli
|
||||||
|
|
@ -2334,18 +2401,19 @@ async fn main() {
|
||||||
.map(|u| normalize_url(u))
|
.map(|u| normalize_url(u))
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let custom_name = entries.first().and_then(|(_, name)| name.clone());
|
let custom_name = entries.first().and_then(|(_, name)| name.clone());
|
||||||
let filename = custom_name.unwrap_or_else(|| url_to_filename(&url, &cli.format));
|
let filename =
|
||||||
let content = format_output(&result, &cli.format, cli.metadata);
|
custom_name.unwrap_or_else(|| url_to_filename(&url, &resolved.format));
|
||||||
|
let content = format_output(&result, &resolved.format, resolved.metadata);
|
||||||
if let Err(e) = write_to_file(dir, &filename, &content) {
|
if let Err(e) = write_to_file(dir, &filename, &content) {
|
||||||
eprintln!("error: {e}");
|
eprintln!("error: {e}");
|
||||||
process::exit(1);
|
process::exit(1);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
print_output(&result, &cli.format, cli.metadata);
|
print_output(&result, &resolved.format, resolved.metadata);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(FetchOutput::Cloud(resp)) => {
|
Ok(FetchOutput::Cloud(resp)) => {
|
||||||
print_cloud_output(&resp, &cli.format);
|
print_cloud_output(&resp, &resolved.format);
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("{e}");
|
eprintln!("{e}");
|
||||||
|
|
@ -2456,3 +2524,28 @@ mod tests {
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod enum_deserialize_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_output_format_deserialize() {
|
||||||
|
let f: OutputFormat = serde_json::from_str("\"llm\"").unwrap();
|
||||||
|
assert!(matches!(f, OutputFormat::Llm));
|
||||||
|
let f: OutputFormat = serde_json::from_str("\"markdown\"").unwrap();
|
||||||
|
assert!(matches!(f, OutputFormat::Markdown));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_browser_deserialize() {
|
||||||
|
let b: Browser = serde_json::from_str("\"firefox\"").unwrap();
|
||||||
|
assert!(matches!(b, Browser::Firefox));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_pdf_mode_deserialize() {
|
||||||
|
let p: PdfModeArg = serde_json::from_str("\"fast\"").unwrap();
|
||||||
|
assert!(matches!(p, PdfModeArg::Fast));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ license.workspace = true
|
||||||
[dependencies]
|
[dependencies]
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
|
jsonschema = { version = "0.46", default-features = false }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,15 @@
|
||||||
/// Default order: Ollama (local, free) -> OpenAI -> Anthropic.
|
/// Default order: Ollama (local, free) -> OpenAI -> Anthropic.
|
||||||
/// Only includes providers that are actually configured/available.
|
/// Only includes providers that are actually configured/available.
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::error::LlmError;
|
use crate::error::LlmError;
|
||||||
use crate::provider::{CompletionRequest, LlmProvider};
|
use crate::provider::{CompletionRequest, LlmProvider};
|
||||||
use crate::providers::{
|
use crate::providers::{
|
||||||
anthropic::AnthropicProvider, ollama::OllamaProvider, openai::OpenAiProvider,
|
anthropic::AnthropicProvider,
|
||||||
|
gemini_cli::GeminiCliProvider,
|
||||||
|
ollama::OllamaProvider,
|
||||||
|
openai::OpenAiProvider,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub struct ProviderChain {
|
pub struct ProviderChain {
|
||||||
|
|
@ -15,12 +18,26 @@ pub struct ProviderChain {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProviderChain {
|
impl ProviderChain {
|
||||||
/// Build the default chain: Ollama -> OpenAI -> Anthropic.
|
/// Build the default chain: Gemini CLI -> OpenAI -> Ollama -> Anthropic.
|
||||||
/// Ollama is always added (availability checked at call time).
|
/// Gemini CLI is the primary backend (subprocess-based, requires `gemini` on PATH).
|
||||||
/// Cloud providers are only added if their API keys are configured.
|
/// Cloud providers are only added if their API keys are configured.
|
||||||
|
/// Ollama is added if reachable at call time.
|
||||||
pub async fn default() -> Self {
|
pub async fn default() -> Self {
|
||||||
let mut providers: Vec<Box<dyn LlmProvider>> = Vec::new();
|
let mut providers: Vec<Box<dyn LlmProvider>> = Vec::new();
|
||||||
|
|
||||||
|
let gemini = GeminiCliProvider::new(None);
|
||||||
|
if gemini.is_available().await {
|
||||||
|
debug!("gemini cli available, adding as primary provider");
|
||||||
|
providers.push(Box::new(gemini));
|
||||||
|
} else {
|
||||||
|
debug!("gemini cli not found on PATH, skipping");
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(openai) = OpenAiProvider::new(None, None, None) {
|
||||||
|
debug!("openai configured, adding to chain");
|
||||||
|
providers.push(Box::new(openai));
|
||||||
|
}
|
||||||
|
|
||||||
let ollama = OllamaProvider::new(None, None);
|
let ollama = OllamaProvider::new(None, None);
|
||||||
if ollama.is_available().await {
|
if ollama.is_available().await {
|
||||||
debug!("ollama is available, adding to chain");
|
debug!("ollama is available, adding to chain");
|
||||||
|
|
@ -29,11 +46,6 @@ impl ProviderChain {
|
||||||
debug!("ollama not available, skipping");
|
debug!("ollama not available, skipping");
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(openai) = OpenAiProvider::new(None, None, None) {
|
|
||||||
debug!("openai configured, adding to chain");
|
|
||||||
providers.push(Box::new(openai));
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(anthropic) = AnthropicProvider::new(None, None) {
|
if let Some(anthropic) = AnthropicProvider::new(None, None) {
|
||||||
debug!("anthropic configured, adding to chain");
|
debug!("anthropic configured, adding to chain");
|
||||||
providers.push(Box::new(anthropic));
|
providers.push(Box::new(anthropic));
|
||||||
|
|
@ -79,9 +91,10 @@ impl LlmProvider for ProviderChain {
|
||||||
for provider in &self.providers {
|
for provider in &self.providers {
|
||||||
debug!(provider = provider.name(), "attempting completion");
|
debug!(provider = provider.name(), "attempting completion");
|
||||||
|
|
||||||
|
let t = std::time::Instant::now();
|
||||||
match provider.complete(request).await {
|
match provider.complete(request).await {
|
||||||
Ok(response) => {
|
Ok(response) => {
|
||||||
debug!(provider = provider.name(), "completion succeeded");
|
info!(provider = provider.name(), elapsed_ms = t.elapsed().as_millis(), "completion succeeded");
|
||||||
return Ok(response);
|
return Ok(response);
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -202,4 +215,46 @@ mod tests {
|
||||||
assert_eq!(chain.len(), 2);
|
assert_eq!(chain.len(), 2);
|
||||||
assert!(!chain.is_empty());
|
assert!(!chain.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Gemini-first chain ordering ───────────────────────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gemini_first_in_single_provider_chain() {
|
||||||
|
// When we build a chain with a mock "gemini" provider first, it should
|
||||||
|
// be used before any fallback.
|
||||||
|
let chain = ProviderChain::from_providers(vec![
|
||||||
|
Box::new(MockProvider {
|
||||||
|
name: "gemini",
|
||||||
|
response: Ok("from gemini".into()),
|
||||||
|
available: true,
|
||||||
|
}),
|
||||||
|
Box::new(MockProvider {
|
||||||
|
name: "openai",
|
||||||
|
response: Ok("from openai".into()),
|
||||||
|
available: true,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
let result = chain.complete(&test_request()).await.unwrap();
|
||||||
|
assert_eq!(result, "from gemini");
|
||||||
|
// Confirm order: first provider name is "gemini"
|
||||||
|
assert_eq!(chain.providers[0].name(), "gemini");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gemini_failure_falls_back_to_openai() {
|
||||||
|
let chain = ProviderChain::from_providers(vec![
|
||||||
|
Box::new(MockProvider {
|
||||||
|
name: "gemini",
|
||||||
|
response: Err("subprocess timed out".into()),
|
||||||
|
available: true,
|
||||||
|
}),
|
||||||
|
Box::new(MockProvider {
|
||||||
|
name: "openai",
|
||||||
|
response: Ok("from openai".into()),
|
||||||
|
available: true,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
let result = chain.complete(&test_request()).await.unwrap();
|
||||||
|
assert_eq!(result, "from openai");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,12 @@ pub enum LlmError {
|
||||||
#[error("HTTP error: {0}")]
|
#[error("HTTP error: {0}")]
|
||||||
Http(#[from] reqwest::Error),
|
Http(#[from] reqwest::Error),
|
||||||
|
|
||||||
|
#[error("subprocess error: {0}")]
|
||||||
|
Subprocess(#[from] std::io::Error),
|
||||||
|
|
||||||
|
#[error("subprocess timed out")]
|
||||||
|
Timeout,
|
||||||
|
|
||||||
#[error("no providers available")]
|
#[error("no providers available")]
|
||||||
NoProviders,
|
NoProviders,
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,45 @@
|
||||||
/// Schema-based and prompt-based LLM extraction.
|
/// Schema-based and prompt-based LLM extraction.
|
||||||
/// Both functions build a system prompt, send content to the LLM, and parse JSON back.
|
/// Both functions build a system prompt, send content to the LLM, and parse JSON back.
|
||||||
|
use jsonschema;
|
||||||
|
|
||||||
use crate::clean::strip_thinking_tags;
|
use crate::clean::strip_thinking_tags;
|
||||||
use crate::error::LlmError;
|
use crate::error::LlmError;
|
||||||
use crate::provider::{CompletionRequest, LlmProvider, Message};
|
use crate::provider::{CompletionRequest, LlmProvider, Message};
|
||||||
|
|
||||||
|
/// Validate a JSON value against a schema. Returns Ok(()) on success or
|
||||||
|
/// Err(LlmError::InvalidJson) with a concise error message on failure.
|
||||||
|
fn validate_schema(
|
||||||
|
value: &serde_json::Value,
|
||||||
|
schema: &serde_json::Value,
|
||||||
|
) -> Result<(), LlmError> {
|
||||||
|
let compiled = jsonschema::validator_for(schema).map_err(|e| {
|
||||||
|
LlmError::InvalidJson(format!("invalid schema: {e}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let errors: Vec<String> = compiled
|
||||||
|
.iter_errors(value)
|
||||||
|
.map(|e| format!("{} at {}", e, e.instance_path()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(LlmError::InvalidJson(format!(
|
||||||
|
"schema validation failed: {}",
|
||||||
|
errors.join("; ")
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Extract structured JSON from content using a JSON schema.
|
/// Extract structured JSON from content using a JSON schema.
|
||||||
/// The schema tells the LLM exactly what fields to extract and their types.
|
/// The schema tells the LLM exactly what fields to extract and their types.
|
||||||
|
///
|
||||||
|
/// Retry policy:
|
||||||
|
/// - If the response cannot be parsed as JSON at all: retry once with the
|
||||||
|
/// identical request (handles transient formatting issues).
|
||||||
|
/// - If the response is valid JSON but fails schema validation: return
|
||||||
|
/// `LlmError::InvalidJson` immediately — the schema is likely unsatisfiable
|
||||||
|
/// for this content, so retrying would produce the same result.
|
||||||
pub async fn extract_json(
|
pub async fn extract_json(
|
||||||
content: &str,
|
content: &str,
|
||||||
schema: &serde_json::Value,
|
schema: &serde_json::Value,
|
||||||
|
|
@ -37,7 +71,22 @@ pub async fn extract_json(
|
||||||
};
|
};
|
||||||
|
|
||||||
let response = provider.complete(&request).await?;
|
let response = provider.complete(&request).await?;
|
||||||
parse_json_response(&response)
|
|
||||||
|
match parse_json_response(&response) {
|
||||||
|
Ok(value) => {
|
||||||
|
// Valid JSON — now validate against the schema.
|
||||||
|
// Schema mismatches do not retry (unsatisfiable → same result).
|
||||||
|
validate_schema(&value, schema)?;
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
Err(_parse_err) => {
|
||||||
|
// Unparseable JSON — retry once with the identical request.
|
||||||
|
let retry_response = provider.complete(&request).await?;
|
||||||
|
let value = parse_json_response(&retry_response)?;
|
||||||
|
validate_schema(&value, schema)?;
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract information using a natural language prompt.
|
/// Extract information using a natural language prompt.
|
||||||
|
|
@ -184,4 +233,130 @@ mod tests {
|
||||||
|
|
||||||
assert_eq!(result["emails"][0], "test@example.com");
|
assert_eq!(result["emails"][0], "test@example.com");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Schema validation ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn schema_validation_passes_for_matching_json() {
|
||||||
|
let schema = serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["price"],
|
||||||
|
"properties": {
|
||||||
|
"price": { "type": "number" }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let mock = MockProvider::ok(r#"{"price": 9.99}"#);
|
||||||
|
let result = extract_json("content", &schema, &mock, None).await.unwrap();
|
||||||
|
assert_eq!(result["price"], 9.99);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn schema_validation_fails_for_wrong_type() {
|
||||||
|
let schema = serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["price"],
|
||||||
|
"properties": {
|
||||||
|
"price": { "type": "number" }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// Model returns valid JSON but wrong type ("string" instead of number).
|
||||||
|
// Should NOT retry (schema mismatch ≠ parse failure) — returns InvalidJson immediately.
|
||||||
|
let mock = MockProvider::ok(r#"{"price": "not-a-number"}"#);
|
||||||
|
let result = extract_json("content", &schema, &mock, None).await;
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(LlmError::InvalidJson(_))),
|
||||||
|
"expected InvalidJson for schema mismatch, got {result:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn schema_validation_fails_for_missing_required_field() {
|
||||||
|
let schema = serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["title"],
|
||||||
|
"properties": {
|
||||||
|
"title": { "type": "string" }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let mock = MockProvider::ok(r#"{"other": "value"}"#);
|
||||||
|
let result = extract_json("content", &schema, &mock, None).await;
|
||||||
|
assert!(matches!(result, Err(LlmError::InvalidJson(_))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn parse_failure_triggers_one_retry() {
|
||||||
|
use crate::testing::mock::SequenceMockProvider;
|
||||||
|
|
||||||
|
let schema = serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": { "title": { "type": "string" } }
|
||||||
|
});
|
||||||
|
|
||||||
|
// First call: unparseable JSON. Second call: valid JSON matching schema.
|
||||||
|
let mock = SequenceMockProvider::new(
|
||||||
|
"mock-seq",
|
||||||
|
vec![
|
||||||
|
Ok("this is not json at all".to_string()),
|
||||||
|
Ok(r#"{"title": "Retry succeeded"}"#.to_string()),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = extract_json("content", &schema, &mock, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result["title"], "Retry succeeded");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn both_attempts_fail_returns_invalid_json() {
|
||||||
|
use crate::testing::mock::SequenceMockProvider;
|
||||||
|
|
||||||
|
let schema = serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": { "title": { "type": "string" } }
|
||||||
|
});
|
||||||
|
|
||||||
|
let mock = SequenceMockProvider::new(
|
||||||
|
"mock-seq",
|
||||||
|
vec![
|
||||||
|
Ok("not json".to_string()),
|
||||||
|
Ok("also not json".to_string()),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = extract_json("content", &schema, &mock, None).await;
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(LlmError::InvalidJson(_))),
|
||||||
|
"expected InvalidJson after both attempts fail"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn schema_mismatch_does_not_retry() {
|
||||||
|
use crate::testing::mock::SequenceMockProvider;
|
||||||
|
|
||||||
|
let schema = serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"required": ["price"],
|
||||||
|
"properties": {
|
||||||
|
"price": { "type": "number" }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Both calls return valid JSON with wrong schema — but only one call should happen.
|
||||||
|
let mock = SequenceMockProvider::new(
|
||||||
|
"mock-seq",
|
||||||
|
vec![
|
||||||
|
Ok(r#"{"price": "wrong-type"}"#.to_string()),
|
||||||
|
Ok(r#"{"price": 9.99}"#.to_string()), // would succeed — but shouldn't be called
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Should return InvalidJson without calling second response.
|
||||||
|
let result = extract_json("content", &schema, &mock, None).await;
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(LlmError::InvalidJson(_))),
|
||||||
|
"schema mismatch should not trigger retry"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
/// noxa-llm: LLM integration with local-first hybrid architecture.
|
/// noxa-llm: LLM integration with Gemini-CLI-first hybrid architecture.
|
||||||
///
|
///
|
||||||
/// Provider chain tries Ollama (local) first, falls back to OpenAI, then Anthropic.
|
/// Provider chain: Gemini CLI (primary) → OpenAI → Ollama → Anthropic.
|
||||||
/// Provides schema-based extraction, prompt extraction, and summarization
|
/// Gemini CLI requires the `gemini` binary on PATH; GEMINI_MODEL env var sets the model.
|
||||||
/// on top of noxa-core's content pipeline.
|
/// Provides schema-validated extraction (with one retry on parse failure),
|
||||||
|
/// prompt extraction, and summarization on top of noxa-core's content pipeline.
|
||||||
pub mod chain;
|
pub mod chain;
|
||||||
pub mod clean;
|
pub mod clean;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
|
|
|
||||||
392
crates/noxa-llm/src/providers/gemini_cli.rs
Normal file
392
crates/noxa-llm/src/providers/gemini_cli.rs
Normal file
|
|
@ -0,0 +1,392 @@
|
||||||
|
/// Gemini CLI provider — shells out to `gemini -p` for completions.
|
||||||
|
/// Primary provider in the default chain; requires the `gemini` binary on PATH.
|
||||||
|
///
|
||||||
|
/// Prompts are passed via the `-p` flag (not via stdin or as a positional) to prevent
|
||||||
|
/// command injection from web-scraped content. Output is parsed from `--output-format json`.
|
||||||
|
///
|
||||||
|
/// # Startup optimizations
|
||||||
|
///
|
||||||
|
/// The gemini CLI is an agentic Node.js application that connects to every configured MCP
|
||||||
|
/// server at startup (the user has 6). Without mitigation this can add 10-60+ seconds per
|
||||||
|
/// call as those servers spin up and time out.
|
||||||
|
///
|
||||||
|
/// Two flags reduce this:
|
||||||
|
/// - `--extensions ""` — skips extension loading (~3 s saved)
|
||||||
|
/// - `current_dir` set to a temp workdir containing `.gemini/settings.json` with
|
||||||
|
/// `{"mcpServers":{}}` — workspace settings override user settings, so all 6 MCP
|
||||||
|
/// servers are skipped at subprocess startup (major speedup).
|
||||||
|
///
|
||||||
|
/// The workdir is created once at construction and reused for every call.
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use tokio::process::Command;
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
use tracing::debug;
|
||||||
|
|
||||||
|
use crate::clean::strip_thinking_tags;
|
||||||
|
use crate::error::LlmError;
|
||||||
|
use crate::provider::{CompletionRequest, LlmProvider};
|
||||||
|
|
||||||
|
/// Maximum concurrent Gemini subprocess calls.
|
||||||
|
const MAX_CONCURRENT: usize = 6;
|
||||||
|
/// Subprocess deadline — prevents hung `gemini` processes blocking the chain.
|
||||||
|
const SUBPROCESS_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
|
/// Fixed workdir used for every subprocess call.
|
||||||
|
/// A workspace-level `.gemini/settings.json` here overrides the user's MCP server config.
|
||||||
|
const NOXA_GEMINI_WORKDIR: &str = "/tmp/noxa-gemini";
|
||||||
|
|
||||||
|
pub struct GeminiCliProvider {
|
||||||
|
default_model: String,
|
||||||
|
semaphore: Arc<Semaphore>,
|
||||||
|
/// Workdir with a minimal `.gemini/settings.json` that disables MCP servers.
|
||||||
|
workdir: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GeminiCliProvider {
|
||||||
|
/// Construct the provider.
|
||||||
|
/// Model resolves as: `model` arg → `GEMINI_MODEL` env → `"gemini-2.5-pro"`.
|
||||||
|
pub fn new(model: Option<String>) -> Self {
|
||||||
|
let default_model = model
|
||||||
|
.or_else(|| std::env::var("GEMINI_MODEL").ok())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or_else(|| "gemini-2.5-pro".into());
|
||||||
|
|
||||||
|
let workdir = PathBuf::from(NOXA_GEMINI_WORKDIR);
|
||||||
|
ensure_gemini_workdir(&workdir);
|
||||||
|
|
||||||
|
Self {
|
||||||
|
default_model,
|
||||||
|
semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT)),
|
||||||
|
workdir,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn default_model(&self) -> &str {
|
||||||
|
&self.default_model
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl LlmProvider for GeminiCliProvider {
|
||||||
|
async fn complete(&self, request: &CompletionRequest) -> Result<String, LlmError> {
|
||||||
|
let model = if request.model.is_empty() {
|
||||||
|
&self.default_model
|
||||||
|
} else {
|
||||||
|
&request.model
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build the prompt text from all messages.
|
||||||
|
let prompt = build_prompt(&request.messages);
|
||||||
|
|
||||||
|
// Acquire concurrency slot before spawning.
|
||||||
|
let _permit = self
|
||||||
|
.semaphore
|
||||||
|
.acquire()
|
||||||
|
.await
|
||||||
|
.map_err(|_| LlmError::ProviderError("gemini semaphore closed".into()))?;
|
||||||
|
|
||||||
|
let mut cmd = Command::new("gemini");
|
||||||
|
// -p STRING: headless mode with prompt as the flag value (never positional arg).
|
||||||
|
// Passing via -p prevents command injection; the value is never interpreted as a shell command.
|
||||||
|
cmd.arg("-p").arg(&prompt);
|
||||||
|
cmd.arg("--model").arg(model);
|
||||||
|
// Always request structured JSON output so we can extract the `response` field
|
||||||
|
// and skip any preceding noise lines (e.g. MCP status warnings).
|
||||||
|
cmd.arg("--output-format").arg("json");
|
||||||
|
// --yolo suppresses any interactive confirmation prompts in headless mode.
|
||||||
|
cmd.arg("--yolo");
|
||||||
|
// --extensions "" skips loading user extensions (~3 s startup savings).
|
||||||
|
cmd.arg("--extensions").arg("");
|
||||||
|
// Workspace settings in self.workdir override the user's ~/.gemini/settings.json,
|
||||||
|
// replacing the user's MCP server list with {} so none are spawned at startup.
|
||||||
|
// Without this, each of the user's MCP servers adds latency to every call.
|
||||||
|
cmd.current_dir(&self.workdir);
|
||||||
|
|
||||||
|
cmd.stdin(std::process::Stdio::null());
|
||||||
|
cmd.stdout(std::process::Stdio::piped());
|
||||||
|
cmd.stderr(std::process::Stdio::piped());
|
||||||
|
|
||||||
|
debug!(model, workdir = %self.workdir.display(), "spawning gemini subprocess");
|
||||||
|
|
||||||
|
let child = cmd.spawn().map_err(LlmError::Subprocess)?;
|
||||||
|
|
||||||
|
// Bounded wait — prevents indefinite hangs on auth expiry or network stall.
|
||||||
|
let output = match timeout(SUBPROCESS_TIMEOUT, child.wait_with_output()).await {
|
||||||
|
Ok(Ok(out)) => out,
|
||||||
|
Ok(Err(e)) => return Err(LlmError::Subprocess(e)),
|
||||||
|
Err(_elapsed) => return Err(LlmError::Timeout),
|
||||||
|
};
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
let stderr_preview = String::from_utf8_lossy(&output.stderr);
|
||||||
|
let preview = &stderr_preview[..stderr_preview.len().min(500)];
|
||||||
|
return Err(LlmError::ProviderError(format!(
|
||||||
|
"gemini exited with {}: {preview}",
|
||||||
|
output.status
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
|
let response = extract_response_from_output(&stdout)?;
|
||||||
|
let cleaned = strip_code_fences(strip_thinking_tags(&response).trim());
|
||||||
|
Ok(cleaned)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn is_available(&self) -> bool {
|
||||||
|
// Pure PATH check — no inference call, fast.
|
||||||
|
matches!(
|
||||||
|
Command::new("gemini")
|
||||||
|
.arg("--version")
|
||||||
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::null())
|
||||||
|
.status()
|
||||||
|
.await,
|
||||||
|
Ok(s) if s.success()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
"gemini"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the `response` field from gemini's `--output-format json` output.
|
||||||
|
///
|
||||||
|
/// The CLI emits lines before the JSON object (e.g. MCP status warnings).
|
||||||
|
/// We find the first `{` to locate the JSON, parse it, and extract `.response`.
|
||||||
|
fn extract_response_from_output(stdout: &str) -> Result<String, LlmError> {
|
||||||
|
let json_start = stdout.find('{').ok_or_else(|| {
|
||||||
|
let preview = &stdout[..stdout.len().min(300)];
|
||||||
|
LlmError::ProviderError(format!("gemini produced no JSON output: {preview}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let json_str = &stdout[json_start..];
|
||||||
|
let outer: serde_json::Value = serde_json::from_str(json_str).map_err(|e| {
|
||||||
|
let preview = &json_str[..json_str.len().min(300)];
|
||||||
|
LlmError::ProviderError(format!("failed to parse gemini JSON output: {e} — {preview}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// `response` holds the model's actual text output.
|
||||||
|
outer["response"]
|
||||||
|
.as_str()
|
||||||
|
.ok_or_else(|| {
|
||||||
|
LlmError::ProviderError(format!(
|
||||||
|
"gemini JSON output missing 'response' field: {}",
|
||||||
|
&json_str[..json_str.len().min(300)]
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create the noxa gemini workdir with a minimal workspace settings file.
|
||||||
|
///
|
||||||
|
/// The `.gemini/settings.json` written here overrides the user's `~/.gemini/settings.json`
|
||||||
|
/// for any `gemini` subprocess run from this directory. Setting `mcpServers` to `{}` prevents
|
||||||
|
/// the CLI from spawning the user's configured MCP servers on every headless call.
|
||||||
|
///
|
||||||
|
/// Errors are intentionally ignored — if the write fails, the subprocess still works,
|
||||||
|
/// just without the startup optimization (and with a warning in the logs).
|
||||||
|
fn ensure_gemini_workdir(workdir: &std::path::Path) {
|
||||||
|
let settings_dir = workdir.join(".gemini");
|
||||||
|
let settings_path = settings_dir.join("settings.json");
|
||||||
|
|
||||||
|
if settings_path.exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = std::fs::create_dir_all(&settings_dir) {
|
||||||
|
tracing::warn!(path = %settings_dir.display(), error = %e, "failed to create gemini workdir");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Minimal workspace settings: disable all MCP servers.
|
||||||
|
// Workspace settings override ~/.gemini/settings.json per gemini CLI docs.
|
||||||
|
let content = r#"{"mcpServers":{}}"#;
|
||||||
|
if let Err(e) = std::fs::write(&settings_path, content) {
|
||||||
|
tracing::warn!(path = %settings_path.display(), error = %e, "failed to write gemini workspace settings");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Concatenate all messages into a single prompt string for the CLI.
|
||||||
|
fn build_prompt(messages: &[crate::provider::Message]) -> String {
|
||||||
|
messages
|
||||||
|
.iter()
|
||||||
|
.map(|m| match m.role.as_str() {
|
||||||
|
"system" => format!("[System]: {}", m.content),
|
||||||
|
"assistant" => format!("[Assistant]: {}", m.content),
|
||||||
|
_ => m.content.clone(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strip markdown code fences from a response string.
|
||||||
|
fn strip_code_fences(s: &str) -> String {
|
||||||
|
let trimmed = s.trim();
|
||||||
|
if trimmed.starts_with("```") {
|
||||||
|
let without_opener = trimmed
|
||||||
|
.strip_prefix("```json")
|
||||||
|
.or_else(|| trimmed.strip_prefix("```"))
|
||||||
|
.unwrap_or(trimmed);
|
||||||
|
without_opener
|
||||||
|
.strip_suffix("```")
|
||||||
|
.unwrap_or(without_opener)
|
||||||
|
.trim()
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
trimmed.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
// ── Construction ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_model_used() {
|
||||||
|
let p = GeminiCliProvider::new(Some("gemini-1.5-flash".into()));
|
||||||
|
assert_eq!(p.default_model(), "gemini-1.5-flash");
|
||||||
|
assert_eq!(p.name(), "gemini");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_model_fallback() {
|
||||||
|
// Explicit None + no GEMINI_MODEL env → hardcoded default.
|
||||||
|
// We unset the env to avoid flakiness (it may or may not be set).
|
||||||
|
unsafe { std::env::remove_var("GEMINI_MODEL") };
|
||||||
|
let p = GeminiCliProvider::new(None);
|
||||||
|
assert_eq!(p.default_model(), "gemini-2.5-pro");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Env var tests mutate process-global state and race with parallel tests.
|
||||||
|
// Run in isolation if needed:
|
||||||
|
// cargo test -p noxa-llm env_model_override -- --ignored --test-threads=1
|
||||||
|
#[test]
|
||||||
|
#[ignore = "mutates process env; run with --test-threads=1"]
|
||||||
|
fn env_model_override() {
|
||||||
|
unsafe { std::env::set_var("GEMINI_MODEL", "gemini-1.5-pro") };
|
||||||
|
let p = GeminiCliProvider::new(None);
|
||||||
|
assert_eq!(p.default_model(), "gemini-1.5-pro");
|
||||||
|
unsafe { std::env::remove_var("GEMINI_MODEL") };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── build_prompt ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_prompt_user_only() {
|
||||||
|
use crate::provider::Message;
|
||||||
|
let messages = vec![Message {
|
||||||
|
role: "user".into(),
|
||||||
|
content: "hello world".into(),
|
||||||
|
}];
|
||||||
|
assert_eq!(build_prompt(&messages), "hello world");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_prompt_system_and_user() {
|
||||||
|
use crate::provider::Message;
|
||||||
|
let messages = vec![
|
||||||
|
Message {
|
||||||
|
role: "system".into(),
|
||||||
|
content: "You are helpful.".into(),
|
||||||
|
},
|
||||||
|
Message {
|
||||||
|
role: "user".into(),
|
||||||
|
content: "Tell me something.".into(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let result = build_prompt(&messages);
|
||||||
|
assert!(result.contains("[System]: You are helpful."));
|
||||||
|
assert!(result.contains("Tell me something."));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── extract_response_from_output ──────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_response_from_clean_json() {
|
||||||
|
let stdout = r#"{"session_id":"abc","response":"Hello world","stats":{}}"#;
|
||||||
|
assert_eq!(extract_response_from_output(stdout).unwrap(), "Hello world");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_response_skipping_mcp_noise() {
|
||||||
|
// MCP warning line appears before the JSON object in real gemini output.
|
||||||
|
let stdout = "MCP issues detected. Run /mcp list for status.\n{\"session_id\":\"abc\",\"response\":\"the answer\",\"stats\":{}}";
|
||||||
|
assert_eq!(
|
||||||
|
extract_response_from_output(stdout).unwrap(),
|
||||||
|
"the answer"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn error_when_no_json_in_output() {
|
||||||
|
let result = extract_response_from_output("MCP issues detected. No JSON follows.");
|
||||||
|
assert!(matches!(result, Err(LlmError::ProviderError(_))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn error_when_response_field_missing() {
|
||||||
|
let stdout = r#"{"session_id":"abc","stats":{}}"#;
|
||||||
|
let result = extract_response_from_output(stdout);
|
||||||
|
assert!(matches!(result, Err(LlmError::ProviderError(_))));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── strip_code_fences ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strips_json_fence() {
|
||||||
|
let input = "```json\n{\"key\": \"value\"}\n```";
|
||||||
|
assert_eq!(strip_code_fences(input), "{\"key\": \"value\"}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strips_plain_fence() {
|
||||||
|
let input = "```\nhello\n```";
|
||||||
|
assert_eq!(strip_code_fences(input), "hello");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn passthrough_no_fence() {
|
||||||
|
let input = "{\"key\": \"value\"}";
|
||||||
|
assert_eq!(strip_code_fences(input), "{\"key\": \"value\"}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── is_available returns false when binary absent ──────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unavailable_when_binary_missing() {
|
||||||
|
let result = tokio::process::Command::new("__noxa_nonexistent_binary_xyz__")
|
||||||
|
.arg("--version")
|
||||||
|
.stdout(std::process::Stdio::null())
|
||||||
|
.stderr(std::process::Stdio::null())
|
||||||
|
.status()
|
||||||
|
.await;
|
||||||
|
assert!(result.is_err(), "missing binary should fail to spawn");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── thinking tag stripping ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strips_thinking_tags_from_output() {
|
||||||
|
let raw = "<think>internal reasoning</think>{\"result\": true}";
|
||||||
|
let after_thinking = strip_thinking_tags(raw);
|
||||||
|
let after_fences = strip_code_fences(after_thinking.trim());
|
||||||
|
assert_eq!(after_fences, "{\"result\": true}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strips_code_fence_after_thinking() {
|
||||||
|
let raw = "<think>let me check</think>\n```json\n{\"ok\": 1}\n```";
|
||||||
|
let after_thinking = strip_thinking_tags(raw);
|
||||||
|
let after_fences = strip_code_fences(after_thinking.trim());
|
||||||
|
assert_eq!(after_fences, "{\"ok\": 1}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
pub mod anthropic;
|
pub mod anthropic;
|
||||||
|
pub mod gemini_cli;
|
||||||
pub mod ollama;
|
pub mod ollama;
|
||||||
pub mod openai;
|
pub mod openai;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
/// First choice in the provider chain: free, private, fast on Apple Silicon.
|
/// First choice in the provider chain: free, private, fast on Apple Silicon.
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::clean::strip_thinking_tags;
|
use crate::clean::strip_thinking_tags;
|
||||||
use crate::error::LlmError;
|
use crate::error::LlmError;
|
||||||
|
|
@ -96,7 +97,10 @@ impl LlmProvider for OllamaProvider {
|
||||||
|
|
||||||
async fn is_available(&self) -> bool {
|
async fn is_available(&self) -> bool {
|
||||||
let url = format!("{}/api/tags", self.base_url);
|
let url = format!("{}/api/tags", self.base_url);
|
||||||
matches!(self.client.get(&url).send().await, Ok(r) if r.status().is_success())
|
matches!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(500), self.client.get(&url).send()).await,
|
||||||
|
Ok(Ok(r)) if r.status().is_success()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn name(&self) -> &str {
|
fn name(&self) -> &str {
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,9 @@
|
||||||
/// extract, chain, and other modules that need a fake LLM backend.
|
/// extract, chain, and other modules that need a fake LLM backend.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) mod mock {
|
pub(crate) mod mock {
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use crate::error::LlmError;
|
use crate::error::LlmError;
|
||||||
|
|
@ -45,4 +48,48 @@ pub(crate) mod mock {
|
||||||
self.name
|
self.name
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A mock provider that returns responses from a sequence.
|
||||||
|
/// Call N → returns responses[N], wrapping at the end.
|
||||||
|
/// Useful for testing first-failure / second-success retry paths.
|
||||||
|
pub struct SequenceMockProvider {
|
||||||
|
pub name: &'static str,
|
||||||
|
pub responses: Vec<Result<String, String>>,
|
||||||
|
pub available: bool,
|
||||||
|
call_count: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SequenceMockProvider {
|
||||||
|
pub fn new(
|
||||||
|
name: &'static str,
|
||||||
|
responses: Vec<Result<String, String>>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
name,
|
||||||
|
responses,
|
||||||
|
available: true,
|
||||||
|
call_count: Arc::new(AtomicUsize::new(0)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl LlmProvider for SequenceMockProvider {
|
||||||
|
async fn complete(&self, _request: &CompletionRequest) -> Result<String, LlmError> {
|
||||||
|
let idx = self.call_count.fetch_add(1, Ordering::SeqCst);
|
||||||
|
let response = &self.responses[idx.min(self.responses.len() - 1)];
|
||||||
|
match response {
|
||||||
|
Ok(text) => Ok(text.clone()),
|
||||||
|
Err(msg) => Err(LlmError::ProviderError(msg.clone())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn is_available(&self) -> bool {
|
||||||
|
self.available
|
||||||
|
}
|
||||||
|
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
self.name
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,10 @@ version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
name = "noxa_mcp"
|
||||||
|
path = "src/lib.rs"
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "noxa-mcp"
|
name = "noxa-mcp"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
@ -14,8 +18,8 @@ noxa-core = { workspace = true }
|
||||||
noxa-fetch = { workspace = true }
|
noxa-fetch = { workspace = true }
|
||||||
noxa-llm = { workspace = true }
|
noxa-llm = { workspace = true }
|
||||||
noxa-pdf = { workspace = true }
|
noxa-pdf = { workspace = true }
|
||||||
rmcp = { version = "1.2", features = ["server", "macros", "transport-io", "schemars"] }
|
rmcp = { workspace = true }
|
||||||
schemars = "1.0"
|
schemars = { workspace = true }
|
||||||
dotenvy = { workspace = true }
|
dotenvy = { workspace = true }
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
|
@ -24,4 +28,4 @@ tracing = { workspace = true }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||||
url = "2"
|
url = "2"
|
||||||
dirs = "6.0.0"
|
dirs = { workspace = true }
|
||||||
|
|
|
||||||
20
crates/noxa-mcp/src/lib.rs
Normal file
20
crates/noxa-mcp/src/lib.rs
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
/// noxa-mcp library wrapper.
|
||||||
|
///
|
||||||
|
/// This exposes the MCP server so it can be embedded by the `noxa` CLI via
|
||||||
|
/// `noxa mcp` without duplicating the transport/bootstrap code.
|
||||||
|
///
|
||||||
|
/// Callers must initialize tracing before calling `run()`. Stdout must remain
|
||||||
|
/// untouched after `run()` begins because it carries the MCP wire protocol.
|
||||||
|
pub(crate) mod cloud;
|
||||||
|
pub(crate) mod server;
|
||||||
|
pub(crate) mod tools;
|
||||||
|
|
||||||
|
use rmcp::ServiceExt;
|
||||||
|
use rmcp::transport::stdio;
|
||||||
|
|
||||||
|
/// Start the MCP server over stdio and block until the client disconnects.
|
||||||
|
pub async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let service = server::NoxaMcp::new().await.serve(stdio()).await?;
|
||||||
|
service.waiting().await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -1,15 +1,6 @@
|
||||||
/// noxa-mcp: MCP (Model Context Protocol) server for noxa.
|
/// noxa-mcp: MCP (Model Context Protocol) server for noxa.
|
||||||
/// Exposes web extraction tools over stdio transport for AI agents
|
/// Exposes web extraction tools over stdio transport for AI agents
|
||||||
/// like Claude Desktop, Claude Code, and other MCP clients.
|
/// like Claude Desktop, Claude Code, and other MCP clients.
|
||||||
mod cloud;
|
|
||||||
mod server;
|
|
||||||
mod tools;
|
|
||||||
|
|
||||||
use rmcp::ServiceExt;
|
|
||||||
use rmcp::transport::stdio;
|
|
||||||
|
|
||||||
use server::NoxaMcp;
|
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
dotenvy::dotenv().ok();
|
dotenvy::dotenv().ok();
|
||||||
|
|
@ -21,8 +12,5 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
let service = NoxaMcp::new().await.serve(stdio()).await?;
|
noxa_mcp::run().await
|
||||||
|
|
||||||
service.waiting().await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,7 @@ impl NoxaMcp {
|
||||||
|
|
||||||
let chain = noxa_llm::ProviderChain::default().await;
|
let chain = noxa_llm::ProviderChain::default().await;
|
||||||
let llm_chain = if chain.is_empty() {
|
let llm_chain = if chain.is_empty() {
|
||||||
warn!("no LLM providers available -- extract/summarize tools will fail");
|
warn!("no LLM providers available (gemini CLI, OPENAI_API_KEY, ANTHROPIC_API_KEY) -- extract/summarize tools will fail");
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
info!(providers = chain.len(), "LLM provider chain ready");
|
info!(providers = chain.len(), "LLM provider chain ready");
|
||||||
|
|
@ -334,7 +334,7 @@ impl NoxaMcp {
|
||||||
// No local LLM — fall back to cloud API directly
|
// No local LLM — fall back to cloud API directly
|
||||||
if self.llm_chain.is_none() {
|
if self.llm_chain.is_none() {
|
||||||
let cloud = self.cloud.as_ref().ok_or(
|
let cloud = self.cloud.as_ref().ok_or(
|
||||||
"No LLM providers available. Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or NOXA_API_KEY for cloud fallback.",
|
"No LLM providers available. Install the gemini CLI, set OPENAI_API_KEY, ANTHROPIC_API_KEY, or NOXA_API_KEY for cloud fallback.",
|
||||||
)?;
|
)?;
|
||||||
let mut body = json!({"url": params.url});
|
let mut body = json!({"url": params.url});
|
||||||
if let Some(ref schema) = params.schema {
|
if let Some(ref schema) = params.schema {
|
||||||
|
|
@ -387,7 +387,7 @@ impl NoxaMcp {
|
||||||
// No local LLM — fall back to cloud API directly
|
// No local LLM — fall back to cloud API directly
|
||||||
if self.llm_chain.is_none() {
|
if self.llm_chain.is_none() {
|
||||||
let cloud = self.cloud.as_ref().ok_or(
|
let cloud = self.cloud.as_ref().ok_or(
|
||||||
"No LLM providers available. Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or NOXA_API_KEY for cloud fallback.",
|
"No LLM providers available. Install the gemini CLI, set OPENAI_API_KEY, ANTHROPIC_API_KEY, or NOXA_API_KEY for cloud fallback.",
|
||||||
)?;
|
)?;
|
||||||
let mut body = json!({"url": params.url});
|
let mut body = json!({"url": params.url});
|
||||||
if let Some(sentences) = params.max_sentences {
|
if let Some(sentences) = params.max_sentences {
|
||||||
|
|
|
||||||
51
env.example
51
env.example
|
|
@ -1,43 +1,20 @@
|
||||||
# ============================================
|
# Secrets, URLs, and path overrides only — everything else goes in config.json
|
||||||
# Noxa Configuration
|
# See config.example.json for the full list of configurable defaults.
|
||||||
# Copy to .env and fill in your values
|
|
||||||
# ============================================
|
|
||||||
|
|
||||||
# --- LLM Providers ---
|
# Cloud API key (required for --cloud / --research)
|
||||||
|
NOXA_API_KEY=
|
||||||
|
|
||||||
# Ollama (local, default provider)
|
# Single proxy URL (or use NOXA_PROXY_FILE for pool rotation)
|
||||||
OLLAMA_HOST=http://localhost:11434
|
NOXA_PROXY=
|
||||||
OLLAMA_MODEL=qwen3:8b
|
|
||||||
|
|
||||||
# OpenAI (optional cloud fallback)
|
# Proxy pool file path for rotating proxies
|
||||||
# OPENAI_API_KEY — set your OpenAI key
|
NOXA_PROXY_FILE=
|
||||||
# OPENAI_BASE_URL — defaults to https://api.openai.com/v1
|
|
||||||
# OPENAI_MODEL — defaults to gpt-4o-mini
|
|
||||||
|
|
||||||
# Anthropic (optional cloud fallback)
|
# Webhook URL for completion notifications
|
||||||
# ANTHROPIC_API_KEY — set your Anthropic key
|
NOXA_WEBHOOK_URL=
|
||||||
# ANTHROPIC_MODEL — defaults to claude-sonnet-4-20250514
|
|
||||||
|
|
||||||
# --- Proxy ---
|
# LLM base URL (Ollama or OpenAI-compatible endpoint)
|
||||||
|
NOXA_LLM_BASE_URL=
|
||||||
|
|
||||||
# Single proxy
|
# Optional: path to a non-default config file (default: ./config.json)
|
||||||
# NOXA_PROXY=http://user:pass@host:port
|
# NOXA_CONFIG=/path/to/my-config.json
|
||||||
|
|
||||||
# Proxy file (one per line: host:port:user:pass)
|
|
||||||
# NOXA_PROXY_FILE=/path/to/proxies.txt
|
|
||||||
|
|
||||||
# --- Server (noxa-server only) ---
|
|
||||||
# NOXA_PORT=3000
|
|
||||||
# NOXA_HOST=0.0.0.0
|
|
||||||
# NOXA_AUTH_KEY=your-auth-key
|
|
||||||
# NOXA_MAX_CONCURRENCY=50
|
|
||||||
# NOXA_JOB_TTL_SECS=3600
|
|
||||||
# NOXA_MAX_JOBS=100
|
|
||||||
|
|
||||||
# --- CLI LLM overrides ---
|
|
||||||
# NOXA_LLM_PROVIDER=ollama
|
|
||||||
# NOXA_LLM_MODEL=qwen3:8b
|
|
||||||
# NOXA_LLM_BASE_URL=http://localhost:11434
|
|
||||||
|
|
||||||
# --- Logging ---
|
|
||||||
# NOXA_LOG=info
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue